mirror of
https://github.com/memohai/Memoh.git
synced 2026-04-25 07:00:48 +09:00
refactor: memory provider (#140)
* refactor: memory provider * fix: migrations * feat: divide collection from different built-in memory * feat: add `MEMORY.md` and `PROFILES.md` * use .env for docker compose. fix #142 (#143) * feat(web): add brand icons for search providers (#144) Add custom FontAwesome icon definitions for all 9 search providers: - Yandex: uses existing faYandex from FA free brands - Tavily, Jina, Exa, Bocha, Serper: custom icons from brand SVGs - DuckDuckGo, SearXNG, Sogou: custom icons from Simple Icons Icons are registered with a custom 'fac' prefix and rendered as monochrome (currentColor) via FontAwesome's standard rendering. * fix: resolve multiple UI bugs (#147) * feat: add email service with multi-adapter support (#146) * feat: add email service with multi-adapter support Implement a full-stack email service with global provider management, per-bot bindings with granular read/write permissions, outbox audit storage, and MCP tool integration for direct mailbox access. Backend: - Email providers: CRUD with dynamic config schema (generic SMTP/IMAP, Mailgun) - Generic adapter: go-mail (SMTP) + go-imap/v2 (IMAP IDLE real-time push via UnilateralDataHandler + UID-based tracking + periodic check fallback) - Mailgun adapter: mailgun-go/v5 with dual inbound mode (webhook + poll) - Bot email bindings: per-bot provider binding with independent r/w permissions - Outbox: outbound email audit log with status tracking - Trigger: inbound emails push notification to bot_inbox (from/subject only, LLM reads full content on demand via MCP tools) - MailboxReader interface: on-demand IMAP queries for listing/reading emails - MCP tools: email_accounts, email_send, email_list (paginated mailbox), email_read (by UID) — all with multi-binding and provider_id selection - Webhook: /email/mailgun/webhook/:config_id (JWT-skipped, signature-verified) - DB migration: 0019_add_email (email_providers, bot_email_bindings, email_outbox) Frontend: - Email Providers page: /email-providers with MasterDetailSidebarLayout - Dynamic config form rendered from ordered provider meta schema with i18n keys - Bot detail: Email tab with bindings management + outbox audit table - Sidebar navigation entry - Full i18n support (en + zh) - Auto-generated SDK from Swagger Closes #17 * feat(email): trigger bot conversation immediately on inbound email Instead of only storing an inbox item and waiting for the next chat, the email trigger now proactively invokes the conversation resolver so the bot processes new emails right away — aligned with the schedule/heartbeat trigger pattern. * fix: lint --------- Co-authored-by: Acbox <acbox0328@gmail.com> * chore: update AGENTS.md * feat: files preview * feat(web): improve MCP details page * refactor(skills): import skill with pure markdown string * merge main into refactor/memory * fix: migration * refactor: temp delete qdrant and bm25 index * fix: clean merge code * fix: update memory handler --------- Co-authored-by: Leohearts <leohearts@leohearts.com> Co-authored-by: Menci <mencici@msn.com> Co-authored-by: Quincy <69751197+dqygit@users.noreply.github.com> Co-authored-by: BBQ <35603386+HoneyBBQ@users.noreply.github.com> Co-authored-by: Ran <16112591+chen-ran@users.noreply.github.com>
This commit is contained in:
Acbox Liu
@@ -109,6 +109,17 @@ CREATE TABLE IF NOT EXISTS model_variants (
|
||||
CREATE INDEX IF NOT EXISTS idx_model_variants_model_uuid ON model_variants(model_uuid);
|
||||
CREATE INDEX IF NOT EXISTS idx_model_variants_variant_id ON model_variants(variant_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS memory_providers (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name TEXT NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
config JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
is_default BOOLEAN NOT NULL DEFAULT false,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT memory_providers_name_unique UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bots (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
owner_user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
@@ -125,9 +136,8 @@ CREATE TABLE IF NOT EXISTS bots (
|
||||
reasoning_effort TEXT NOT NULL DEFAULT 'medium',
|
||||
max_inbox_items INTEGER NOT NULL DEFAULT 50,
|
||||
chat_model_id UUID REFERENCES models(id) ON DELETE SET NULL,
|
||||
memory_model_id UUID REFERENCES models(id) ON DELETE SET NULL,
|
||||
embedding_model_id UUID REFERENCES models(id) ON DELETE SET NULL,
|
||||
search_provider_id UUID REFERENCES search_providers(id) ON DELETE SET NULL,
|
||||
memory_provider_id UUID REFERENCES memory_providers(id) ON DELETE SET NULL,
|
||||
heartbeat_enabled BOOLEAN NOT NULL DEFAULT false,
|
||||
heartbeat_interval INTEGER NOT NULL DEFAULT 30,
|
||||
heartbeat_prompt TEXT NOT NULL DEFAULT '',
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
-- 0020_memory_providers (rollback)
|
||||
|
||||
ALTER TABLE bots ADD COLUMN IF NOT EXISTS memory_model_id UUID REFERENCES models(id) ON DELETE SET NULL;
|
||||
ALTER TABLE bots ADD COLUMN IF NOT EXISTS embedding_model_id UUID REFERENCES models(id) ON DELETE SET NULL;
|
||||
ALTER TABLE bots DROP COLUMN IF EXISTS memory_provider_id;
|
||||
DROP TABLE IF EXISTS memory_providers;
|
||||
@@ -0,0 +1,50 @@
|
||||
-- 0020_memory_providers
|
||||
-- Add memory_providers table, migrate bot memory/embedding model into provider config,
|
||||
-- and drop the now-redundant columns from bots.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS memory_providers (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name TEXT NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
config JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
is_default BOOLEAN NOT NULL DEFAULT false,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT memory_providers_name_unique UNIQUE (name)
|
||||
);
|
||||
|
||||
ALTER TABLE bots ADD COLUMN IF NOT EXISTS memory_provider_id UUID REFERENCES memory_providers(id) ON DELETE SET NULL;
|
||||
|
||||
-- Migrate: create a default builtin provider with existing model IDs, then link bots to it.
|
||||
-- Guard: only reference old columns if they actually exist (fresh installs won't have them).
|
||||
-- Uses dynamic SQL (EXECUTE) so PL/pgSQL doesn't validate column names at parse time.
|
||||
DO $$
|
||||
DECLARE
|
||||
_provider_id UUID;
|
||||
_has_old_cols BOOLEAN;
|
||||
_any_set BOOLEAN;
|
||||
BEGIN
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'bots' AND column_name = 'memory_model_id'
|
||||
) INTO _has_old_cols;
|
||||
|
||||
IF _has_old_cols THEN
|
||||
EXECUTE 'SELECT EXISTS (SELECT 1 FROM bots WHERE memory_model_id IS NOT NULL OR embedding_model_id IS NOT NULL)'
|
||||
INTO _any_set;
|
||||
|
||||
IF _any_set THEN
|
||||
INSERT INTO memory_providers (name, provider, config, is_default)
|
||||
VALUES ('Built-in Memory', 'builtin', '{}'::jsonb, true)
|
||||
ON CONFLICT (name) DO UPDATE SET updated_at = now()
|
||||
RETURNING id INTO _provider_id;
|
||||
|
||||
EXECUTE 'UPDATE bots SET memory_provider_id = $1 WHERE memory_model_id IS NOT NULL OR embedding_model_id IS NOT NULL'
|
||||
USING _provider_id;
|
||||
END IF;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- Drop the old columns (safe even if they don't exist).
|
||||
ALTER TABLE bots DROP COLUMN IF EXISTS memory_model_id;
|
||||
ALTER TABLE bots DROP COLUMN IF EXISTS embedding_model_id;
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
-- 0020_add_model_id_tracking (rollback)
|
||||
-- 0021_add_model_id_tracking (rollback)
|
||||
-- Remove model_id column from bot_history_messages and bot_heartbeat_logs
|
||||
|
||||
ALTER TABLE bot_heartbeat_logs
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
-- 0020_add_model_id_tracking
|
||||
-- 0021_add_model_id_tracking
|
||||
-- Add model_id column to bot_history_messages and bot_heartbeat_logs for per-model usage tracking
|
||||
|
||||
ALTER TABLE bot_history_messages
|
||||
+6
-6
@@ -1,21 +1,21 @@
|
||||
-- name: CreateBot :one
|
||||
INSERT INTO bots (owner_user_id, type, display_name, avatar_url, is_active, metadata, status)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
RETURNING id, owner_user_id, type, display_name, avatar_url, is_active, status, max_context_load_time, max_context_tokens, max_inbox_items, language, allow_guest, reasoning_enabled, reasoning_effort, chat_model_id, memory_model_id, embedding_model_id, search_provider_id, heartbeat_enabled, heartbeat_interval, heartbeat_prompt, metadata, created_at, updated_at;
|
||||
RETURNING id, owner_user_id, type, display_name, avatar_url, is_active, status, max_context_load_time, max_context_tokens, max_inbox_items, language, allow_guest, reasoning_enabled, reasoning_effort, chat_model_id, search_provider_id, memory_provider_id, heartbeat_enabled, heartbeat_interval, heartbeat_prompt, metadata, created_at, updated_at;
|
||||
|
||||
-- name: GetBotByID :one
|
||||
SELECT id, owner_user_id, type, display_name, avatar_url, is_active, status, max_context_load_time, max_context_tokens, max_inbox_items, language, allow_guest, reasoning_enabled, reasoning_effort, chat_model_id, memory_model_id, embedding_model_id, search_provider_id, heartbeat_enabled, heartbeat_interval, heartbeat_prompt, metadata, created_at, updated_at
|
||||
SELECT id, owner_user_id, type, display_name, avatar_url, is_active, status, max_context_load_time, max_context_tokens, max_inbox_items, language, allow_guest, reasoning_enabled, reasoning_effort, chat_model_id, search_provider_id, memory_provider_id, heartbeat_enabled, heartbeat_interval, heartbeat_prompt, metadata, created_at, updated_at
|
||||
FROM bots
|
||||
WHERE id = $1;
|
||||
|
||||
-- name: ListBotsByOwner :many
|
||||
SELECT id, owner_user_id, type, display_name, avatar_url, is_active, status, max_context_load_time, max_context_tokens, max_inbox_items, language, allow_guest, reasoning_enabled, reasoning_effort, chat_model_id, memory_model_id, embedding_model_id, search_provider_id, heartbeat_enabled, heartbeat_interval, heartbeat_prompt, metadata, created_at, updated_at
|
||||
SELECT id, owner_user_id, type, display_name, avatar_url, is_active, status, max_context_load_time, max_context_tokens, max_inbox_items, language, allow_guest, reasoning_enabled, reasoning_effort, chat_model_id, search_provider_id, memory_provider_id, heartbeat_enabled, heartbeat_interval, heartbeat_prompt, metadata, created_at, updated_at
|
||||
FROM bots
|
||||
WHERE owner_user_id = $1
|
||||
ORDER BY created_at DESC;
|
||||
|
||||
-- name: ListBotsByMember :many
|
||||
SELECT b.id, b.owner_user_id, b.type, b.display_name, b.avatar_url, b.is_active, b.status, b.max_context_load_time, b.max_context_tokens, b.max_inbox_items, b.language, b.allow_guest, b.reasoning_enabled, b.reasoning_effort, b.chat_model_id, b.memory_model_id, b.embedding_model_id, b.search_provider_id, b.heartbeat_enabled, b.heartbeat_interval, b.heartbeat_prompt, b.metadata, b.created_at, b.updated_at
|
||||
SELECT b.id, b.owner_user_id, b.type, b.display_name, b.avatar_url, b.is_active, b.status, b.max_context_load_time, b.max_context_tokens, b.max_inbox_items, b.language, b.allow_guest, b.reasoning_enabled, b.reasoning_effort, b.chat_model_id, b.search_provider_id, b.memory_provider_id, b.heartbeat_enabled, b.heartbeat_interval, b.heartbeat_prompt, b.metadata, b.created_at, b.updated_at
|
||||
FROM bots b
|
||||
JOIN bot_members m ON m.bot_id = b.id
|
||||
WHERE m.user_id = $1
|
||||
@@ -29,14 +29,14 @@ SET display_name = $2,
|
||||
metadata = $5,
|
||||
updated_at = now()
|
||||
WHERE id = $1
|
||||
RETURNING id, owner_user_id, type, display_name, avatar_url, is_active, status, max_context_load_time, max_context_tokens, max_inbox_items, language, allow_guest, reasoning_enabled, reasoning_effort, chat_model_id, memory_model_id, embedding_model_id, search_provider_id, heartbeat_enabled, heartbeat_interval, heartbeat_prompt, metadata, created_at, updated_at;
|
||||
RETURNING id, owner_user_id, type, display_name, avatar_url, is_active, status, max_context_load_time, max_context_tokens, max_inbox_items, language, allow_guest, reasoning_enabled, reasoning_effort, chat_model_id, search_provider_id, memory_provider_id, heartbeat_enabled, heartbeat_interval, heartbeat_prompt, metadata, created_at, updated_at;
|
||||
|
||||
-- name: UpdateBotOwner :one
|
||||
UPDATE bots
|
||||
SET owner_user_id = $2,
|
||||
updated_at = now()
|
||||
WHERE id = $1
|
||||
RETURNING id, owner_user_id, type, display_name, avatar_url, is_active, status, max_context_load_time, max_context_tokens, max_inbox_items, language, allow_guest, reasoning_enabled, reasoning_effort, chat_model_id, memory_model_id, embedding_model_id, search_provider_id, heartbeat_enabled, heartbeat_interval, heartbeat_prompt, metadata, created_at, updated_at;
|
||||
RETURNING id, owner_user_id, type, display_name, avatar_url, is_active, status, max_context_load_time, max_context_tokens, max_inbox_items, language, allow_guest, reasoning_enabled, reasoning_effort, chat_model_id, search_provider_id, memory_provider_id, heartbeat_enabled, heartbeat_interval, heartbeat_prompt, metadata, created_at, updated_at;
|
||||
|
||||
-- name: UpdateBotStatus :exec
|
||||
UPDATE bots
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
-- name: ListMemoryProviders :many
|
||||
SELECT * FROM memory_providers ORDER BY created_at ASC;
|
||||
|
||||
-- name: GetMemoryProviderByID :one
|
||||
SELECT * FROM memory_providers WHERE id = $1;
|
||||
|
||||
-- name: GetDefaultMemoryProvider :one
|
||||
SELECT * FROM memory_providers WHERE is_default = true LIMIT 1;
|
||||
|
||||
-- name: CreateMemoryProvider :one
|
||||
INSERT INTO memory_providers (name, provider, config, is_default)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
RETURNING *;
|
||||
|
||||
-- name: UpdateMemoryProvider :one
|
||||
UPDATE memory_providers
|
||||
SET name = $2,
|
||||
config = $3,
|
||||
updated_at = now()
|
||||
WHERE id = $1
|
||||
RETURNING *;
|
||||
|
||||
-- name: DeleteMemoryProvider :exec
|
||||
DELETE FROM memory_providers WHERE id = $1;
|
||||
|
||||
-- name: CountMemoryProvidersByDefault :one
|
||||
SELECT COUNT(*) FROM memory_providers WHERE is_default = true;
|
||||
+10
-16
@@ -12,16 +12,14 @@ SELECT
|
||||
bots.heartbeat_interval,
|
||||
bots.heartbeat_prompt,
|
||||
chat_models.id AS chat_model_id,
|
||||
memory_models.id AS memory_model_id,
|
||||
embedding_models.id AS embedding_model_id,
|
||||
heartbeat_models.id AS heartbeat_model_id,
|
||||
search_providers.id AS search_provider_id
|
||||
search_providers.id AS search_provider_id,
|
||||
memory_providers.id AS memory_provider_id
|
||||
FROM bots
|
||||
LEFT JOIN models AS chat_models ON chat_models.id = bots.chat_model_id
|
||||
LEFT JOIN models AS memory_models ON memory_models.id = bots.memory_model_id
|
||||
LEFT JOIN models AS embedding_models ON embedding_models.id = bots.embedding_model_id
|
||||
LEFT JOIN models AS heartbeat_models ON heartbeat_models.id = bots.heartbeat_model_id
|
||||
LEFT JOIN search_providers ON search_providers.id = bots.search_provider_id
|
||||
LEFT JOIN memory_providers ON memory_providers.id = bots.memory_provider_id
|
||||
WHERE bots.id = $1;
|
||||
|
||||
-- name: UpsertBotSettings :one
|
||||
@@ -38,13 +36,12 @@ WITH updated AS (
|
||||
heartbeat_interval = sqlc.arg(heartbeat_interval),
|
||||
heartbeat_prompt = sqlc.arg(heartbeat_prompt),
|
||||
chat_model_id = COALESCE(sqlc.narg(chat_model_id)::uuid, bots.chat_model_id),
|
||||
memory_model_id = COALESCE(sqlc.narg(memory_model_id)::uuid, bots.memory_model_id),
|
||||
embedding_model_id = COALESCE(sqlc.narg(embedding_model_id)::uuid, bots.embedding_model_id),
|
||||
heartbeat_model_id = COALESCE(sqlc.narg(heartbeat_model_id)::uuid, bots.heartbeat_model_id),
|
||||
search_provider_id = COALESCE(sqlc.narg(search_provider_id)::uuid, bots.search_provider_id),
|
||||
memory_provider_id = COALESCE(sqlc.narg(memory_provider_id)::uuid, bots.memory_provider_id),
|
||||
updated_at = now()
|
||||
WHERE bots.id = sqlc.arg(id)
|
||||
RETURNING bots.id, bots.max_context_load_time, bots.max_context_tokens, bots.max_inbox_items, bots.language, bots.allow_guest, bots.reasoning_enabled, bots.reasoning_effort, bots.heartbeat_enabled, bots.heartbeat_interval, bots.heartbeat_prompt, bots.chat_model_id, bots.memory_model_id, bots.embedding_model_id, bots.heartbeat_model_id, bots.search_provider_id
|
||||
RETURNING bots.id, bots.max_context_load_time, bots.max_context_tokens, bots.max_inbox_items, bots.language, bots.allow_guest, bots.reasoning_enabled, bots.reasoning_effort, bots.heartbeat_enabled, bots.heartbeat_interval, bots.heartbeat_prompt, bots.chat_model_id, bots.heartbeat_model_id, bots.search_provider_id, bots.memory_provider_id
|
||||
)
|
||||
SELECT
|
||||
updated.id AS bot_id,
|
||||
@@ -59,16 +56,14 @@ SELECT
|
||||
updated.heartbeat_interval,
|
||||
updated.heartbeat_prompt,
|
||||
chat_models.id AS chat_model_id,
|
||||
memory_models.id AS memory_model_id,
|
||||
embedding_models.id AS embedding_model_id,
|
||||
heartbeat_models.id AS heartbeat_model_id,
|
||||
search_providers.id AS search_provider_id
|
||||
search_providers.id AS search_provider_id,
|
||||
memory_providers.id AS memory_provider_id
|
||||
FROM updated
|
||||
LEFT JOIN models AS chat_models ON chat_models.id = updated.chat_model_id
|
||||
LEFT JOIN models AS memory_models ON memory_models.id = updated.memory_model_id
|
||||
LEFT JOIN models AS embedding_models ON embedding_models.id = updated.embedding_model_id
|
||||
LEFT JOIN models AS heartbeat_models ON heartbeat_models.id = updated.heartbeat_model_id
|
||||
LEFT JOIN search_providers ON search_providers.id = updated.search_provider_id;
|
||||
LEFT JOIN search_providers ON search_providers.id = updated.search_provider_id
|
||||
LEFT JOIN memory_providers ON memory_providers.id = updated.memory_provider_id;
|
||||
|
||||
-- name: DeleteSettingsByBotID :exec
|
||||
UPDATE bots
|
||||
@@ -83,9 +78,8 @@ SET max_context_load_time = 1440,
|
||||
heartbeat_interval = 30,
|
||||
heartbeat_prompt = '',
|
||||
chat_model_id = NULL,
|
||||
memory_model_id = NULL,
|
||||
embedding_model_id = NULL,
|
||||
heartbeat_model_id = NULL,
|
||||
search_provider_id = NULL,
|
||||
memory_provider_id = NULL,
|
||||
updated_at = now()
|
||||
WHERE id = $1;
|
||||
|
||||
Reference in New Issue
Block a user