refactor: multi-provider memory adapters with scan-based builtin (#227)

* refactor: restructure memory into multi-provider adapters, remove manifest.json dependency - Rename internal/memory/provider to internal/memory/adapters with per-provider subdirectories (builtin, mem0, openviking) - Replace manifest.json-based delete/update with scan-based index from daily files - Add mem0 and openviking provider adapters with HTTP client, chat hooks, MCP tools, and CRUD - Wire provider lifecycle into registry (auto-instantiate on create, evict on update/delete) - Split docker-compose into base stack + optional overlays (qdrant, browser, mem0, openviking) - Update admin UI to support dynamic provider config schema rendering * chore(lint): fix all golangci-lint issues for clean CI * refactor(docker): replace compose overlay files with profiles * feat(memory): add built-in memory multi modes * fix(ci): golangci lint * feat(memory): edit built-in memory sparse design
2026-04-25 07:00:48 +09:00 · 2026-03-14 06:04:13 +08:00
parent 27607d582d
commit 627b673a5c
75 changed files with 8253 additions and 2107 deletions
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"log/slog"
 	"net/http"
+	"path"
 	"sort"
 	"strconv"
 	"strings"
@@ -16,8 +17,9 @@ import (

 	"github.com/memohai/memoh/internal/accounts"
 	"github.com/memohai/memoh/internal/bots"
+	"github.com/memohai/memoh/internal/config"
 	"github.com/memohai/memoh/internal/mcp/mcpclient"
-	memprovider "github.com/memohai/memoh/internal/memory/provider"
+	memprovider "github.com/memohai/memoh/internal/memory/adapters"
 	storefs "github.com/memohai/memoh/internal/memory/storefs"
 	"github.com/memohai/memoh/internal/settings"
 )
@@ -133,6 +135,7 @@ func (h *MemoryHandler) Register(e *echo.Echo) {
 	chatGroup.POST("/search", h.ChatSearch)
 	chatGroup.POST("/compact", h.ChatCompact)
 	chatGroup.POST("/rebuild", h.ChatRebuild)
+	chatGroup.GET("/status", h.ChatStatus)
 	chatGroup.GET("", h.ChatGetAll)
 	chatGroup.GET("/usage", h.ChatUsage)
 	chatGroup.DELETE("", h.ChatDelete)
@@ -156,7 +159,7 @@ func (h *MemoryHandler) checkService(ctx context.Context, botID string) (memprov
 // @Produce json
 // @Param bot_id path string true "Bot ID"
 // @Param payload body memoryAddPayload true "Memory add payload"
-// @Success 200 {object} provider.SearchResponse
+// @Success 200 {object} adapters.SearchResponse
 // @Failure 400 {object} ErrorResponse
 // @Failure 403 {object} ErrorResponse
 // @Failure 500 {object} ErrorResponse
@@ -184,12 +187,16 @@ func (h *MemoryHandler) ChatAdd(c echo.Context) error {
 	}

 	filters := buildNamespaceFilters(namespace, scopeID, payload.Filters)
+	channelIdentityID, identityErr := h.requireChannelIdentityID(c)
+	if identityErr != nil {
+		return identityErr
+	}
 	req := memprovider.AddRequest{
 		Message:          payload.Message,
 		Messages:         payload.Messages,
 		BotID:            resolvedBotID,
 		RunID:            payload.RunID,
-		Metadata:         payload.Metadata,
+		Metadata:         memprovider.MergeMetadata(payload.Metadata, memprovider.BuildProfileMetadata("", channelIdentityID, "")),
 		Filters:          filters,
 		Infer:            payload.Infer,
 		EmbeddingEnabled: payload.EmbeddingEnabled,
@@ -214,7 +221,7 @@ func (h *MemoryHandler) ChatAdd(c echo.Context) error {
 // @Produce json
 // @Param bot_id path string true "Bot ID"
 // @Param payload body memorySearchPayload true "Memory search payload"
-// @Success 200 {object} provider.SearchResponse
+// @Success 200 {object} adapters.SearchResponse
 // @Failure 400 {object} ErrorResponse
 // @Failure 403 {object} ErrorResponse
 // @Failure 404 {object} ErrorResponse
@@ -272,7 +279,7 @@ func (h *MemoryHandler) ChatSearch(c echo.Context) error {
 // @Produce json
 // @Param bot_id path string true "Bot ID"
 // @Param no_stats query bool false "Skip optional stats in memory search response"
-// @Success 200 {object} provider.SearchResponse
+// @Success 200 {object} adapters.SearchResponse
 // @Failure 400 {object} ErrorResponse
 // @Failure 403 {object} ErrorResponse
 // @Failure 500 {object} ErrorResponse
@@ -320,7 +327,7 @@ func (h *MemoryHandler) ChatGetAll(c echo.Context) error {
 // @Produce json
 // @Param bot_id path string true "Bot ID"
 // @Param payload body memoryDeletePayload false "Optional: specify memory_ids to delete; if omitted, deletes all"
-// @Success 200 {object} provider.DeleteResponse
+// @Success 200 {object} adapters.DeleteResponse
 // @Failure 400 {object} ErrorResponse
 // @Failure 403 {object} ErrorResponse
 // @Failure 500 {object} ErrorResponse
@@ -369,7 +376,7 @@ func (h *MemoryHandler) ChatDelete(c echo.Context) error {
 // @Produce json
 // @Param bot_id path string true "Bot ID"
 // @Param id path string true "Memory ID"
-// @Success 200 {object} provider.DeleteResponse
+// @Success 200 {object} adapters.DeleteResponse
 // @Failure 400 {object} ErrorResponse
 // @Failure 403 {object} ErrorResponse
 // @Failure 500 {object} ErrorResponse
@@ -411,7 +418,7 @@ func (h *MemoryHandler) ChatDeleteOne(c echo.Context) error {
 // @Produce json
 // @Param bot_id path string true "Bot ID"
 // @Param payload body memoryCompactPayload true "ratio (0,1] required; decay_days optional"
-// @Success 200 {object} provider.CompactResult
+// @Success 200 {object} adapters.CompactResult
 // @Failure 400 {object} ErrorResponse
 // @Failure 403 {object} ErrorResponse
 // @Failure 500 {object} ErrorResponse
@@ -463,7 +470,7 @@ func (h *MemoryHandler) ChatCompact(c echo.Context) error {
 // @Tags memory
 // @Produce json
 // @Param bot_id path string true "Bot ID"
-// @Success 200 {object} provider.UsageResponse
+// @Success 200 {object} adapters.UsageResponse
 // @Failure 400 {object} ErrorResponse
 // @Failure 403 {object} ErrorResponse
 // @Failure 500 {object} ErrorResponse
@@ -508,34 +515,71 @@ func (h *MemoryHandler) ChatUsage(c echo.Context) error {
 // @Tags memory
 // @Produce json
 // @Param bot_id path string true "Bot ID"
-// @Success 200 {object} provider.RebuildResult
+// @Success 200 {object} adapters.RebuildResult
 // @Failure 400 {object} ErrorResponse
 // @Failure 403 {object} ErrorResponse
+// @Failure 409 {object} ErrorResponse
 // @Failure 500 {object} ErrorResponse
 // @Failure 503 {object} ErrorResponse
 // @Router /bots/{bot_id}/memory/rebuild [post].
 func (h *MemoryHandler) ChatRebuild(c echo.Context) error {
-	if h.memoryStore == nil {
-		return echo.NewHTTPError(http.StatusServiceUnavailable, "memory filesystem not configured")
-	}
 	botID, err := h.requireBotAccess(c)
 	if err != nil {
 		return err
 	}
-	fsItems, err := h.memoryStore.ReadAllMemoryFiles(c.Request().Context(), botID)
+	provider, checkErr := h.checkService(c.Request().Context(), botID)
+	if checkErr != nil {
+		return checkErr
+	}
+	syncProvider, ok := provider.(memprovider.SourceSyncProvider)
+	if !ok {
+		return echo.NewHTTPError(http.StatusConflict, "selected memory provider does not support rebuild from markdown source")
+	}
+	status, err := syncProvider.Status(c.Request().Context(), botID)
 	if err != nil {
-		return echo.NewHTTPError(http.StatusInternalServerError, "read memory files failed: "+err.Error())
+		return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
 	}
-	if err := h.memoryStore.SyncOverview(c.Request().Context(), botID); err != nil {
-		return echo.NewHTTPError(http.StatusInternalServerError, "sync memory overview failed: "+err.Error())
+	if !status.CanManualSync {
+		return echo.NewHTTPError(http.StatusConflict, "manual sync is not available for the selected memory provider")
 	}
+	result, err := syncProvider.Rebuild(c.Request().Context(), botID)
+	if err != nil {
+		return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
+	}
+	return c.JSON(http.StatusOK, result)
+}

-	return c.JSON(http.StatusOK, memprovider.RebuildResult{
-		FsCount:       len(fsItems),
-		QdrantCount:   len(fsItems),
-		MissingCount:  0,
-		RestoredCount: 0,
-	})
+// ChatStatus godoc
+// @Summary Get memory runtime status
+// @Description Get the resolved memory runtime status for a bot, including index health and source counts
+// @Tags memory
+// @Produce json
+// @Param bot_id path string true "Bot ID"
+// @Success 200 {object} adapters.MemoryStatusResponse
+// @Failure 400 {object} ErrorResponse
+// @Failure 403 {object} ErrorResponse
+// @Failure 409 {object} ErrorResponse
+// @Failure 500 {object} ErrorResponse
+// @Failure 503 {object} ErrorResponse
+// @Router /bots/{bot_id}/memory/status [get].
+func (h *MemoryHandler) ChatStatus(c echo.Context) error {
+	botID, err := h.requireBotAccess(c)
+	if err != nil {
+		return err
+	}
+	provider, checkErr := h.checkService(c.Request().Context(), botID)
+	if checkErr != nil {
+		return checkErr
+	}
+	syncProvider, ok := provider.(memprovider.SourceSyncProvider)
+	if !ok {
+		return echo.NewHTTPError(http.StatusConflict, "selected memory provider does not expose runtime status")
+	}
+	status, err := syncProvider.Status(c.Request().Context(), botID)
+	if err != nil {
+		return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
+	}
+	return c.JSON(http.StatusOK, status)
 }

 // --- helpers ---
@@ -669,6 +713,7 @@ func (r *fileMemoryRuntime) Add(ctx context.Context, req memprovider.AddRequest)
 		Hash:      runtimeHash(text),
 		CreatedAt: now,
 		UpdatedAt: now,
+		Metadata:  req.Metadata,
 		BotID:     botID,
 	}
 	itemsToPersist := []storefs.MemoryItem{runtimeToStoreItem(item)}
@@ -864,6 +909,44 @@ func (r *fileMemoryRuntime) Usage(ctx context.Context, filters map[string]any) (
 	return usage, nil
 }

+func (*fileMemoryRuntime) Mode() string {
+	return "off"
+}
+
+func (r *fileMemoryRuntime) Status(ctx context.Context, botID string) (memprovider.MemoryStatusResponse, error) {
+	fileCount, err := r.store.CountMemoryFiles(ctx, botID)
+	if err != nil {
+		return memprovider.MemoryStatusResponse{}, err
+	}
+	items, err := r.store.ReadAllMemoryFiles(ctx, botID)
+	if err != nil {
+		return memprovider.MemoryStatusResponse{}, err
+	}
+	return memprovider.MemoryStatusResponse{
+		ProviderType:      "builtin",
+		MemoryMode:        "off",
+		CanManualSync:     false,
+		SourceDir:         path.Join(config.DefaultDataMount, "memory"),
+		OverviewPath:      path.Join(config.DefaultDataMount, "MEMORY.md"),
+		MarkdownFileCount: fileCount,
+		SourceCount:       len(items),
+	}, nil
+}
+
+func (r *fileMemoryRuntime) Rebuild(ctx context.Context, botID string) (memprovider.RebuildResult, error) {
+	items, err := r.store.ReadAllMemoryFiles(ctx, botID)
+	if err != nil {
+		return memprovider.RebuildResult{}, err
+	}
+	if err := r.store.SyncOverview(ctx, botID); err != nil {
+		return memprovider.RebuildResult{}, err
+	}
+	return memprovider.RebuildResult{
+		FsCount:      len(items),
+		StorageCount: len(items),
+	}, nil
+}
+
 func runtimeBotID(botID string, filters map[string]any) (string, error) {
 	botID = strings.TrimSpace(botID)
 	if botID == "" {
@@ -7,7 +7,7 @@ import (

 	"github.com/labstack/echo/v4"

-	memprovider "github.com/memohai/memoh/internal/memory/provider"
+	memprovider "github.com/memohai/memoh/internal/memory/adapters"
 )

 type MemoryProvidersHandler struct {
@@ -28,6 +28,7 @@ func (h *MemoryProvidersHandler) Register(e *echo.Echo) {
 	group.POST("", h.Create)
 	group.GET("", h.List)
 	group.GET("/:id", h.Get)
+	group.GET("/:id/status", h.Status)
 	group.PUT("/:id", h.Update)
 	group.DELETE("/:id", h.Delete)
 }
@@ -36,7 +37,7 @@ func (h *MemoryProvidersHandler) Register(e *echo.Echo) {
 // @Summary List memory provider metadata
 // @Description List available memory provider types and config schemas
 // @Tags memory-providers
-// @Success 200 {array} provider.ProviderMeta
+// @Success 200 {array} adapters.ProviderMeta
 // @Router /memory-providers/meta [get].
 func (h *MemoryProvidersHandler) ListMeta(c echo.Context) error {
 	return c.JSON(http.StatusOK, h.service.ListMeta(c.Request().Context()))
@@ -48,8 +49,8 @@ func (h *MemoryProvidersHandler) ListMeta(c echo.Context) error {
 // @Tags memory-providers
 // @Accept json
 // @Produce json
-// @Param request body provider.ProviderCreateRequest true "Memory provider configuration"
-// @Success 201 {object} provider.ProviderGetResponse
+// @Param request body adapters.ProviderCreateRequest true "Memory provider configuration"
+// @Success 201 {object} adapters.ProviderGetResponse
 // @Failure 400 {object} ErrorResponse
 // @Failure 500 {object} ErrorResponse
 // @Router /memory-providers [post].
@@ -76,7 +77,7 @@ func (h *MemoryProvidersHandler) Create(c echo.Context) error {
 // @Description List configured memory providers
 // @Tags memory-providers
 // @Produce json
-// @Success 200 {array} provider.ProviderGetResponse
+// @Success 200 {array} adapters.ProviderGetResponse
 // @Failure 500 {object} ErrorResponse
 // @Router /memory-providers [get].
 func (h *MemoryProvidersHandler) List(c echo.Context) error {
@@ -93,7 +94,7 @@ func (h *MemoryProvidersHandler) List(c echo.Context) error {
 // @Tags memory-providers
 // @Produce json
 // @Param id path string true "Provider ID"
-// @Success 200 {object} provider.ProviderGetResponse
+// @Success 200 {object} adapters.ProviderGetResponse
 // @Failure 400 {object} ErrorResponse
 // @Failure 404 {object} ErrorResponse
 // @Router /memory-providers/{id} [get].
@@ -109,6 +110,29 @@ func (h *MemoryProvidersHandler) Get(c echo.Context) error {
 	return c.JSON(http.StatusOK, resp)
 }

+// Status godoc
+// @Summary Get memory provider status
+// @Description Get runtime status data for a memory provider
+// @Tags memory-providers
+// @Produce json
+// @Param id path string true "Provider ID"
+// @Success 200 {object} adapters.ProviderStatusResponse
+// @Failure 400 {object} ErrorResponse
+// @Failure 404 {object} ErrorResponse
+// @Failure 500 {object} ErrorResponse
+// @Router /memory-providers/{id}/status [get].
+func (h *MemoryProvidersHandler) Status(c echo.Context) error {
+	id := strings.TrimSpace(c.Param("id"))
+	if id == "" {
+		return echo.NewHTTPError(http.StatusBadRequest, "id is required")
+	}
+	resp, err := h.service.Status(c.Request().Context(), id)
+	if err != nil {
+		return echo.NewHTTPError(http.StatusNotFound, err.Error())
+	}
+	return c.JSON(http.StatusOK, resp)
+}
+
 // Update godoc
 // @Summary Update a memory provider
 // @Description Update memory provider by ID
@@ -116,8 +140,8 @@ func (h *MemoryProvidersHandler) Get(c echo.Context) error {
 // @Accept json
 // @Produce json
 // @Param id path string true "Provider ID"
-// @Param request body provider.ProviderUpdateRequest true "Updated configuration"
-// @Success 200 {object} provider.ProviderGetResponse
+// @Param request body adapters.ProviderUpdateRequest true "Updated configuration"
+// @Success 200 {object} adapters.ProviderGetResponse
 // @Failure 400 {object} ErrorResponse
 // @Failure 500 {object} ErrorResponse
 // @Router /memory-providers/{id} [put].