Files
Memoh/internal/models/models.go
T
Acbox Liu ea719f7ca7 refactor: memory provider (#140)
* refactor: memory provider

* fix: migrations

* feat: divide collection from different built-in memory

* feat: add `MEMORY.md` and `PROFILES.md`

* use .env for docker compose. fix #142 (#143)

* feat(web): add brand icons for search providers (#144)

Add custom FontAwesome icon definitions for all 9 search providers:
- Yandex: uses existing faYandex from FA free brands
- Tavily, Jina, Exa, Bocha, Serper: custom icons from brand SVGs
- DuckDuckGo, SearXNG, Sogou: custom icons from Simple Icons

Icons are registered with a custom 'fac' prefix and rendered as
monochrome (currentColor) via FontAwesome's standard rendering.

* fix: resolve multiple UI bugs (#147)

* feat: add email service with multi-adapter support (#146)

* feat: add email service with multi-adapter support

Implement a full-stack email service with global provider management,
per-bot bindings with granular read/write permissions, outbox audit
storage, and MCP tool integration for direct mailbox access.

Backend:
- Email providers: CRUD with dynamic config schema (generic SMTP/IMAP, Mailgun)
- Generic adapter: go-mail (SMTP) + go-imap/v2 (IMAP IDLE real-time push via
  UnilateralDataHandler + UID-based tracking + periodic check fallback)
- Mailgun adapter: mailgun-go/v5 with dual inbound mode (webhook + poll)
- Bot email bindings: per-bot provider binding with independent r/w permissions
- Outbox: outbound email audit log with status tracking
- Trigger: inbound emails push notification to bot_inbox (from/subject only,
  LLM reads full content on demand via MCP tools)
- MailboxReader interface: on-demand IMAP queries for listing/reading emails
- MCP tools: email_accounts, email_send, email_list (paginated mailbox),
  email_read (by UID) — all with multi-binding and provider_id selection
- Webhook: /email/mailgun/webhook/:config_id (JWT-skipped, signature-verified)
- DB migration: 0019_add_email (email_providers, bot_email_bindings, email_outbox)

Frontend:
- Email Providers page: /email-providers with MasterDetailSidebarLayout
- Dynamic config form rendered from ordered provider meta schema with i18n keys
- Bot detail: Email tab with bindings management + outbox audit table
- Sidebar navigation entry
- Full i18n support (en + zh)
- Auto-generated SDK from Swagger

Closes #17

* feat(email): trigger bot conversation immediately on inbound email

Instead of only storing an inbox item and waiting for the next chat,
the email trigger now proactively invokes the conversation resolver
so the bot processes new emails right away — aligned with the
schedule/heartbeat trigger pattern.

* fix: lint

---------

Co-authored-by: Acbox <acbox0328@gmail.com>

* chore: update AGENTS.md

* feat: files preview

* feat(web): improve MCP details page

* refactor(skills): import skill with pure markdown string

* merge main into refactor/memory

* fix: migration

* refactor: temp delete qdrant and bm25 index

* fix: clean merge code

* fix: update memory handler

---------

Co-authored-by: Leohearts <leohearts@leohearts.com>
Co-authored-by: Menci <mencici@msn.com>
Co-authored-by: Quincy <69751197+dqygit@users.noreply.github.com>
Co-authored-by: BBQ <35603386+HoneyBBQ@users.noreply.github.com>
Co-authored-by: Ran <16112591+chen-ran@users.noreply.github.com>
2026-03-03 15:33:50 +08:00

476 lines
14 KiB
Go

package models
import (
"context"
"errors"
"fmt"
"log/slog"
"strings"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/memohai/memoh/internal/db"
"github.com/memohai/memoh/internal/db/sqlc"
)
var ErrModelIDAlreadyExists = errors.New("model_id already exists")
var ErrModelIDAmbiguous = errors.New("model_id is ambiguous across providers")
// Service provides CRUD operations for models
type Service struct {
queries *sqlc.Queries
logger *slog.Logger
}
// NewService creates a new models service
func NewService(log *slog.Logger, queries *sqlc.Queries) *Service {
return &Service{
queries: queries,
logger: log.With(slog.String("service", "models")),
}
}
// Create adds a new model to the database
func (s *Service) Create(ctx context.Context, req AddRequest) (AddResponse, error) {
model := Model(req)
if err := model.Validate(); err != nil {
return AddResponse{}, fmt.Errorf("validation failed: %w", err)
}
// Convert to sqlc params
llmProviderID, err := db.ParseUUID(model.LlmProviderID)
if err != nil {
return AddResponse{}, fmt.Errorf("invalid llm provider ID: %w", err)
}
inputMod := []string{}
if model.Type == ModelTypeChat {
inputMod = normalizeModalities(model.InputModalities, []string{ModelInputText})
}
params := sqlc.CreateModelParams{
ModelID: model.ModelID,
LlmProviderID: llmProviderID,
InputModalities: inputMod,
SupportsReasoning: model.SupportsReasoning,
Type: string(model.Type),
}
if model.ClientType != "" {
params.ClientType = pgtype.Text{String: string(model.ClientType), Valid: true}
}
// Handle optional name field
if model.Name != "" {
params.Name = pgtype.Text{String: model.Name, Valid: true}
}
// Handle optional dimensions field (only for embedding models)
if model.Type == ModelTypeEmbedding && model.Dimensions > 0 {
params.Dimensions = pgtype.Int4{Int32: int32(model.Dimensions), Valid: true}
}
created, err := s.queries.CreateModel(ctx, params)
if err != nil {
if db.IsUniqueViolation(err) {
return AddResponse{}, ErrModelIDAlreadyExists
}
return AddResponse{}, fmt.Errorf("failed to create model: %w", err)
}
// Convert pgtype.UUID to string
var idStr string
if created.ID.Valid {
id, err := uuid.FromBytes(created.ID.Bytes[:])
if err != nil {
return AddResponse{}, fmt.Errorf("failed to convert UUID: %w", err)
}
idStr = id.String()
}
return AddResponse{
ID: idStr,
ModelID: created.ModelID,
}, nil
}
// GetByID retrieves a model by its internal UUID
func (s *Service) GetByID(ctx context.Context, id string) (GetResponse, error) {
uuid, err := db.ParseUUID(id)
if err != nil {
return GetResponse{}, fmt.Errorf("invalid ID: %w", err)
}
dbModel, err := s.queries.GetModelByID(ctx, uuid)
if err != nil {
return GetResponse{}, fmt.Errorf("failed to get model: %w", err)
}
return convertToGetResponse(dbModel), nil
}
// GetByModelID retrieves a model by its model_id field
func (s *Service) GetByModelID(ctx context.Context, modelID string) (GetResponse, error) {
if modelID == "" {
return GetResponse{}, fmt.Errorf("model_id is required")
}
dbModel, err := s.findUniqueByModelID(ctx, modelID)
if err != nil {
return GetResponse{}, fmt.Errorf("failed to get model: %w", err)
}
return convertToGetResponse(dbModel), nil
}
// List returns all models
func (s *Service) List(ctx context.Context) ([]GetResponse, error) {
dbModels, err := s.queries.ListModels(ctx)
if err != nil {
return nil, fmt.Errorf("failed to list models: %w", err)
}
return convertToGetResponseList(dbModels), nil
}
// ListByType returns models filtered by type (chat or embedding)
func (s *Service) ListByType(ctx context.Context, modelType ModelType) ([]GetResponse, error) {
if modelType != ModelTypeChat && modelType != ModelTypeEmbedding {
return nil, fmt.Errorf("invalid model type: %s", modelType)
}
dbModels, err := s.queries.ListModelsByType(ctx, string(modelType))
if err != nil {
return nil, fmt.Errorf("failed to list models by type: %w", err)
}
return convertToGetResponseList(dbModels), nil
}
// ListByClientType returns models filtered by client type
func (s *Service) ListByClientType(ctx context.Context, clientType ClientType) ([]GetResponse, error) {
if !isValidClientType(clientType) {
return nil, fmt.Errorf("invalid client type: %s", clientType)
}
dbModels, err := s.queries.ListModelsByClientType(ctx, pgtype.Text{String: string(clientType), Valid: true})
if err != nil {
return nil, fmt.Errorf("failed to list models by client type: %w", err)
}
return convertToGetResponseList(dbModels), nil
}
// ListByProviderID returns models filtered by provider ID.
func (s *Service) ListByProviderID(ctx context.Context, providerID string) ([]GetResponse, error) {
if strings.TrimSpace(providerID) == "" {
return nil, fmt.Errorf("provider id is required")
}
uuid, err := db.ParseUUID(providerID)
if err != nil {
return nil, fmt.Errorf("invalid provider id: %w", err)
}
dbModels, err := s.queries.ListModelsByProviderID(ctx, uuid)
if err != nil {
return nil, fmt.Errorf("failed to list models by provider: %w", err)
}
return convertToGetResponseList(dbModels), nil
}
// ListByProviderIDAndType returns models filtered by provider ID and type.
func (s *Service) ListByProviderIDAndType(ctx context.Context, providerID string, modelType ModelType) ([]GetResponse, error) {
if modelType != ModelTypeChat && modelType != ModelTypeEmbedding {
return nil, fmt.Errorf("invalid model type: %s", modelType)
}
if strings.TrimSpace(providerID) == "" {
return nil, fmt.Errorf("provider id is required")
}
uuid, err := db.ParseUUID(providerID)
if err != nil {
return nil, fmt.Errorf("invalid provider id: %w", err)
}
dbModels, err := s.queries.ListModelsByProviderIDAndType(ctx, sqlc.ListModelsByProviderIDAndTypeParams{
LlmProviderID: uuid,
Type: string(modelType),
})
if err != nil {
return nil, fmt.Errorf("failed to list models by provider and type: %w", err)
}
return convertToGetResponseList(dbModels), nil
}
// UpdateByID updates a model by its internal UUID
func (s *Service) UpdateByID(ctx context.Context, id string, req UpdateRequest) (GetResponse, error) {
uuid, err := db.ParseUUID(id)
if err != nil {
return GetResponse{}, fmt.Errorf("invalid ID: %w", err)
}
model := Model(req)
if err := model.Validate(); err != nil {
return GetResponse{}, fmt.Errorf("validation failed: %w", err)
}
inputMod := []string{}
if model.Type == ModelTypeChat {
inputMod = normalizeModalities(model.InputModalities, []string{ModelInputText})
}
params := sqlc.UpdateModelParams{
ID: uuid,
ModelID: model.ModelID,
InputModalities: inputMod,
SupportsReasoning: model.SupportsReasoning,
Type: string(model.Type),
}
if model.ClientType != "" {
params.ClientType = pgtype.Text{String: string(model.ClientType), Valid: true}
}
llmProviderID, err := db.ParseUUID(model.LlmProviderID)
if err != nil {
return GetResponse{}, fmt.Errorf("invalid llm provider ID: %w", err)
}
params.LlmProviderID = llmProviderID
if model.Name != "" {
params.Name = pgtype.Text{String: model.Name, Valid: true}
}
if model.Type == ModelTypeEmbedding && model.Dimensions > 0 {
params.Dimensions = pgtype.Int4{Int32: int32(model.Dimensions), Valid: true}
}
updated, err := s.queries.UpdateModel(ctx, params)
if err != nil {
if db.IsUniqueViolation(err) {
return GetResponse{}, ErrModelIDAlreadyExists
}
return GetResponse{}, fmt.Errorf("failed to update model: %w", err)
}
return convertToGetResponse(updated), nil
}
// UpdateByModelID updates a model by its model_id field
func (s *Service) UpdateByModelID(ctx context.Context, modelID string, req UpdateRequest) (GetResponse, error) {
if modelID == "" {
return GetResponse{}, fmt.Errorf("model_id is required")
}
current, err := s.findUniqueByModelID(ctx, modelID)
if err != nil {
return GetResponse{}, fmt.Errorf("failed to update model: %w", err)
}
model := Model(req)
if err := model.Validate(); err != nil {
return GetResponse{}, fmt.Errorf("validation failed: %w", err)
}
inputMod := []string{}
if model.Type == ModelTypeChat {
inputMod = normalizeModalities(model.InputModalities, []string{ModelInputText})
}
params := sqlc.UpdateModelParams{
ID: current.ID,
InputModalities: inputMod,
SupportsReasoning: model.SupportsReasoning,
Type: string(model.Type),
}
if model.ClientType != "" {
params.ClientType = pgtype.Text{String: string(model.ClientType), Valid: true}
}
llmProviderID, err := db.ParseUUID(model.LlmProviderID)
if err != nil {
return GetResponse{}, fmt.Errorf("invalid llm provider ID: %w", err)
}
params.LlmProviderID = llmProviderID
if model.Name != "" {
params.Name = pgtype.Text{String: model.Name, Valid: true}
}
if model.Type == ModelTypeEmbedding && model.Dimensions > 0 {
params.Dimensions = pgtype.Int4{Int32: int32(model.Dimensions), Valid: true}
}
params.ModelID = model.ModelID
updated, err := s.queries.UpdateModel(ctx, params)
if err != nil {
if db.IsUniqueViolation(err) {
return GetResponse{}, ErrModelIDAlreadyExists
}
return GetResponse{}, fmt.Errorf("failed to update model: %w", err)
}
return convertToGetResponse(updated), nil
}
// DeleteByID deletes a model by its internal UUID
func (s *Service) DeleteByID(ctx context.Context, id string) error {
uuid, err := db.ParseUUID(id)
if err != nil {
return fmt.Errorf("invalid ID: %w", err)
}
if err := s.queries.DeleteModel(ctx, uuid); err != nil {
return fmt.Errorf("failed to delete model: %w", err)
}
return nil
}
// DeleteByModelID deletes a model by its model_id field
func (s *Service) DeleteByModelID(ctx context.Context, modelID string) error {
if modelID == "" {
return fmt.Errorf("model_id is required")
}
current, err := s.findUniqueByModelID(ctx, modelID)
if err != nil {
return fmt.Errorf("failed to delete model: %w", err)
}
if err := s.queries.DeleteModel(ctx, current.ID); err != nil {
return fmt.Errorf("failed to delete model: %w", err)
}
return nil
}
// Count returns the total number of models
func (s *Service) Count(ctx context.Context) (int64, error) {
count, err := s.queries.CountModels(ctx)
if err != nil {
return 0, fmt.Errorf("failed to count models: %w", err)
}
return count, nil
}
// CountByType returns the number of models of a specific type
func (s *Service) CountByType(ctx context.Context, modelType ModelType) (int64, error) {
if modelType != ModelTypeChat && modelType != ModelTypeEmbedding {
return 0, fmt.Errorf("invalid model type: %s", modelType)
}
count, err := s.queries.CountModelsByType(ctx, string(modelType))
if err != nil {
return 0, fmt.Errorf("failed to count models by type: %w", err)
}
return count, nil
}
// Helper functions
func convertToGetResponse(dbModel sqlc.Model) GetResponse {
resp := GetResponse{
ID: dbModel.ID.String(),
ModelID: dbModel.ModelID,
Model: Model{
ModelID: dbModel.ModelID,
SupportsReasoning: dbModel.SupportsReasoning,
Type: ModelType(dbModel.Type),
},
}
if dbModel.ClientType.Valid {
resp.Model.ClientType = ClientType(dbModel.ClientType.String)
}
if resp.Model.Type == ModelTypeChat {
resp.Model.InputModalities = normalizeModalities(dbModel.InputModalities, []string{ModelInputText})
}
if dbModel.LlmProviderID.Valid {
resp.Model.LlmProviderID = dbModel.LlmProviderID.String()
}
if dbModel.Name.Valid {
resp.Model.Name = dbModel.Name.String
}
if dbModel.Dimensions.Valid {
resp.Model.Dimensions = int(dbModel.Dimensions.Int32)
}
return resp
}
func convertToGetResponseList(dbModels []sqlc.Model) []GetResponse {
responses := make([]GetResponse, 0, len(dbModels))
for _, dbModel := range dbModels {
responses = append(responses, convertToGetResponse(dbModel))
}
return responses
}
func (s *Service) findUniqueByModelID(ctx context.Context, modelID string) (sqlc.Model, error) {
rows, err := s.queries.ListModelsByModelID(ctx, modelID)
if err != nil {
return sqlc.Model{}, err
}
if len(rows) == 0 {
return sqlc.Model{}, pgx.ErrNoRows
}
if len(rows) > 1 {
return sqlc.Model{}, ErrModelIDAmbiguous
}
return rows[0], nil
}
// normalizeModalities returns modalities if non-empty, otherwise the provided fallback.
func normalizeModalities(modalities []string, fallback []string) []string {
if len(modalities) == 0 {
return fallback
}
return modalities
}
func isValidClientType(clientType ClientType) bool {
switch clientType {
case ClientTypeOpenAIResponses,
ClientTypeOpenAICompletions,
ClientTypeAnthropicMessages,
ClientTypeGoogleGenerativeAI:
return true
default:
return false
}
}
// SelectMemoryModel selects a chat model for memory operations.
func SelectMemoryModel(ctx context.Context, modelsService *Service, queries *sqlc.Queries) (GetResponse, sqlc.LlmProvider, error) {
if modelsService == nil {
return GetResponse{}, sqlc.LlmProvider{}, fmt.Errorf("models service not configured")
}
if queries == nil {
return GetResponse{}, sqlc.LlmProvider{}, fmt.Errorf("queries not configured")
}
candidates, err := modelsService.ListByType(ctx, ModelTypeChat)
if err != nil || len(candidates) == 0 {
return GetResponse{}, sqlc.LlmProvider{}, fmt.Errorf("no chat models available for memory operations")
}
selected := candidates[0]
provider, err := FetchProviderByID(ctx, queries, selected.LlmProviderID)
if err != nil {
return GetResponse{}, sqlc.LlmProvider{}, err
}
return selected, provider, nil
}
// SelectMemoryModelForBot selects memory model for a bot.
// Since memory model configuration has moved to the memory provider config,
// this now delegates directly to SelectMemoryModel.
func SelectMemoryModelForBot(ctx context.Context, modelsService *Service, queries *sqlc.Queries, _ string) (GetResponse, sqlc.LlmProvider, error) {
return SelectMemoryModel(ctx, modelsService, queries)
}
// FetchProviderByID fetches a provider by ID.
func FetchProviderByID(ctx context.Context, queries *sqlc.Queries, providerID string) (sqlc.LlmProvider, error) {
if strings.TrimSpace(providerID) == "" {
return sqlc.LlmProvider{}, fmt.Errorf("provider id missing")
}
parsed, err := db.ParseUUID(providerID)
if err != nil {
return sqlc.LlmProvider{}, err
}
return queries.GetLlmProviderByID(ctx, parsed)
}