mirror of
https://github.com/memohai/Memoh.git
synced 2026-04-27 07:16:19 +09:00
ea719f7ca7
* refactor: memory provider * fix: migrations * feat: divide collection from different built-in memory * feat: add `MEMORY.md` and `PROFILES.md` * use .env for docker compose. fix #142 (#143) * feat(web): add brand icons for search providers (#144) Add custom FontAwesome icon definitions for all 9 search providers: - Yandex: uses existing faYandex from FA free brands - Tavily, Jina, Exa, Bocha, Serper: custom icons from brand SVGs - DuckDuckGo, SearXNG, Sogou: custom icons from Simple Icons Icons are registered with a custom 'fac' prefix and rendered as monochrome (currentColor) via FontAwesome's standard rendering. * fix: resolve multiple UI bugs (#147) * feat: add email service with multi-adapter support (#146) * feat: add email service with multi-adapter support Implement a full-stack email service with global provider management, per-bot bindings with granular read/write permissions, outbox audit storage, and MCP tool integration for direct mailbox access. Backend: - Email providers: CRUD with dynamic config schema (generic SMTP/IMAP, Mailgun) - Generic adapter: go-mail (SMTP) + go-imap/v2 (IMAP IDLE real-time push via UnilateralDataHandler + UID-based tracking + periodic check fallback) - Mailgun adapter: mailgun-go/v5 with dual inbound mode (webhook + poll) - Bot email bindings: per-bot provider binding with independent r/w permissions - Outbox: outbound email audit log with status tracking - Trigger: inbound emails push notification to bot_inbox (from/subject only, LLM reads full content on demand via MCP tools) - MailboxReader interface: on-demand IMAP queries for listing/reading emails - MCP tools: email_accounts, email_send, email_list (paginated mailbox), email_read (by UID) — all with multi-binding and provider_id selection - Webhook: /email/mailgun/webhook/:config_id (JWT-skipped, signature-verified) - DB migration: 0019_add_email (email_providers, bot_email_bindings, email_outbox) Frontend: - Email Providers page: /email-providers with MasterDetailSidebarLayout - Dynamic config form rendered from ordered provider meta schema with i18n keys - Bot detail: Email tab with bindings management + outbox audit table - Sidebar navigation entry - Full i18n support (en + zh) - Auto-generated SDK from Swagger Closes #17 * feat(email): trigger bot conversation immediately on inbound email Instead of only storing an inbox item and waiting for the next chat, the email trigger now proactively invokes the conversation resolver so the bot processes new emails right away — aligned with the schedule/heartbeat trigger pattern. * fix: lint --------- Co-authored-by: Acbox <acbox0328@gmail.com> * chore: update AGENTS.md * feat: files preview * feat(web): improve MCP details page * refactor(skills): import skill with pure markdown string * merge main into refactor/memory * fix: migration * refactor: temp delete qdrant and bm25 index * fix: clean merge code * fix: update memory handler --------- Co-authored-by: Leohearts <leohearts@leohearts.com> Co-authored-by: Menci <mencici@msn.com> Co-authored-by: Quincy <69751197+dqygit@users.noreply.github.com> Co-authored-by: BBQ <35603386+HoneyBBQ@users.noreply.github.com> Co-authored-by: Ran <16112591+chen-ran@users.noreply.github.com>
476 lines
14 KiB
Go
476 lines
14 KiB
Go
package models
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"strings"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
"github.com/memohai/memoh/internal/db"
|
|
"github.com/memohai/memoh/internal/db/sqlc"
|
|
)
|
|
|
|
var ErrModelIDAlreadyExists = errors.New("model_id already exists")
|
|
var ErrModelIDAmbiguous = errors.New("model_id is ambiguous across providers")
|
|
|
|
// Service provides CRUD operations for models
|
|
type Service struct {
|
|
queries *sqlc.Queries
|
|
logger *slog.Logger
|
|
}
|
|
|
|
// NewService creates a new models service
|
|
func NewService(log *slog.Logger, queries *sqlc.Queries) *Service {
|
|
return &Service{
|
|
queries: queries,
|
|
logger: log.With(slog.String("service", "models")),
|
|
}
|
|
}
|
|
|
|
// Create adds a new model to the database
|
|
func (s *Service) Create(ctx context.Context, req AddRequest) (AddResponse, error) {
|
|
model := Model(req)
|
|
if err := model.Validate(); err != nil {
|
|
return AddResponse{}, fmt.Errorf("validation failed: %w", err)
|
|
}
|
|
|
|
// Convert to sqlc params
|
|
llmProviderID, err := db.ParseUUID(model.LlmProviderID)
|
|
if err != nil {
|
|
return AddResponse{}, fmt.Errorf("invalid llm provider ID: %w", err)
|
|
}
|
|
|
|
inputMod := []string{}
|
|
if model.Type == ModelTypeChat {
|
|
inputMod = normalizeModalities(model.InputModalities, []string{ModelInputText})
|
|
}
|
|
params := sqlc.CreateModelParams{
|
|
ModelID: model.ModelID,
|
|
LlmProviderID: llmProviderID,
|
|
InputModalities: inputMod,
|
|
SupportsReasoning: model.SupportsReasoning,
|
|
Type: string(model.Type),
|
|
}
|
|
if model.ClientType != "" {
|
|
params.ClientType = pgtype.Text{String: string(model.ClientType), Valid: true}
|
|
}
|
|
|
|
// Handle optional name field
|
|
if model.Name != "" {
|
|
params.Name = pgtype.Text{String: model.Name, Valid: true}
|
|
}
|
|
|
|
// Handle optional dimensions field (only for embedding models)
|
|
if model.Type == ModelTypeEmbedding && model.Dimensions > 0 {
|
|
params.Dimensions = pgtype.Int4{Int32: int32(model.Dimensions), Valid: true}
|
|
}
|
|
|
|
created, err := s.queries.CreateModel(ctx, params)
|
|
if err != nil {
|
|
if db.IsUniqueViolation(err) {
|
|
return AddResponse{}, ErrModelIDAlreadyExists
|
|
}
|
|
return AddResponse{}, fmt.Errorf("failed to create model: %w", err)
|
|
}
|
|
|
|
// Convert pgtype.UUID to string
|
|
var idStr string
|
|
if created.ID.Valid {
|
|
id, err := uuid.FromBytes(created.ID.Bytes[:])
|
|
if err != nil {
|
|
return AddResponse{}, fmt.Errorf("failed to convert UUID: %w", err)
|
|
}
|
|
idStr = id.String()
|
|
}
|
|
|
|
return AddResponse{
|
|
ID: idStr,
|
|
ModelID: created.ModelID,
|
|
}, nil
|
|
}
|
|
|
|
// GetByID retrieves a model by its internal UUID
|
|
func (s *Service) GetByID(ctx context.Context, id string) (GetResponse, error) {
|
|
uuid, err := db.ParseUUID(id)
|
|
if err != nil {
|
|
return GetResponse{}, fmt.Errorf("invalid ID: %w", err)
|
|
}
|
|
|
|
dbModel, err := s.queries.GetModelByID(ctx, uuid)
|
|
if err != nil {
|
|
return GetResponse{}, fmt.Errorf("failed to get model: %w", err)
|
|
}
|
|
|
|
return convertToGetResponse(dbModel), nil
|
|
}
|
|
|
|
// GetByModelID retrieves a model by its model_id field
|
|
func (s *Service) GetByModelID(ctx context.Context, modelID string) (GetResponse, error) {
|
|
if modelID == "" {
|
|
return GetResponse{}, fmt.Errorf("model_id is required")
|
|
}
|
|
|
|
dbModel, err := s.findUniqueByModelID(ctx, modelID)
|
|
if err != nil {
|
|
return GetResponse{}, fmt.Errorf("failed to get model: %w", err)
|
|
}
|
|
|
|
return convertToGetResponse(dbModel), nil
|
|
}
|
|
|
|
// List returns all models
|
|
func (s *Service) List(ctx context.Context) ([]GetResponse, error) {
|
|
dbModels, err := s.queries.ListModels(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to list models: %w", err)
|
|
}
|
|
|
|
return convertToGetResponseList(dbModels), nil
|
|
}
|
|
|
|
// ListByType returns models filtered by type (chat or embedding)
|
|
func (s *Service) ListByType(ctx context.Context, modelType ModelType) ([]GetResponse, error) {
|
|
if modelType != ModelTypeChat && modelType != ModelTypeEmbedding {
|
|
return nil, fmt.Errorf("invalid model type: %s", modelType)
|
|
}
|
|
|
|
dbModels, err := s.queries.ListModelsByType(ctx, string(modelType))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to list models by type: %w", err)
|
|
}
|
|
|
|
return convertToGetResponseList(dbModels), nil
|
|
}
|
|
|
|
// ListByClientType returns models filtered by client type
|
|
func (s *Service) ListByClientType(ctx context.Context, clientType ClientType) ([]GetResponse, error) {
|
|
if !isValidClientType(clientType) {
|
|
return nil, fmt.Errorf("invalid client type: %s", clientType)
|
|
}
|
|
|
|
dbModels, err := s.queries.ListModelsByClientType(ctx, pgtype.Text{String: string(clientType), Valid: true})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to list models by client type: %w", err)
|
|
}
|
|
|
|
return convertToGetResponseList(dbModels), nil
|
|
}
|
|
|
|
// ListByProviderID returns models filtered by provider ID.
|
|
func (s *Service) ListByProviderID(ctx context.Context, providerID string) ([]GetResponse, error) {
|
|
if strings.TrimSpace(providerID) == "" {
|
|
return nil, fmt.Errorf("provider id is required")
|
|
}
|
|
uuid, err := db.ParseUUID(providerID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("invalid provider id: %w", err)
|
|
}
|
|
dbModels, err := s.queries.ListModelsByProviderID(ctx, uuid)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to list models by provider: %w", err)
|
|
}
|
|
return convertToGetResponseList(dbModels), nil
|
|
}
|
|
|
|
// ListByProviderIDAndType returns models filtered by provider ID and type.
|
|
func (s *Service) ListByProviderIDAndType(ctx context.Context, providerID string, modelType ModelType) ([]GetResponse, error) {
|
|
if modelType != ModelTypeChat && modelType != ModelTypeEmbedding {
|
|
return nil, fmt.Errorf("invalid model type: %s", modelType)
|
|
}
|
|
if strings.TrimSpace(providerID) == "" {
|
|
return nil, fmt.Errorf("provider id is required")
|
|
}
|
|
uuid, err := db.ParseUUID(providerID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("invalid provider id: %w", err)
|
|
}
|
|
dbModels, err := s.queries.ListModelsByProviderIDAndType(ctx, sqlc.ListModelsByProviderIDAndTypeParams{
|
|
LlmProviderID: uuid,
|
|
Type: string(modelType),
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to list models by provider and type: %w", err)
|
|
}
|
|
return convertToGetResponseList(dbModels), nil
|
|
}
|
|
|
|
// UpdateByID updates a model by its internal UUID
|
|
func (s *Service) UpdateByID(ctx context.Context, id string, req UpdateRequest) (GetResponse, error) {
|
|
uuid, err := db.ParseUUID(id)
|
|
if err != nil {
|
|
return GetResponse{}, fmt.Errorf("invalid ID: %w", err)
|
|
}
|
|
|
|
model := Model(req)
|
|
if err := model.Validate(); err != nil {
|
|
return GetResponse{}, fmt.Errorf("validation failed: %w", err)
|
|
}
|
|
|
|
inputMod := []string{}
|
|
if model.Type == ModelTypeChat {
|
|
inputMod = normalizeModalities(model.InputModalities, []string{ModelInputText})
|
|
}
|
|
params := sqlc.UpdateModelParams{
|
|
ID: uuid,
|
|
ModelID: model.ModelID,
|
|
InputModalities: inputMod,
|
|
SupportsReasoning: model.SupportsReasoning,
|
|
Type: string(model.Type),
|
|
}
|
|
if model.ClientType != "" {
|
|
params.ClientType = pgtype.Text{String: string(model.ClientType), Valid: true}
|
|
}
|
|
|
|
llmProviderID, err := db.ParseUUID(model.LlmProviderID)
|
|
if err != nil {
|
|
return GetResponse{}, fmt.Errorf("invalid llm provider ID: %w", err)
|
|
}
|
|
params.LlmProviderID = llmProviderID
|
|
|
|
if model.Name != "" {
|
|
params.Name = pgtype.Text{String: model.Name, Valid: true}
|
|
}
|
|
|
|
if model.Type == ModelTypeEmbedding && model.Dimensions > 0 {
|
|
params.Dimensions = pgtype.Int4{Int32: int32(model.Dimensions), Valid: true}
|
|
}
|
|
|
|
updated, err := s.queries.UpdateModel(ctx, params)
|
|
if err != nil {
|
|
if db.IsUniqueViolation(err) {
|
|
return GetResponse{}, ErrModelIDAlreadyExists
|
|
}
|
|
return GetResponse{}, fmt.Errorf("failed to update model: %w", err)
|
|
}
|
|
|
|
return convertToGetResponse(updated), nil
|
|
}
|
|
|
|
// UpdateByModelID updates a model by its model_id field
|
|
func (s *Service) UpdateByModelID(ctx context.Context, modelID string, req UpdateRequest) (GetResponse, error) {
|
|
if modelID == "" {
|
|
return GetResponse{}, fmt.Errorf("model_id is required")
|
|
}
|
|
current, err := s.findUniqueByModelID(ctx, modelID)
|
|
if err != nil {
|
|
return GetResponse{}, fmt.Errorf("failed to update model: %w", err)
|
|
}
|
|
|
|
model := Model(req)
|
|
if err := model.Validate(); err != nil {
|
|
return GetResponse{}, fmt.Errorf("validation failed: %w", err)
|
|
}
|
|
|
|
inputMod := []string{}
|
|
if model.Type == ModelTypeChat {
|
|
inputMod = normalizeModalities(model.InputModalities, []string{ModelInputText})
|
|
}
|
|
params := sqlc.UpdateModelParams{
|
|
ID: current.ID,
|
|
InputModalities: inputMod,
|
|
SupportsReasoning: model.SupportsReasoning,
|
|
Type: string(model.Type),
|
|
}
|
|
if model.ClientType != "" {
|
|
params.ClientType = pgtype.Text{String: string(model.ClientType), Valid: true}
|
|
}
|
|
|
|
llmProviderID, err := db.ParseUUID(model.LlmProviderID)
|
|
if err != nil {
|
|
return GetResponse{}, fmt.Errorf("invalid llm provider ID: %w", err)
|
|
}
|
|
params.LlmProviderID = llmProviderID
|
|
|
|
if model.Name != "" {
|
|
params.Name = pgtype.Text{String: model.Name, Valid: true}
|
|
}
|
|
|
|
if model.Type == ModelTypeEmbedding && model.Dimensions > 0 {
|
|
params.Dimensions = pgtype.Int4{Int32: int32(model.Dimensions), Valid: true}
|
|
}
|
|
|
|
params.ModelID = model.ModelID
|
|
|
|
updated, err := s.queries.UpdateModel(ctx, params)
|
|
if err != nil {
|
|
if db.IsUniqueViolation(err) {
|
|
return GetResponse{}, ErrModelIDAlreadyExists
|
|
}
|
|
return GetResponse{}, fmt.Errorf("failed to update model: %w", err)
|
|
}
|
|
|
|
return convertToGetResponse(updated), nil
|
|
}
|
|
|
|
// DeleteByID deletes a model by its internal UUID
|
|
func (s *Service) DeleteByID(ctx context.Context, id string) error {
|
|
uuid, err := db.ParseUUID(id)
|
|
if err != nil {
|
|
return fmt.Errorf("invalid ID: %w", err)
|
|
}
|
|
|
|
if err := s.queries.DeleteModel(ctx, uuid); err != nil {
|
|
return fmt.Errorf("failed to delete model: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// DeleteByModelID deletes a model by its model_id field
|
|
func (s *Service) DeleteByModelID(ctx context.Context, modelID string) error {
|
|
if modelID == "" {
|
|
return fmt.Errorf("model_id is required")
|
|
}
|
|
current, err := s.findUniqueByModelID(ctx, modelID)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to delete model: %w", err)
|
|
}
|
|
|
|
if err := s.queries.DeleteModel(ctx, current.ID); err != nil {
|
|
return fmt.Errorf("failed to delete model: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Count returns the total number of models
|
|
func (s *Service) Count(ctx context.Context) (int64, error) {
|
|
count, err := s.queries.CountModels(ctx)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("failed to count models: %w", err)
|
|
}
|
|
return count, nil
|
|
}
|
|
|
|
// CountByType returns the number of models of a specific type
|
|
func (s *Service) CountByType(ctx context.Context, modelType ModelType) (int64, error) {
|
|
if modelType != ModelTypeChat && modelType != ModelTypeEmbedding {
|
|
return 0, fmt.Errorf("invalid model type: %s", modelType)
|
|
}
|
|
|
|
count, err := s.queries.CountModelsByType(ctx, string(modelType))
|
|
if err != nil {
|
|
return 0, fmt.Errorf("failed to count models by type: %w", err)
|
|
}
|
|
return count, nil
|
|
}
|
|
|
|
// Helper functions
|
|
|
|
func convertToGetResponse(dbModel sqlc.Model) GetResponse {
|
|
resp := GetResponse{
|
|
ID: dbModel.ID.String(),
|
|
ModelID: dbModel.ModelID,
|
|
Model: Model{
|
|
ModelID: dbModel.ModelID,
|
|
SupportsReasoning: dbModel.SupportsReasoning,
|
|
Type: ModelType(dbModel.Type),
|
|
},
|
|
}
|
|
if dbModel.ClientType.Valid {
|
|
resp.Model.ClientType = ClientType(dbModel.ClientType.String)
|
|
}
|
|
if resp.Model.Type == ModelTypeChat {
|
|
resp.Model.InputModalities = normalizeModalities(dbModel.InputModalities, []string{ModelInputText})
|
|
}
|
|
|
|
if dbModel.LlmProviderID.Valid {
|
|
resp.Model.LlmProviderID = dbModel.LlmProviderID.String()
|
|
}
|
|
|
|
if dbModel.Name.Valid {
|
|
resp.Model.Name = dbModel.Name.String
|
|
}
|
|
|
|
if dbModel.Dimensions.Valid {
|
|
resp.Model.Dimensions = int(dbModel.Dimensions.Int32)
|
|
}
|
|
|
|
return resp
|
|
}
|
|
|
|
func convertToGetResponseList(dbModels []sqlc.Model) []GetResponse {
|
|
responses := make([]GetResponse, 0, len(dbModels))
|
|
for _, dbModel := range dbModels {
|
|
responses = append(responses, convertToGetResponse(dbModel))
|
|
}
|
|
return responses
|
|
}
|
|
|
|
func (s *Service) findUniqueByModelID(ctx context.Context, modelID string) (sqlc.Model, error) {
|
|
rows, err := s.queries.ListModelsByModelID(ctx, modelID)
|
|
if err != nil {
|
|
return sqlc.Model{}, err
|
|
}
|
|
if len(rows) == 0 {
|
|
return sqlc.Model{}, pgx.ErrNoRows
|
|
}
|
|
if len(rows) > 1 {
|
|
return sqlc.Model{}, ErrModelIDAmbiguous
|
|
}
|
|
return rows[0], nil
|
|
}
|
|
|
|
// normalizeModalities returns modalities if non-empty, otherwise the provided fallback.
|
|
func normalizeModalities(modalities []string, fallback []string) []string {
|
|
if len(modalities) == 0 {
|
|
return fallback
|
|
}
|
|
return modalities
|
|
}
|
|
|
|
func isValidClientType(clientType ClientType) bool {
|
|
switch clientType {
|
|
case ClientTypeOpenAIResponses,
|
|
ClientTypeOpenAICompletions,
|
|
ClientTypeAnthropicMessages,
|
|
ClientTypeGoogleGenerativeAI:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// SelectMemoryModel selects a chat model for memory operations.
|
|
func SelectMemoryModel(ctx context.Context, modelsService *Service, queries *sqlc.Queries) (GetResponse, sqlc.LlmProvider, error) {
|
|
if modelsService == nil {
|
|
return GetResponse{}, sqlc.LlmProvider{}, fmt.Errorf("models service not configured")
|
|
}
|
|
if queries == nil {
|
|
return GetResponse{}, sqlc.LlmProvider{}, fmt.Errorf("queries not configured")
|
|
}
|
|
candidates, err := modelsService.ListByType(ctx, ModelTypeChat)
|
|
if err != nil || len(candidates) == 0 {
|
|
return GetResponse{}, sqlc.LlmProvider{}, fmt.Errorf("no chat models available for memory operations")
|
|
}
|
|
selected := candidates[0]
|
|
provider, err := FetchProviderByID(ctx, queries, selected.LlmProviderID)
|
|
if err != nil {
|
|
return GetResponse{}, sqlc.LlmProvider{}, err
|
|
}
|
|
return selected, provider, nil
|
|
}
|
|
|
|
// SelectMemoryModelForBot selects memory model for a bot.
|
|
// Since memory model configuration has moved to the memory provider config,
|
|
// this now delegates directly to SelectMemoryModel.
|
|
func SelectMemoryModelForBot(ctx context.Context, modelsService *Service, queries *sqlc.Queries, _ string) (GetResponse, sqlc.LlmProvider, error) {
|
|
return SelectMemoryModel(ctx, modelsService, queries)
|
|
}
|
|
|
|
// FetchProviderByID fetches a provider by ID.
|
|
func FetchProviderByID(ctx context.Context, queries *sqlc.Queries, providerID string) (sqlc.LlmProvider, error) {
|
|
if strings.TrimSpace(providerID) == "" {
|
|
return sqlc.LlmProvider{}, fmt.Errorf("provider id missing")
|
|
}
|
|
parsed, err := db.ParseUUID(providerID)
|
|
if err != nil {
|
|
return sqlc.LlmProvider{}, err
|
|
}
|
|
return queries.GetLlmProviderByID(ctx, parsed)
|
|
}
|