From 0406f42e862026a3afc885ea449987acaede369a Mon Sep 17 00:00:00 2001 From: Ran <16112591+chen-ran@users.noreply.github.com> Date: Fri, 13 Feb 2026 06:14:57 +0800 Subject: [PATCH] feat: memory search/compact/rebuild api --- cmd/agent/main.go | 22 +- go.mod | 20 - go.sum | 55 +- internal/handlers/memory.go | 395 ++++++- internal/mcp/providers/container/fsops.go | 33 +- .../mcp/providers/container/fsops_test.go | 4 +- internal/mcp/providers/container/provider.go | 10 +- internal/memory/llm_client.go | 30 + internal/memory/memoryfs.go | 327 ++++++ internal/memory/prompts.go | 36 + internal/memory/qdrant_store.go | 29 + internal/memory/service.go | 193 ++++ internal/memory/service_test.go | 7 + internal/memory/types.go | 39 +- spec/docs.go | 997 +++++++++++++++++- spec/swagger.json | 997 +++++++++++++++++- spec/swagger.yaml | 673 +++++++++++- 17 files changed, 3725 insertions(+), 142 deletions(-) create mode 100644 internal/memory/memoryfs.go diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 1159117d..40b6b1a1 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -119,7 +119,7 @@ func main() { // http handlers (group:"server_handlers") provideServerHandler(handlers.NewPingHandler), provideServerHandler(provideAuthHandler), - provideServerHandler(handlers.NewMemoryHandler), + provideServerHandler(provideMemoryHandler), provideServerHandler(handlers.NewEmbeddingsHandler), provideServerHandler(provideMessageHandler), provideServerHandler(handlers.NewSwaggerHandler), @@ -371,6 +371,18 @@ func provideToolGatewayService(log *slog.Logger, cfg config.Config, channelManag // handler providers (interface adaptation / config extraction) // --------------------------------------------------------------------------- +func provideMemoryHandler(log *slog.Logger, service *memory.Service, chatService *conversation.Service, accountService *accounts.Service, cfg config.Config, manager *mcp.Manager) *handlers.MemoryHandler { + h := handlers.NewMemoryHandler(log, service, chatService, accountService) + if manager != nil { + execWorkDir := cfg.MCP.DataMount + if strings.TrimSpace(execWorkDir) == "" { + execWorkDir = config.DefaultDataMount + } + h.SetMemoryFS(memory.NewMemoryFS(log, manager, execWorkDir)) + } + return h +} + func provideAuthHandler(log *slog.Logger, accountService *accounts.Service, rc *boot.RuntimeConfig) *handlers.AuthHandler { return handlers.NewAuthHandler(log, accountService, rc.JwtSecret, rc.JwtExpiresIn) } @@ -594,6 +606,14 @@ func (c *lazyLLMClient) Decide(ctx context.Context, req memory.DecideRequest) (m return client.Decide(ctx, req) } +func (c *lazyLLMClient) Compact(ctx context.Context, req memory.CompactRequest) (memory.CompactResponse, error) { + client, err := c.resolve(ctx) + if err != nil { + return memory.CompactResponse{}, err + } + return client.Compact(ctx, req) +} + func (c *lazyLLMClient) DetectLanguage(ctx context.Context, text string) (string, error) { client, err := c.resolve(ctx) if err != nil { diff --git a/go.mod b/go.mod index 1a206485..4ec4a115 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,6 @@ require ( github.com/robfig/cron/v3 v3.0.1 github.com/stretchr/testify v1.11.1 github.com/swaggo/swag v1.16.6 - github.com/tmc/langchaingo v0.1.14 go.uber.org/fx v1.24.0 golang.org/x/crypto v0.48.0 google.golang.org/grpc v1.78.0 @@ -33,15 +32,6 @@ require ( ) require ( - cloud.google.com/go v0.116.0 // indirect - cloud.google.com/go/ai v0.7.0 // indirect - cloud.google.com/go/aiplatform v1.69.0 // indirect - cloud.google.com/go/auth v0.14.0 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect - cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.2.2 // indirect - cloud.google.com/go/longrunning v0.6.2 // indirect - cloud.google.com/go/vertexai v0.12.0 // indirect cyphar.com/go-pathrs v0.2.3 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect @@ -67,7 +57,6 @@ require ( github.com/cyphar/filepath-securejoin v0.6.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/dlclark/regexp2 v1.10.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -83,12 +72,8 @@ require ( github.com/go-openapi/swag/yamlutils v0.25.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect - github.com/google/generative-ai-go v0.15.1 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/jsonschema-go v0.4.2 // indirect - github.com/google/s2a-go v0.1.9 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect - github.com/googleapis/gax-go/v2 v2.14.1 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect @@ -109,7 +94,6 @@ require ( github.com/opencontainers/selinux v1.13.1 // indirect github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pkoukk/tiktoken-go v0.1.6 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/sasha-s/go-deadlock v0.3.6 // indirect github.com/sirupsen/logrus v1.9.4 // indirect @@ -118,7 +102,6 @@ require ( github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect go.opentelemetry.io/otel v1.40.0 // indirect go.opentelemetry.io/otel/metric v1.40.0 // indirect @@ -135,9 +118,6 @@ require ( golang.org/x/text v0.34.0 // indirect golang.org/x/time v0.14.0 // indirect golang.org/x/tools v0.42.0 // indirect - google.golang.org/api v0.218.0 // indirect - google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20251029180050-ab9386a59fda // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/go.sum b/go.sum index 33e332d6..3f987f7b 100644 --- a/go.sum +++ b/go.sum @@ -1,22 +1,4 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= -cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= -cloud.google.com/go/ai v0.7.0 h1:P6+b5p4gXlza5E+u7uvcgYlzZ7103ACg70YdZeC6oGE= -cloud.google.com/go/ai v0.7.0/go.mod h1:7ozuEcraovh4ABsPbrec3o4LmFl9HigNI3D5haxYeQo= -cloud.google.com/go/aiplatform v1.69.0 h1:XvBzK8e6/6ufbi/i129Vmn/gVqFwbNPmRQ89K+MGlgc= -cloud.google.com/go/aiplatform v1.69.0/go.mod h1:nUsIqzS3khlnWvpjfJbP+2+h+VrFyYsTm7RNCAViiY8= -cloud.google.com/go/auth v0.14.0 h1:A5C4dKV/Spdvxcl0ggWwWEzzP7AZMJSEIgrkngwhGYM= -cloud.google.com/go/auth v0.14.0/go.mod h1:CYsoRL1PdiDuqeQpZE0bP2pnPrGqFcOkI0nldEQis+A= -cloud.google.com/go/auth/oauth2adapt v0.2.7 h1:/Lc7xODdqcEw8IrZ9SvwnlLX6j9FHQM74z6cBk9Rw6M= -cloud.google.com/go/auth/oauth2adapt v0.2.7/go.mod h1:NTbTTzfvPl1Y3V1nPpOgl2w6d/FjO7NNUQaWSox6ZMc= -cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= -cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.2.2 h1:ozUSofHUGf/F4tCNy/mu9tHLTaxZFLOUiKzjcgWHGIA= -cloud.google.com/go/iam v1.2.2/go.mod h1:0Ys8ccaZHdI1dEUilwzqng/6ps2YB6vRsjIe00/+6JY= -cloud.google.com/go/longrunning v0.6.2 h1:xjDfh1pQcWPEvnfjZmwjKQEcHnpz6lHjfy7Fo0MK+hc= -cloud.google.com/go/longrunning v0.6.2/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI= -cloud.google.com/go/vertexai v0.12.0 h1:zTadEo/CtsoyRXNx3uGCncoWAP1H2HakGqwznt+iMo8= -cloud.google.com/go/vertexai v0.12.0/go.mod h1:8u+d0TsvBfAAd2x5R6GMgbYhsLgo3J7lmP4bR8g2ig8= cyphar.com/go-pathrs v0.2.3 h1:0pH8gep37wB0BgaXrEaN1OtZhUMeS7VvaejSr6i822o= cyphar.com/go-pathrs v0.2.3/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= @@ -53,8 +35,6 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0= -github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4= github.com/containerd/cgroups/v3 v3.1.2 h1:OSosXMtkhI6Qove637tg1XgK4q+DhR0mX8Wi8EhrHa4= github.com/containerd/cgroups/v3 v3.1.2/go.mod h1:PKZ2AcWmSBsY/tJUVhtS/rluX0b1uq1GmPO1ElCmbOw= github.com/containerd/containerd/api v1.10.0 h1:5n0oHYVBwN4VhoX9fFykCV9dF1/BvAXeg2F8W6UYq1o= @@ -91,17 +71,10 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= -github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM= -github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo= -github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= -github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -115,7 +88,7 @@ github.com/go-openapi/jsonreference v0.21.4 h1:24qaE2y9bx/q3uRK/qN+TDwbok1NhbSmG github.com/go-openapi/jsonreference v0.21.4/go.mod h1:rIENPTjDbLpzQmQWCj5kKj3ZlmEh+EFVbz3RTUh30/4= github.com/go-openapi/spec v0.22.3 h1:qRSmj6Smz2rEBxMnLRBMeBWxbbOvuOoElvSvObIgwQc= github.com/go-openapi/spec v0.22.3/go.mod h1:iIImLODL2loCh3Vnox8TY2YWYJZjMAKYyLH2Mu8lOZs= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4= github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU= github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI= @@ -160,8 +133,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/generative-ai-go v0.15.1 h1:n8aQUpvhPOlGVuM2DRkJ2jvx04zpp42B778AROJa+pQ= -github.com/google/generative-ai-go v0.15.1/go.mod h1:AAucpWZjXsDKhQYWvCYuP6d0yB1kX998pJlOW1rAesw= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -175,15 +146,9 @@ github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbc github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k= github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= -github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= -github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= -github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= -github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= -github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= @@ -254,10 +219,6 @@ github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 h1:KPpdlQLZcHfTMQ github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw= -github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -288,8 +249,6 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI= github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg= -github.com/tmc/langchaingo v0.1.14 h1:o1qWBPigAIuFvrG6cjTFo0cZPFEZ47ZqpOYMjM15yZc= -github.com/tmc/langchaingo v0.1.14/go.mod h1:aKKYXYoqhIDEv7WKdpnnCLRaqXic69cX9MnDUk72378= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= @@ -304,8 +263,6 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= @@ -328,8 +285,6 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= -go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -398,17 +353,11 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/api v0.218.0 h1:x6JCjEWeZ9PFCRe9z0FBrNwj7pB7DOAqT35N+IPnAUA= -google.golang.org/api v0.218.0/go.mod h1:5VGHBAkxrA/8EFjLVEYmMUJ8/8+gWWQ3s4cFH0FxG2M= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 h1:ToEetK57OidYuqD4Q5w+vfEnPvPpuTwedCNVohYJfNk= -google.golang.org/genproto v0.0.0-20241118233622-e639e219e697/go.mod h1:JJrvXBWRZaFMxBufik1a4RpFw4HhgVtBBWQeQgUj2cc= -google.golang.org/genproto/googleapis/api v0.0.0-20251029180050-ab9386a59fda h1:+2XxjfsAu6vqFxwGBRcHiMaDCuZiqXGDUDVWVtrFAnE= -google.golang.org/genproto/googleapis/api v0.0.0-20251029180050-ab9386a59fda/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -437,5 +386,3 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= -sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/internal/handlers/memory.go b/internal/handlers/memory.go index 477ad70f..5cf12dda 100644 --- a/internal/handlers/memory.go +++ b/internal/handlers/memory.go @@ -6,6 +6,7 @@ import ( "net/http" "sort" "strings" + "time" "github.com/labstack/echo/v4" @@ -19,6 +20,7 @@ type MemoryHandler struct { service *memory.Service chatService *conversation.Service accountService *accounts.Service + memoryFS *memory.MemoryFS logger *slog.Logger } @@ -42,6 +44,15 @@ type memorySearchPayload struct { EmbeddingEnabled *bool `json:"embedding_enabled,omitempty"` } +type memoryDeletePayload struct { + MemoryIDs []string `json:"memory_ids,omitempty"` +} + +type memoryCompactPayload struct { + Ratio float64 `json:"ratio"` + DecayDays *int `json:"decay_days,omitempty"` +} + // namespaceScope holds namespace + scopeId for a single memory scope. type namespaceScope struct { Namespace string @@ -60,13 +71,22 @@ func NewMemoryHandler(log *slog.Logger, service *memory.Service, chatService *co } } +// SetMemoryFS sets the optional filesystem persistence layer. +func (h *MemoryHandler) SetMemoryFS(fs *memory.MemoryFS) { + h.memoryFS = fs +} + // Register registers chat-level memory routes. func (h *MemoryHandler) Register(e *echo.Echo) { chatGroup := e.Group("/bots/:bot_id/memory") chatGroup.POST("", h.ChatAdd) chatGroup.POST("/search", h.ChatSearch) + chatGroup.POST("/compact", h.ChatCompact) + chatGroup.POST("/rebuild", h.ChatRebuild) chatGroup.GET("", h.ChatGetAll) - chatGroup.DELETE("", h.ChatDeleteAll) + chatGroup.GET("/usage", h.ChatUsage) + chatGroup.DELETE("", h.ChatDelete) + chatGroup.DELETE("/:memory_id", h.ChatDeleteOne) } func (h *MemoryHandler) checkService() error { @@ -78,7 +98,20 @@ func (h *MemoryHandler) checkService() error { // --- Chat-level memory endpoints --- -// ChatAdd adds memory into the bot-shared namespace. +// ChatAdd godoc +// @Summary Add memory +// @Description Add memory into the bot-shared namespace +// @Tags memory +// @Accept json +// @Produce json +// @Param bot_id path string true "Bot ID" +// @Param payload body memoryAddPayload true "Memory add payload" +// @Success 200 {object} memory.SearchResponse +// @Failure 400 {object} ErrorResponse +// @Failure 403 {object} ErrorResponse +// @Failure 500 {object} ErrorResponse +// @Failure 503 {object} ErrorResponse +// @Router /bots/{bot_id}/memory [post] func (h *MemoryHandler) ChatAdd(c echo.Context) error { if err := h.checkService(); err != nil { return err @@ -126,10 +159,37 @@ func (h *MemoryHandler) ChatAdd(c echo.Context) error { if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) } + + // Async persist to filesystem. + if h.memoryFS != nil && len(resp.Results) > 0 { + items := resp.Results + go func() { + bgCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := h.memoryFS.PersistMemories(bgCtx, botID, items, filters); err != nil { + h.logger.Warn("async memory persist failed", slog.Any("error", err)) + } + }() + } + return c.JSON(http.StatusOK, resp) } -// ChatSearch searches memory in the bot-shared namespace. +// ChatSearch godoc +// @Summary Search memory +// @Description Search memory in the bot-shared namespace +// @Tags memory +// @Accept json +// @Produce json +// @Param bot_id path string true "Bot ID" +// @Param payload body memorySearchPayload true "Memory search payload" +// @Success 200 {object} memory.SearchResponse +// @Failure 400 {object} ErrorResponse +// @Failure 403 {object} ErrorResponse +// @Failure 404 {object} ErrorResponse +// @Failure 500 {object} ErrorResponse +// @Failure 503 {object} ErrorResponse +// @Router /bots/{bot_id}/memory/search [post] func (h *MemoryHandler) ChatSearch(c echo.Context) error { if err := h.checkService(); err != nil { return err @@ -197,7 +257,18 @@ func (h *MemoryHandler) ChatSearch(c echo.Context) error { return c.JSON(http.StatusOK, memory.SearchResponse{Results: allResults}) } -// ChatGetAll lists all memories in the bot-shared namespace. +// ChatGetAll godoc +// @Summary Get all memories +// @Description List all memories in the bot-shared namespace +// @Tags memory +// @Produce json +// @Param bot_id path string true "Bot ID" +// @Success 200 {object} memory.SearchResponse +// @Failure 400 {object} ErrorResponse +// @Failure 403 {object} ErrorResponse +// @Failure 500 {object} ErrorResponse +// @Failure 503 {object} ErrorResponse +// @Router /bots/{bot_id}/memory [get] func (h *MemoryHandler) ChatGetAll(c echo.Context) error { if err := h.checkService(); err != nil { return err @@ -236,8 +307,212 @@ func (h *MemoryHandler) ChatGetAll(c echo.Context) error { return c.JSON(http.StatusOK, memory.SearchResponse{Results: allResults}) } -// ChatDeleteAll deletes all memories in the bot-shared namespace. -func (h *MemoryHandler) ChatDeleteAll(c echo.Context) error { +// ChatDelete godoc +// @Summary Delete memories +// @Description Delete specific memories by IDs, or delete all memories if no IDs are provided +// @Tags memory +// @Accept json +// @Produce json +// @Param bot_id path string true "Bot ID" +// @Param payload body memoryDeletePayload false "Optional: specify memory_ids to delete; if omitted, deletes all" +// @Success 200 {object} memory.DeleteResponse +// @Failure 400 {object} ErrorResponse +// @Failure 403 {object} ErrorResponse +// @Failure 500 {object} ErrorResponse +// @Failure 503 {object} ErrorResponse +// @Router /bots/{bot_id}/memory [delete] +func (h *MemoryHandler) ChatDelete(c echo.Context) error { + if err := h.checkService(); err != nil { + return err + } + channelIdentityID, err := h.requireChannelIdentityID(c) + if err != nil { + return err + } + containerID, err := h.resolveBotContainerID(c) + if err != nil { + return err + } + if err := h.requireChatParticipant(c.Request().Context(), containerID, channelIdentityID); err != nil { + return err + } + + var payload memoryDeletePayload + // Body is optional; ignore bind errors for empty body. + _ = c.Bind(&payload) + + // If memory_ids provided, delete specific memories. + if len(payload.MemoryIDs) > 0 { + resp, err := h.service.DeleteBatch(c.Request().Context(), payload.MemoryIDs) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + // Sync remove from filesystem. + if h.memoryFS != nil { + if err := h.memoryFS.RemoveMemories(c.Request().Context(), containerID, payload.MemoryIDs); err != nil { + h.logger.Warn("delete memory fs remove failed", slog.Any("error", err)) + } + } + return c.JSON(http.StatusOK, resp) + } + + // Otherwise delete all memories in the bot-shared namespace. + scopes, err := h.resolveEnabledScopes(c.Request().Context(), containerID) + if err != nil { + return err + } + for _, scope := range scopes { + req := memory.DeleteAllRequest{ + Filters: buildNamespaceFilters(scope.Namespace, scope.ScopeID, nil), + } + if _, err := h.service.DeleteAll(c.Request().Context(), req); err != nil { + h.logger.Warn("deleteall namespace failed", slog.String("namespace", scope.Namespace), slog.Any("error", err)) + } + } + // Sync remove all from filesystem. + if h.memoryFS != nil { + if err := h.memoryFS.RemoveAllMemories(c.Request().Context(), containerID); err != nil { + h.logger.Warn("deleteall memory fs remove failed", slog.Any("error", err)) + } + } + return c.JSON(http.StatusOK, memory.DeleteResponse{Message: "All memories deleted successfully!"}) +} + +// ChatDeleteOne godoc +// @Summary Delete a single memory +// @Description Delete a single memory by its ID +// @Tags memory +// @Produce json +// @Param bot_id path string true "Bot ID" +// @Param id path string true "Memory ID" +// @Success 200 {object} memory.DeleteResponse +// @Failure 400 {object} ErrorResponse +// @Failure 403 {object} ErrorResponse +// @Failure 500 {object} ErrorResponse +// @Failure 503 {object} ErrorResponse +// @Router /bots/{bot_id}/memory/{id} [delete] +func (h *MemoryHandler) ChatDeleteOne(c echo.Context) error { + if err := h.checkService(); err != nil { + return err + } + channelIdentityID, err := h.requireChannelIdentityID(c) + if err != nil { + return err + } + containerID, err := h.resolveBotContainerID(c) + if err != nil { + return err + } + if err := h.requireChatParticipant(c.Request().Context(), containerID, channelIdentityID); err != nil { + return err + } + + memoryID := strings.TrimSpace(c.Param("memory_id")) + if memoryID == "" { + return echo.NewHTTPError(http.StatusBadRequest, "memory_id is required") + } + resp, err := h.service.Delete(c.Request().Context(), memoryID) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + // Sync remove from filesystem. + if h.memoryFS != nil { + if err := h.memoryFS.RemoveMemories(c.Request().Context(), containerID, []string{memoryID}); err != nil { + h.logger.Warn("delete one memory fs remove failed", slog.Any("error", err)) + } + } + return c.JSON(http.StatusOK, resp) +} + +// ChatCompact godoc +// @Summary Compact memories +// @Description Consolidate memories by merging similar/redundant entries using LLM. +// @Description +// @Description **ratio** (required, range (0,1]): +// @Description - 0.8 = light compression, mostly dedup, keep ~80% of entries +// @Description - 0.5 = moderate compression, merge similar facts, keep ~50% +// @Description - 0.3 = aggressive compression, heavily consolidate, keep ~30% +// @Description +// @Description **decay_days** (optional): enable time decay — memories older than N days are treated as low priority and more likely to be merged/dropped. +// @Tags memory +// @Accept json +// @Produce json +// @Param bot_id path string true "Bot ID" +// @Param payload body memoryCompactPayload true "ratio (0,1] required; decay_days optional" +// @Success 200 {object} memory.CompactResult +// @Failure 400 {object} ErrorResponse +// @Failure 403 {object} ErrorResponse +// @Failure 500 {object} ErrorResponse +// @Failure 503 {object} ErrorResponse +// @Router /bots/{bot_id}/memory/compact [post] +func (h *MemoryHandler) ChatCompact(c echo.Context) error { + if err := h.checkService(); err != nil { + return err + } + channelIdentityID, err := h.requireChannelIdentityID(c) + if err != nil { + return err + } + containerID, err := h.resolveBotContainerID(c) + if err != nil { + return err + } + if err := h.requireChatParticipant(c.Request().Context(), containerID, channelIdentityID); err != nil { + return err + } + + var payload memoryCompactPayload + if err := c.Bind(&payload); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + if payload.Ratio <= 0 || payload.Ratio > 1 { + return echo.NewHTTPError(http.StatusBadRequest, "ratio is required and must be in range (0, 1]") + } + ratio := payload.Ratio + var decayDays int + if payload.DecayDays != nil && *payload.DecayDays > 0 { + decayDays = *payload.DecayDays + } + + scopes, err := h.resolveEnabledScopes(c.Request().Context(), containerID) + if err != nil { + return err + } + if len(scopes) == 0 { + return echo.NewHTTPError(http.StatusBadRequest, "no memory scopes found") + } + + // Compact the first (primary) scope. + scope := scopes[0] + filters := buildNamespaceFilters(scope.Namespace, scope.ScopeID, nil) + result, err := h.service.Compact(c.Request().Context(), filters, ratio, decayDays) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + // Sync rebuild filesystem. + if h.memoryFS != nil { + if err := h.memoryFS.RebuildFiles(c.Request().Context(), containerID, result.Results, filters); err != nil { + h.logger.Warn("compact memory fs rebuild failed", slog.Any("error", err)) + } + } + + return c.JSON(http.StatusOK, result) +} + +// ChatUsage godoc +// @Summary Get memory usage +// @Description Query the estimated storage usage of current memories +// @Tags memory +// @Produce json +// @Param bot_id path string true "Bot ID" +// @Success 200 {object} memory.UsageResponse +// @Failure 400 {object} ErrorResponse +// @Failure 403 {object} ErrorResponse +// @Failure 500 {object} ErrorResponse +// @Failure 503 {object} ErrorResponse +// @Router /bots/{bot_id}/memory/usage [get] +func (h *MemoryHandler) ChatUsage(c echo.Context) error { if err := h.checkService(); err != nil { return err } @@ -258,15 +533,115 @@ func (h *MemoryHandler) ChatDeleteAll(c echo.Context) error { return err } + var totalUsage memory.UsageResponse for _, scope := range scopes { - req := memory.DeleteAllRequest{ + filters := buildNamespaceFilters(scope.Namespace, scope.ScopeID, nil) + usage, err := h.service.Usage(c.Request().Context(), filters) + if err != nil { + h.logger.Warn("usage namespace failed", slog.String("namespace", scope.Namespace), slog.Any("error", err)) + continue + } + totalUsage.Count += usage.Count + totalUsage.TotalTextBytes += usage.TotalTextBytes + totalUsage.EstimatedStorageBytes += usage.EstimatedStorageBytes + } + if totalUsage.Count > 0 { + totalUsage.AvgTextBytes = totalUsage.TotalTextBytes / int64(totalUsage.Count) + } + return c.JSON(http.StatusOK, totalUsage) +} + +// ChatRebuild godoc +// @Summary Rebuild memories from filesystem +// @Description Read memory files from the container filesystem (source of truth) and restore missing entries to Qdrant +// @Tags memory +// @Produce json +// @Param bot_id path string true "Bot ID" +// @Success 200 {object} memory.RebuildResult +// @Failure 400 {object} ErrorResponse +// @Failure 403 {object} ErrorResponse +// @Failure 500 {object} ErrorResponse +// @Failure 503 {object} ErrorResponse +// @Router /bots/{bot_id}/memory/rebuild [post] +func (h *MemoryHandler) ChatRebuild(c echo.Context) error { + if err := h.checkService(); err != nil { + return err + } + if h.memoryFS == nil { + return echo.NewHTTPError(http.StatusServiceUnavailable, "memory filesystem not configured") + } + channelIdentityID, err := h.requireChannelIdentityID(c) + if err != nil { + return err + } + containerID, err := h.resolveBotContainerID(c) + if err != nil { + return err + } + if err := h.requireChatParticipant(c.Request().Context(), containerID, channelIdentityID); err != nil { + return err + } + + // Read filesystem entries. + fsItems, err := h.memoryFS.ReadAllMemoryFiles(c.Request().Context(), containerID) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "read memory files failed: "+err.Error()) + } + + // Read manifest for filters. + manifest, _ := h.memoryFS.ReadManifest(c.Request().Context(), containerID) + + // Get current Qdrant entries. + scopes, err := h.resolveEnabledScopes(c.Request().Context(), containerID) + if err != nil { + return err + } + + existingIDs := map[string]struct{}{} + for _, scope := range scopes { + req := memory.GetAllRequest{ Filters: buildNamespaceFilters(scope.Namespace, scope.ScopeID, nil), } - if _, err := h.service.DeleteAll(c.Request().Context(), req); err != nil { - h.logger.Warn("deleteall namespace failed", slog.String("namespace", scope.Namespace), slog.Any("error", err)) + resp, err := h.service.GetAll(c.Request().Context(), req) + if err != nil { + h.logger.Warn("rebuild getall failed", slog.String("namespace", scope.Namespace), slog.Any("error", err)) + continue + } + for _, item := range resp.Results { + existingIDs[item.ID] = struct{}{} } } - return c.JSON(http.StatusOK, memory.DeleteResponse{Message: "Memory deleted successfully!"}) + + // Find and restore missing entries. + var restoredCount int + for _, fsItem := range fsItems { + if _, exists := existingIDs[fsItem.ID]; exists { + continue + } + // Resolve filters from manifest, fallback to first scope. + var filters map[string]any + if manifest != nil { + if entry, ok := manifest.Entries[fsItem.ID]; ok && len(entry.Filters) > 0 { + filters = entry.Filters + } + } + if len(filters) == 0 && len(scopes) > 0 { + filters = buildNamespaceFilters(scopes[0].Namespace, scopes[0].ScopeID, nil) + } + + if _, err := h.service.RebuildAdd(c.Request().Context(), fsItem.ID, fsItem.Memory, filters); err != nil { + h.logger.Warn("rebuild add failed", slog.String("id", fsItem.ID), slog.Any("error", err)) + continue + } + restoredCount++ + } + + return c.JSON(http.StatusOK, memory.RebuildResult{ + FsCount: len(fsItems), + QdrantCount: len(existingIDs), + MissingCount: len(fsItems) - len(existingIDs), + RestoredCount: restoredCount, + }) } // --- helpers --- diff --git a/internal/mcp/providers/container/fsops.go b/internal/mcp/providers/container/fsops.go index 49ac9009..f3c05387 100644 --- a/internal/mcp/providers/container/fsops.go +++ b/internal/mcp/providers/container/fsops.go @@ -13,7 +13,8 @@ import ( mcpgw "github.com/memohai/memoh/internal/mcp" ) -type fileEntry struct { +// FileEntry represents a filesystem entry returned by ExecList. +type FileEntry struct { Path string IsDir bool Size int64 @@ -21,11 +22,11 @@ type fileEntry struct { ModTime time.Time } -// execRead reads a file inside the container via cat. -func execRead(ctx context.Context, runner ExecRunner, botID, workDir, filePath string) (string, error) { +// ExecRead reads a file inside the container via cat. +func ExecRead(ctx context.Context, runner ExecRunner, botID, workDir, filePath string) (string, error) { result, err := runner.ExecWithCapture(ctx, mcpgw.ExecRequest{ BotID: botID, - Command: []string{"/bin/sh", "-c", "cat " + shellQuote(filePath)}, + Command: []string{"/bin/sh", "-c", "cat " + ShellQuote(filePath)}, WorkDir: workDir, }) if err != nil { @@ -37,13 +38,13 @@ func execRead(ctx context.Context, runner ExecRunner, botID, workDir, filePath s return result.Stdout, nil } -// execWrite writes content to a file inside the container using base64 encoding +// ExecWrite writes content to a file inside the container using base64 encoding // to avoid shell escaping issues. -func execWrite(ctx context.Context, runner ExecRunner, botID, workDir, filePath, content string) error { +func ExecWrite(ctx context.Context, runner ExecRunner, botID, workDir, filePath, content string) error { encoded := base64.StdEncoding.EncodeToString([]byte(content)) dir := path.Dir(filePath) script := fmt.Sprintf("mkdir -p %s && echo %s | base64 -d > %s", - shellQuote(dir), shellQuote(encoded), shellQuote(filePath)) + ShellQuote(dir), ShellQuote(encoded), ShellQuote(filePath)) result, err := runner.ExecWithCapture(ctx, mcpgw.ExecRequest{ BotID: botID, Command: []string{"/bin/sh", "-c", script}, @@ -58,9 +59,9 @@ func execWrite(ctx context.Context, runner ExecRunner, botID, workDir, filePath, return nil } -// execList lists directory entries inside the container via find + stat. +// ExecList lists directory entries inside the container via find + stat. // Output format per line: |||| -func execList(ctx context.Context, runner ExecRunner, botID, workDir, dirPath string, recursive bool) ([]fileEntry, error) { +func ExecList(ctx context.Context, runner ExecRunner, botID, workDir, dirPath string, recursive bool) ([]FileEntry, error) { depthFlag := "-maxdepth 1" if recursive { depthFlag = "" @@ -69,7 +70,7 @@ func execList(ctx context.Context, runner ExecRunner, botID, workDir, dirPath st // busybox stat -c format: %n=name, %F=type, %s=size, %a=octal mode, %Y=mtime epoch script := fmt.Sprintf( `find %s %s ! -path %s -exec stat -c '%%n|%%F|%%s|%%a|%%Y' {} \;`, - shellQuote(dirPath), depthFlag, shellQuote(dirPath), + ShellQuote(dirPath), depthFlag, ShellQuote(dirPath), ) result, err := runner.ExecWithCapture(ctx, mcpgw.ExecRequest{ BotID: botID, @@ -85,10 +86,10 @@ func execList(ctx context.Context, runner ExecRunner, botID, workDir, dirPath st return parseStatOutput(result.Stdout, dirPath), nil } -// parseStatOutput parses lines of "fullpath|type|size|mode|mtime" into fileEntry slices. -func parseStatOutput(output, basePath string) []fileEntry { +// parseStatOutput parses lines of "fullpath|type|size|mode|mtime" into FileEntry slices. +func parseStatOutput(output, basePath string) []FileEntry { lines := strings.Split(strings.TrimSpace(output), "\n") - entries := make([]fileEntry, 0, len(lines)) + entries := make([]FileEntry, 0, len(lines)) // Normalize base path for computing relative paths. base := strings.TrimSuffix(basePath, "/") if base == "" || base == "." { @@ -124,7 +125,7 @@ func parseStatOutput(output, basePath string) []fileEntry { mtimeEpoch, _ := strconv.ParseInt(mtimeStr, 10, 64) modTime := time.Unix(mtimeEpoch, 0) - entries = append(entries, fileEntry{ + entries = append(entries, FileEntry{ Path: rel, IsDir: isDir, Size: size, @@ -171,8 +172,8 @@ func applyEdit(raw, filePath, oldText, newText string) (string, error) { return bom + restoreLineEndings(updated, originalEnding), nil } -// shellQuote wraps a string in single quotes, escaping embedded single quotes. -func shellQuote(s string) string { +// ShellQuote wraps a string in single quotes, escaping embedded single quotes. +func ShellQuote(s string) string { if s == "" { return "''" } diff --git a/internal/mcp/providers/container/fsops_test.go b/internal/mcp/providers/container/fsops_test.go index acfb2f7d..3341bccb 100644 --- a/internal/mcp/providers/container/fsops_test.go +++ b/internal/mcp/providers/container/fsops_test.go @@ -13,9 +13,9 @@ func TestShellQuote(t *testing.T) { {"a b", "'a b'"}, } for _, tt := range tests { - got := shellQuote(tt.in) + got := ShellQuote(tt.in) if got != tt.want { - t.Errorf("shellQuote(%q) = %q, want %q", tt.in, got, tt.want) + t.Errorf("ShellQuote(%q) = %q, want %q", tt.in, got, tt.want) } } } diff --git a/internal/mcp/providers/container/provider.go b/internal/mcp/providers/container/provider.go index f0b95036..fbb2427a 100644 --- a/internal/mcp/providers/container/provider.go +++ b/internal/mcp/providers/container/provider.go @@ -154,7 +154,7 @@ func (p *Executor) CallTool(ctx context.Context, session mcpgw.ToolSessionContex if filePath == "" { return mcpgw.BuildToolErrorResult("path is required"), nil } - content, err := execRead(ctx, p.execRunner, botID, p.execWorkDir, filePath) + content, err := ExecRead(ctx, p.execRunner, botID, p.execWorkDir, filePath) if err != nil { return mcpgw.BuildToolErrorResult(err.Error()), nil } @@ -166,7 +166,7 @@ func (p *Executor) CallTool(ctx context.Context, session mcpgw.ToolSessionContex if filePath == "" { return mcpgw.BuildToolErrorResult("path is required"), nil } - if err := execWrite(ctx, p.execRunner, botID, p.execWorkDir, filePath, content); err != nil { + if err := ExecWrite(ctx, p.execRunner, botID, p.execWorkDir, filePath, content); err != nil { return mcpgw.BuildToolErrorResult(err.Error()), nil } return mcpgw.BuildToolSuccessResult(map[string]any{"ok": true}), nil @@ -177,7 +177,7 @@ func (p *Executor) CallTool(ctx context.Context, session mcpgw.ToolSessionContex dirPath = "." } recursive, _, _ := mcpgw.BoolArg(arguments, "recursive") - entries, err := execList(ctx, p.execRunner, botID, p.execWorkDir, dirPath, recursive) + entries, err := ExecList(ctx, p.execRunner, botID, p.execWorkDir, dirPath, recursive) if err != nil { return mcpgw.BuildToolErrorResult(err.Error()), nil } @@ -201,7 +201,7 @@ func (p *Executor) CallTool(ctx context.Context, session mcpgw.ToolSessionContex return mcpgw.BuildToolErrorResult("path, old_text and new_text are required"), nil } // Step 1: read via exec - raw, err := execRead(ctx, p.execRunner, botID, p.execWorkDir, filePath) + raw, err := ExecRead(ctx, p.execRunner, botID, p.execWorkDir, filePath) if err != nil { return mcpgw.BuildToolErrorResult(err.Error()), nil } @@ -211,7 +211,7 @@ func (p *Executor) CallTool(ctx context.Context, session mcpgw.ToolSessionContex return mcpgw.BuildToolErrorResult(err.Error()), nil } // Step 3: write back via exec - if err := execWrite(ctx, p.execRunner, botID, p.execWorkDir, filePath, updated); err != nil { + if err := ExecWrite(ctx, p.execRunner, botID, p.execWorkDir, filePath, updated); err != nil { return mcpgw.BuildToolErrorResult(err.Error()), nil } return mcpgw.BuildToolSuccessResult(map[string]any{"ok": true}), nil diff --git a/internal/memory/llm_client.go b/internal/memory/llm_client.go index 2c2e9526..967b1c64 100644 --- a/internal/memory/llm_client.go +++ b/internal/memory/llm_client.go @@ -130,6 +130,36 @@ func (c *LLMClient) Decide(ctx context.Context, req DecideRequest) (DecideRespon return DecideResponse{Actions: actions}, nil } +func (c *LLMClient) Compact(ctx context.Context, req CompactRequest) (CompactResponse, error) { + if len(req.Memories) == 0 { + return CompactResponse{}, fmt.Errorf("memories is required") + } + memories := make([]map[string]string, 0, len(req.Memories)) + for _, m := range req.Memories { + entry := map[string]string{ + "id": m.ID, + "text": m.Memory, + } + if m.CreatedAt != "" { + entry["created_at"] = m.CreatedAt + } + memories = append(memories, entry) + } + systemPrompt, userPrompt := getCompactMemoryMessages(memories, req.TargetCount, req.DecayDays) + content, err := c.callChat(ctx, []chatMessage{ + {Role: "system", Content: systemPrompt}, + {Role: "user", Content: userPrompt}, + }) + if err != nil { + return CompactResponse{}, err + } + var parsed CompactResponse + if err := json.Unmarshal([]byte(removeCodeBlocks(content)), &parsed); err != nil { + return CompactResponse{}, fmt.Errorf("failed to parse compact response: %w", err) + } + return parsed, nil +} + func (c *LLMClient) DetectLanguage(ctx context.Context, text string) (string, error) { if strings.TrimSpace(text) == "" { return "", fmt.Errorf("text is required") diff --git a/internal/memory/memoryfs.go b/internal/memory/memoryfs.go new file mode 100644 index 00000000..59b729f8 --- /dev/null +++ b/internal/memory/memoryfs.go @@ -0,0 +1,327 @@ +package memory + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "strings" + "sync" + "time" + + mcpgw "github.com/memohai/memoh/internal/mcp" + "github.com/memohai/memoh/internal/mcp/providers/container" +) + +const ( + manifestPath = "index/manifest.json" + memoryDirPath = "memory" + manifestVer = 1 +) + +// MemoryFS persists memory entries as files inside the bot container via ExecRunner. +type MemoryFS struct { + execRunner container.ExecRunner + workDir string // e.g. "/data" + logger *slog.Logger + mu sync.Mutex // serialize manifest updates +} + +// Manifest is the index file that records everything needed to rebuild memories. +type Manifest struct { + Version int `json:"version"` + UpdatedAt string `json:"updated_at"` + Entries map[string]ManifestEntry `json:"entries"` +} + +// ManifestEntry records metadata for a single memory entry. +type ManifestEntry struct { + Hash string `json:"hash"` + CreatedAt string `json:"created_at"` + Lang string `json:"lang,omitempty"` + Filters map[string]any `json:"filters,omitempty"` +} + +// NewMemoryFS creates a MemoryFS that writes through the given ExecRunner. +func NewMemoryFS(log *slog.Logger, runner container.ExecRunner, workDir string) *MemoryFS { + if log == nil { + log = slog.Default() + } + if strings.TrimSpace(workDir) == "" { + workDir = "/data" + } + return &MemoryFS{ + execRunner: runner, + workDir: workDir, + logger: log.With(slog.String("component", "memoryfs")), + } +} + +// ----- write operations ----- + +// PersistMemories writes .md files for new items and incrementally updates the manifest. +// Used after Add — does NOT delete existing files. +func (fs *MemoryFS) PersistMemories(ctx context.Context, botID string, items []MemoryItem, filters map[string]any) error { + if len(items) == 0 { + return nil + } + fs.mu.Lock() + defer fs.mu.Unlock() + + // Read existing manifest (or create new one). + manifest, _ := fs.readManifestLocked(ctx, botID) + if manifest == nil { + manifest = &Manifest{ + Version: manifestVer, + Entries: map[string]ManifestEntry{}, + } + } + + for _, item := range items { + if strings.TrimSpace(item.ID) == "" || strings.TrimSpace(item.Memory) == "" { + continue + } + // Write individual .md file. + if err := fs.writeMemoryFile(ctx, botID, item); err != nil { + fs.logger.Warn("write memory file failed", slog.String("id", item.ID), slog.Any("error", err)) + continue + } + // Update manifest entry. + manifest.Entries[item.ID] = ManifestEntry{ + Hash: item.Hash, + CreatedAt: item.CreatedAt, + Filters: filters, + } + } + + manifest.UpdatedAt = time.Now().UTC().Format(time.RFC3339) + return fs.writeManifest(ctx, botID, manifest) +} + +// RebuildFiles does a full replace: deletes all old memory/*.md files, writes new ones, +// and rewrites manifest from scratch. Used after Compact. +func (fs *MemoryFS) RebuildFiles(ctx context.Context, botID string, items []MemoryItem, filters map[string]any) error { + fs.mu.Lock() + defer fs.mu.Unlock() + + // Delete old memory dir contents. + fs.execDeleteDir(ctx, botID, memoryDirPath) + + manifest := &Manifest{ + Version: manifestVer, + UpdatedAt: time.Now().UTC().Format(time.RFC3339), + Entries: make(map[string]ManifestEntry, len(items)), + } + + for _, item := range items { + if strings.TrimSpace(item.ID) == "" || strings.TrimSpace(item.Memory) == "" { + continue + } + if err := fs.writeMemoryFile(ctx, botID, item); err != nil { + fs.logger.Warn("rebuild write memory file failed", slog.String("id", item.ID), slog.Any("error", err)) + continue + } + manifest.Entries[item.ID] = ManifestEntry{ + Hash: item.Hash, + CreatedAt: item.CreatedAt, + Filters: filters, + } + } + + return fs.writeManifest(ctx, botID, manifest) +} + +// RemoveMemories removes specific memory files from the FS and updates the manifest. +func (fs *MemoryFS) RemoveMemories(ctx context.Context, botID string, ids []string) error { + if len(ids) == 0 { + return nil + } + fs.mu.Lock() + defer fs.mu.Unlock() + + manifest, _ := fs.readManifestLocked(ctx, botID) + if manifest == nil { + manifest = &Manifest{Version: manifestVer, Entries: map[string]ManifestEntry{}} + } + + for _, id := range ids { + id = strings.TrimSpace(id) + if id == "" { + continue + } + fs.execDeleteFile(ctx, botID, fmt.Sprintf("%s/%s.md", memoryDirPath, id)) + delete(manifest.Entries, id) + } + + manifest.UpdatedAt = time.Now().UTC().Format(time.RFC3339) + return fs.writeManifest(ctx, botID, manifest) +} + +// RemoveAllMemories deletes all memory files and the manifest. +func (fs *MemoryFS) RemoveAllMemories(ctx context.Context, botID string) error { + fs.mu.Lock() + defer fs.mu.Unlock() + + fs.execDeleteDir(ctx, botID, memoryDirPath) + emptyManifest := &Manifest{ + Version: manifestVer, + UpdatedAt: time.Now().UTC().Format(time.RFC3339), + Entries: map[string]ManifestEntry{}, + } + return fs.writeManifest(ctx, botID, emptyManifest) +} + +// ----- read operations ----- + +// ReadManifest reads and parses the manifest.json file. +func (fs *MemoryFS) ReadManifest(ctx context.Context, botID string) (*Manifest, error) { + fs.mu.Lock() + defer fs.mu.Unlock() + return fs.readManifestLocked(ctx, botID) +} + +func (fs *MemoryFS) readManifestLocked(ctx context.Context, botID string) (*Manifest, error) { + content, err := container.ExecRead(ctx, fs.execRunner, botID, fs.workDir, manifestPath) + if err != nil { + return nil, err + } + var manifest Manifest + if err := json.Unmarshal([]byte(content), &manifest); err != nil { + return nil, fmt.Errorf("parse manifest: %w", err) + } + return &manifest, nil +} + +// ReadAllMemoryFiles lists and reads all .md files under memory/ and parses their frontmatter. +func (fs *MemoryFS) ReadAllMemoryFiles(ctx context.Context, botID string) ([]MemoryItem, error) { + entries, err := container.ExecList(ctx, fs.execRunner, botID, fs.workDir, memoryDirPath, false) + if err != nil { + return nil, fmt.Errorf("list memory dir: %w", err) + } + + var items []MemoryItem + for _, entry := range entries { + if entry.IsDir || !strings.HasSuffix(entry.Path, ".md") { + continue + } + filePath := memoryDirPath + "/" + entry.Path + content, err := container.ExecRead(ctx, fs.execRunner, botID, fs.workDir, filePath) + if err != nil { + fs.logger.Warn("read memory file failed", slog.String("path", filePath), slog.Any("error", err)) + continue + } + item, err := parseMemoryMD(content) + if err != nil { + fs.logger.Warn("parse memory file failed", slog.String("path", filePath), slog.Any("error", err)) + continue + } + items = append(items, item) + } + return items, nil +} + +// ----- internal helpers ----- + +func (fs *MemoryFS) writeMemoryFile(ctx context.Context, botID string, item MemoryItem) error { + content := formatMemoryMD(item) + filePath := fmt.Sprintf("%s/%s.md", memoryDirPath, item.ID) + return container.ExecWrite(ctx, fs.execRunner, botID, fs.workDir, filePath, content) +} + +func (fs *MemoryFS) writeManifest(ctx context.Context, botID string, manifest *Manifest) error { + data, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return fmt.Errorf("marshal manifest: %w", err) + } + return container.ExecWrite(ctx, fs.execRunner, botID, fs.workDir, manifestPath, string(data)) +} + +// execDeleteDir removes all files inside a directory (but keeps the directory itself). +func (fs *MemoryFS) execDeleteDir(ctx context.Context, botID, dirPath string) { + // Use find + rm to avoid shell quoting issues with glob wildcards. + script := fmt.Sprintf("find %s -type f -delete 2>/dev/null; true", container.ShellQuote(dirPath)) + _, err := fs.execRunner.ExecWithCapture(ctx, mcpgw.ExecRequest{ + BotID: botID, + Command: []string{"/bin/sh", "-c", script}, + WorkDir: fs.workDir, + }) + if err != nil { + fs.logger.Warn("exec delete dir failed", slog.String("path", dirPath), slog.Any("error", err)) + } +} + +// execDeleteFile removes a single file. +func (fs *MemoryFS) execDeleteFile(ctx context.Context, botID, filePath string) { + script := fmt.Sprintf("rm -f %s", container.ShellQuote(filePath)) + _, err := fs.execRunner.ExecWithCapture(ctx, mcpgw.ExecRequest{ + BotID: botID, + Command: []string{"/bin/sh", "-c", script}, + WorkDir: fs.workDir, + }) + if err != nil { + fs.logger.Warn("exec delete file failed", slog.String("path", filePath), slog.Any("error", err)) + } +} + +// ----- .md formatting / parsing ----- + +func formatMemoryMD(item MemoryItem) string { + var b strings.Builder + b.WriteString("---\n") + b.WriteString(fmt.Sprintf("id: %s\n", item.ID)) + if item.Hash != "" { + b.WriteString(fmt.Sprintf("hash: %s\n", item.Hash)) + } + if item.CreatedAt != "" { + b.WriteString(fmt.Sprintf("created_at: %s\n", item.CreatedAt)) + } + if item.UpdatedAt != "" { + b.WriteString(fmt.Sprintf("updated_at: %s\n", item.UpdatedAt)) + } + b.WriteString("---\n") + b.WriteString(item.Memory) + b.WriteString("\n") + return b.String() +} + +func parseMemoryMD(content string) (MemoryItem, error) { + content = strings.TrimSpace(content) + if !strings.HasPrefix(content, "---") { + return MemoryItem{}, fmt.Errorf("missing frontmatter") + } + // Split on "---" delimiters. + parts := strings.SplitN(content[3:], "---", 2) + if len(parts) < 2 { + return MemoryItem{}, fmt.Errorf("incomplete frontmatter") + } + frontmatter := strings.TrimSpace(parts[0]) + body := strings.TrimSpace(parts[1]) + + item := MemoryItem{Memory: body} + for _, line := range strings.Split(frontmatter, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + key, value, found := strings.Cut(line, ":") + if !found { + continue + } + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + switch key { + case "id": + item.ID = value + case "hash": + item.Hash = value + case "created_at": + item.CreatedAt = value + case "updated_at": + item.UpdatedAt = value + } + } + if item.ID == "" { + return MemoryItem{}, fmt.Errorf("missing id in frontmatter") + } + return item, nil +} diff --git a/internal/memory/prompts.go b/internal/memory/prompts.go index 3ad461c5..42280e4b 100644 --- a/internal/memory/prompts.go +++ b/internal/memory/prompts.go @@ -106,6 +106,42 @@ Follow the instruction mentioned below: Do not return anything except the JSON format.`, toJSON(retrievedOldMemory), toJSON(newRetrievedFacts), "```json", "```") } +func getCompactMemoryMessages(memories []map[string]string, targetCount int, decayDays int) (string, string) { + decayInstruction := "" + if decayDays > 0 { + decayInstruction = fmt.Sprintf(` +10. TIME DECAY: Today's date is %s. Memories older than %d days are LOW PRIORITY. + - When deciding which facts to merge or drop, prefer dropping/merging older low-priority memories over newer ones. + - If an older memory and a newer memory convey similar information, keep the newer one. + - Very old memories should only be kept if they contain unique, still-relevant information (e.g. name, identity, long-term preferences). +`, time.Now().UTC().Format("2006-01-02"), decayDays) + } + + systemPrompt := fmt.Sprintf(`You are a Memory Compactor. Your job is to consolidate a list of memory entries into a smaller, more concise set. + +Guidelines: +1. Merge similar or redundant entries into single, concise facts. +2. If two entries contradict each other, keep only the more recent or more specific one. +3. Preserve all unique, non-redundant information — do not lose important facts. +4. Each output fact should be a single, self-contained statement. +5. Target approximately %d output facts (but use fewer if the information naturally consolidates to less, and never produce more than the input count). +6. Keep the same language as the original memories. Do not translate. +7. Return a JSON object with a single key "facts" containing an array of strings. +8. DO NOT RETURN ANYTHING ELSE OTHER THAN THE JSON FORMAT. +9. DO NOT ADD ANY ADDITIONAL TEXT OR CODEBLOCK IN THE JSON FIELDS WHICH MAKE IT INVALID SUCH AS "%s" OR "%s".%s + +Example: +Input memories: +[{"id":"1","text":"User likes dark mode","created_at":"2026-01-01"},{"id":"2","text":"User prefers dark theme for all apps","created_at":"2026-02-10"},{"id":"3","text":"User is a software engineer","created_at":"2026-01-15"},{"id":"4","text":"User works as a developer","created_at":"2026-02-01"}] +Target: 2 + +Output: {"facts": ["User prefers dark theme for all apps", "User is a software engineer"]} +`, targetCount, "```json", "```", decayInstruction) + + userPrompt := fmt.Sprintf("Consolidate the following memories into approximately %d concise facts:\n\n%s", targetCount, toJSON(memories)) + return systemPrompt, userPrompt +} + func getLanguageDetectionMessages(text string) (string, string) { systemPrompt := `You are a language classifier for the given input text. Return a JSON object with a single key "language" whose value is one of the allowed codes. diff --git a/internal/memory/qdrant_store.go b/internal/memory/qdrant_store.go index 1a8f3241..a8216d27 100644 --- a/internal/memory/qdrant_store.go +++ b/internal/memory/qdrant_store.go @@ -329,6 +329,22 @@ func (s *QdrantStore) Delete(ctx context.Context, id string) error { return err } +func (s *QdrantStore) DeleteBatch(ctx context.Context, ids []string) error { + if len(ids) == 0 { + return nil + } + pointIDs := make([]*qdrant.PointId, 0, len(ids)) + for _, id := range ids { + pointIDs = append(pointIDs, qdrant.NewIDUUID(id)) + } + _, err := s.client.Delete(ctx, &qdrant.DeletePoints{ + CollectionName: s.collection, + Wait: qdrant.PtrOf(true), + Points: qdrant.NewPointsSelectorIDs(pointIDs), + }) + return err +} + func (s *QdrantStore) List(ctx context.Context, limit int, filters map[string]any) ([]qdrantPoint, error) { if limit <= 0 { limit = 100 @@ -379,6 +395,19 @@ func (s *QdrantStore) Scroll(ctx context.Context, limit int, filters map[string] return result, nextOffset, nil } +func (s *QdrantStore) Count(ctx context.Context, filters map[string]any) (uint64, error) { + filter := buildQdrantFilter(filters) + result, err := s.client.Count(ctx, &qdrant.CountPoints{ + CollectionName: s.collection, + Filter: filter, + Exact: qdrant.PtrOf(true), + }) + if err != nil { + return 0, err + } + return result, nil +} + func (s *QdrantStore) DeleteAll(ctx context.Context, filters map[string]any) error { filter := buildQdrantFilter(filters) if filter == nil { diff --git a/internal/memory/service.go b/internal/memory/service.go index 4e46dfaf..f57f7f16 100644 --- a/internal/memory/service.go +++ b/internal/memory/service.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "log/slog" + "math" "sort" "strings" "time" @@ -448,6 +449,26 @@ func (s *Service) Delete(ctx context.Context, memoryID string) (DeleteResponse, return DeleteResponse{Message: "Memory deleted successfully!"}, nil } +func (s *Service) DeleteBatch(ctx context.Context, memoryIDs []string) (DeleteResponse, error) { + if len(memoryIDs) == 0 { + return DeleteResponse{}, fmt.Errorf("memory_ids is required") + } + cleaned := make([]string, 0, len(memoryIDs)) + for _, id := range memoryIDs { + id = strings.TrimSpace(id) + if id != "" { + cleaned = append(cleaned, id) + } + } + if len(cleaned) == 0 { + return DeleteResponse{}, fmt.Errorf("memory_ids is required") + } + if err := s.store.DeleteBatch(ctx, cleaned); err != nil { + return DeleteResponse{}, err + } + return DeleteResponse{Message: fmt.Sprintf("%d memories deleted successfully!", len(cleaned))}, nil +} + func (s *Service) DeleteAll(ctx context.Context, req DeleteAllRequest) (DeleteResponse, error) { filters := map[string]any{} for k, v := range req.Filters { @@ -471,6 +492,142 @@ func (s *Service) DeleteAll(ctx context.Context, req DeleteAllRequest) (DeleteRe return DeleteResponse{Message: "Memories deleted successfully!"}, nil } +func (s *Service) Compact(ctx context.Context, filters map[string]any, ratio float64, decayDays int) (CompactResult, error) { + if s.llm == nil { + return CompactResult{}, fmt.Errorf("llm not configured") + } + if s.store == nil { + return CompactResult{}, fmt.Errorf("qdrant store not configured") + } + if ratio <= 0 || ratio > 1 { + ratio = 0.5 + } + + // Fetch all existing memories. + points, err := s.store.List(ctx, 0, filters) + if err != nil { + return CompactResult{}, err + } + beforeCount := len(points) + if beforeCount <= 1 { + // Nothing to compact. + items := make([]MemoryItem, 0, len(points)) + for _, p := range points { + items = append(items, payloadToMemoryItem(p.ID, p.Payload)) + } + return CompactResult{ + BeforeCount: beforeCount, + AfterCount: beforeCount, + Ratio: 1.0, + Results: items, + }, nil + } + + // Build candidate list and compute target. + candidates := make([]CandidateMemory, 0, beforeCount) + for _, p := range points { + candidates = append(candidates, CandidateMemory{ + ID: p.ID, + Memory: fmt.Sprint(p.Payload["data"]), + CreatedAt: fmt.Sprint(p.Payload["created_at"]), + }) + } + targetCount := int(math.Round(float64(beforeCount) * ratio)) + if targetCount < 1 { + targetCount = 1 + } + + // Ask LLM to consolidate. + compactResp, err := s.llm.Compact(ctx, CompactRequest{ + Memories: candidates, + TargetCount: targetCount, + DecayDays: decayDays, + }) + if err != nil { + return CompactResult{}, fmt.Errorf("compact llm call failed: %w", err) + } + if len(compactResp.Facts) == 0 { + return CompactResult{}, fmt.Errorf("compact returned no facts") + } + + // Delete old memories. + if err := s.store.DeleteAll(ctx, filters); err != nil { + return CompactResult{}, fmt.Errorf("compact delete old failed: %w", err) + } + + // Reset BM25 stats for deleted documents. + if s.bm25 != nil { + for _, p := range points { + text := fmt.Sprint(p.Payload["data"]) + lang := fmt.Sprint(p.Payload["lang"]) + if strings.TrimSpace(text) == "" || strings.TrimSpace(lang) == "" { + continue + } + freq, docLen, err := s.bm25.TermFrequencies(lang, text) + if err != nil { + continue + } + s.bm25.RemoveDocument(lang, freq, docLen) + } + } + + // Add compacted facts. + results := make([]MemoryItem, 0, len(compactResp.Facts)) + for _, fact := range compactResp.Facts { + if strings.TrimSpace(fact) == "" { + continue + } + item, err := s.applyAdd(ctx, fact, filters, nil, false) + if err != nil { + return CompactResult{}, fmt.Errorf("compact add failed: %w", err) + } + results = append(results, item) + } + + afterCount := len(results) + actualRatio := float64(afterCount) / float64(beforeCount) + return CompactResult{ + BeforeCount: beforeCount, + AfterCount: afterCount, + Ratio: math.Round(actualRatio*100) / 100, + Results: results, + }, nil +} + +const ( + // Estimated sparse vector overhead per point: ~200 dims * 8 bytes (4 index + 4 value). + sparseVectorOverheadBytes = 1600 + // Estimated payload metadata overhead per point (hash, dates, filters, lang, metadata JSON). + payloadMetadataOverheadBytes = 256 +) + +func (s *Service) Usage(ctx context.Context, filters map[string]any) (UsageResponse, error) { + if s.store == nil { + return UsageResponse{}, fmt.Errorf("qdrant store not configured") + } + points, err := s.store.List(ctx, 0, filters) + if err != nil { + return UsageResponse{}, err + } + count := len(points) + var totalTextBytes int64 + for _, p := range points { + text := fmt.Sprint(p.Payload["data"]) + totalTextBytes += int64(len(text)) + } + var avgTextBytes int64 + if count > 0 { + avgTextBytes = totalTextBytes / int64(count) + } + estimatedStorage := totalTextBytes + int64(count)*(sparseVectorOverheadBytes+payloadMetadataOverheadBytes) + return UsageResponse{ + Count: count, + TotalTextBytes: totalTextBytes, + AvgTextBytes: avgTextBytes, + EstimatedStorageBytes: estimatedStorage, + }, nil +} + func (s *Service) WarmupBM25(ctx context.Context, batchSize int) error { if s.bm25 == nil || s.store == nil { return nil @@ -602,6 +759,42 @@ func (s *Service) applyAdd(ctx context.Context, text string, filters map[string] return payloadToMemoryItem(id, payload), nil } +// RebuildAdd inserts a memory with a specific ID (from filesystem recovery). +// Like applyAdd but preserves the given ID instead of generating a new UUID. +func (s *Service) RebuildAdd(ctx context.Context, id, text string, filters map[string]any) (MemoryItem, error) { + if s.store == nil { + return MemoryItem{}, fmt.Errorf("qdrant store not configured") + } + if s.bm25 == nil { + return MemoryItem{}, fmt.Errorf("bm25 indexer not configured") + } + if strings.TrimSpace(id) == "" { + return MemoryItem{}, fmt.Errorf("id is required for rebuild") + } + lang, err := s.detectLanguage(ctx, text) + if err != nil { + return MemoryItem{}, err + } + termFreq, docLen, err := s.bm25.TermFrequencies(lang, text) + if err != nil { + return MemoryItem{}, err + } + sparseIndices, sparseValues := s.bm25.AddDocument(lang, termFreq, docLen) + payload := buildPayload(text, filters, nil, "") + payload["lang"] = lang + point := qdrantPoint{ + ID: id, + SparseIndices: sparseIndices, + SparseValues: sparseValues, + SparseVectorName: s.store.sparseVectorName, + Payload: payload, + } + if err := s.store.Upsert(ctx, []qdrantPoint{point}); err != nil { + return MemoryItem{}, err + } + return payloadToMemoryItem(id, payload), nil +} + func (s *Service) applyUpdate(ctx context.Context, id, text string, filters map[string]any, metadata map[string]any, embeddingEnabled bool) (MemoryItem, error) { if strings.TrimSpace(id) == "" { return MemoryItem{}, fmt.Errorf("update action missing id") diff --git a/internal/memory/service_test.go b/internal/memory/service_test.go index bec57bfa..4c2d0175 100644 --- a/internal/memory/service_test.go +++ b/internal/memory/service_test.go @@ -11,6 +11,7 @@ import ( type MockLLM struct { ExtractFunc func(ctx context.Context, req ExtractRequest) (ExtractResponse, error) DecideFunc func(ctx context.Context, req DecideRequest) (DecideResponse, error) + CompactFunc func(ctx context.Context, req CompactRequest) (CompactResponse, error) DetectLanguageFunc func(ctx context.Context, text string) (string, error) } @@ -20,6 +21,12 @@ func (m *MockLLM) Extract(ctx context.Context, req ExtractRequest) (ExtractRespo func (m *MockLLM) Decide(ctx context.Context, req DecideRequest) (DecideResponse, error) { return m.DecideFunc(ctx, req) } +func (m *MockLLM) Compact(ctx context.Context, req CompactRequest) (CompactResponse, error) { + if m.CompactFunc != nil { + return m.CompactFunc(ctx, req) + } + return CompactResponse{}, fmt.Errorf("compact not mocked") +} func (m *MockLLM) DetectLanguage(ctx context.Context, text string) (string, error) { return m.DetectLanguageFunc(ctx, text) } diff --git a/internal/memory/types.go b/internal/memory/types.go index dc262726..f18bd7c0 100644 --- a/internal/memory/types.go +++ b/internal/memory/types.go @@ -6,6 +6,7 @@ import "context" type LLM interface { Extract(ctx context.Context, req ExtractRequest) (ExtractResponse, error) Decide(ctx context.Context, req DecideRequest) (DecideResponse, error) + Compact(ctx context.Context, req CompactRequest) (CompactResponse, error) DetectLanguage(ctx context.Context, text string) (string, error) } @@ -117,9 +118,10 @@ type ExtractResponse struct { } type CandidateMemory struct { - ID string `json:"id"` - Memory string `json:"memory"` - Metadata map[string]any `json:"metadata,omitempty"` + ID string `json:"id"` + Memory string `json:"memory"` + CreatedAt string `json:"created_at,omitempty"` + Metadata map[string]any `json:"metadata,omitempty"` } type DecideRequest struct { @@ -139,3 +141,34 @@ type DecisionAction struct { type DecideResponse struct { Actions []DecisionAction `json:"actions"` } + +type CompactRequest struct { + Memories []CandidateMemory `json:"memories"` + TargetCount int `json:"target_count"` + DecayDays int `json:"decay_days,omitempty"` +} + +type CompactResponse struct { + Facts []string `json:"facts"` +} + +type CompactResult struct { + BeforeCount int `json:"before_count"` + AfterCount int `json:"after_count"` + Ratio float64 `json:"ratio"` + Results []MemoryItem `json:"results"` +} + +type UsageResponse struct { + Count int `json:"count"` + TotalTextBytes int64 `json:"total_text_bytes"` + AvgTextBytes int64 `json:"avg_text_bytes"` + EstimatedStorageBytes int64 `json:"estimated_storage_bytes"` +} + +type RebuildResult struct { + FsCount int `json:"fs_count"` + QdrantCount int `json:"qdrant_count"` + MissingCount int `json:"missing_count"` + RestoredCount int `json:"restored_count"` +} diff --git a/spec/docs.go b/spec/docs.go index bf5706bf..c3ae6cff 100644 --- a/spec/docs.go +++ b/spec/docs.go @@ -644,6 +644,49 @@ const docTemplate = `{ } } }, + "/bots/{bot_id}/mcp-ops/batch-delete": { + "post": { + "description": "Delete multiple MCP connections by IDs.", + "tags": [ + "mcp" + ], + "summary": "Batch delete MCP connections", + "parameters": [ + { + "description": "IDs to delete", + "name": "payload", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/handlers.BatchDeleteRequest" + } + } + ], + "responses": { + "204": { + "description": "No Content" + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + } + }, "/bots/{bot_id}/mcp-stdio": { "post": { "description": "Start a stdio MCP process in the bot container and expose it as MCP HTTP endpoint.", @@ -757,6 +800,87 @@ const docTemplate = `{ } } }, + "/bots/{bot_id}/mcp/export": { + "get": { + "description": "Export all MCP connections for a bot in standard mcpServers format.", + "tags": [ + "mcp" + ], + "summary": "Export MCP connections", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/mcp.ExportResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + } + }, + "/bots/{bot_id}/mcp/import": { + "put": { + "description": "Batch import MCP connections from standard mcpServers format. Existing connections (matched by name) get config updated with is_active preserved. New connections are created as active.", + "tags": [ + "mcp" + ], + "summary": "Import MCP connections", + "parameters": [ + { + "description": "mcpServers dict", + "name": "payload", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/mcp.ImportRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/mcp.ListResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + } + }, "/bots/{bot_id}/mcp/{id}": { "get": { "description": "Get a MCP connection by ID", @@ -909,6 +1033,486 @@ const docTemplate = `{ } } }, + "/bots/{bot_id}/memory": { + "get": { + "description": "List all memories in the bot-shared namespace", + "produces": [ + "application/json" + ], + "tags": [ + "memory" + ], + "summary": "Get all memories", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "bot_id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/memory.SearchResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + }, + "post": { + "description": "Add memory into the bot-shared namespace", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "memory" + ], + "summary": "Add memory", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "bot_id", + "in": "path", + "required": true + }, + { + "description": "Memory add payload", + "name": "payload", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/handlers.memoryAddPayload" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/memory.SearchResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + }, + "delete": { + "description": "Delete specific memories by IDs, or delete all memories if no IDs are provided", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "memory" + ], + "summary": "Delete memories", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "bot_id", + "in": "path", + "required": true + }, + { + "description": "Optional: specify memory_ids to delete; if omitted, deletes all", + "name": "payload", + "in": "body", + "schema": { + "$ref": "#/definitions/handlers.memoryDeletePayload" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/memory.DeleteResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + } + }, + "/bots/{bot_id}/memory/compact": { + "post": { + "description": "Consolidate memories by merging similar/redundant entries using LLM.\n\n**ratio** (required, range (0,1]):\n- 0.8 = light compression, mostly dedup, keep ~80% of entries\n- 0.5 = moderate compression, merge similar facts, keep ~50%\n- 0.3 = aggressive compression, heavily consolidate, keep ~30%\n\n**decay_days** (optional): enable time decay — memories older than N days are treated as low priority and more likely to be merged/dropped.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "memory" + ], + "summary": "Compact memories", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "bot_id", + "in": "path", + "required": true + }, + { + "description": "ratio (0,1] required; decay_days optional", + "name": "payload", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/handlers.memoryCompactPayload" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/memory.CompactResult" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + } + }, + "/bots/{bot_id}/memory/rebuild": { + "post": { + "description": "Read memory files from the container filesystem (source of truth) and restore missing entries to Qdrant", + "produces": [ + "application/json" + ], + "tags": [ + "memory" + ], + "summary": "Rebuild memories from filesystem", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "bot_id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/memory.RebuildResult" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + } + }, + "/bots/{bot_id}/memory/search": { + "post": { + "description": "Search memory in the bot-shared namespace", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "memory" + ], + "summary": "Search memory", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "bot_id", + "in": "path", + "required": true + }, + { + "description": "Memory search payload", + "name": "payload", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/handlers.memorySearchPayload" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/memory.SearchResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + } + }, + "/bots/{bot_id}/memory/usage": { + "get": { + "description": "Query the estimated storage usage of current memories", + "produces": [ + "application/json" + ], + "tags": [ + "memory" + ], + "summary": "Get memory usage", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "bot_id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/memory.UsageResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + } + }, + "/bots/{bot_id}/memory/{id}": { + "delete": { + "description": "Delete a single memory by its ID", + "produces": [ + "application/json" + ], + "tags": [ + "memory" + ], + "summary": "Delete a single memory", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "bot_id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Memory ID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/memory.DeleteResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + } + }, "/bots/{bot_id}/schedule": { "get": { "description": "List schedules for current user", @@ -2189,6 +2793,65 @@ const docTemplate = `{ } } }, + "/bots/{id}/checks/keys": { + "get": { + "description": "Returns all check keys available for a bot (builtin + MCP connections)", + "tags": [ + "bots" + ], + "summary": "List available check keys", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/bots.ListCheckKeysResponse" + } + } + } + } + }, + "/bots/{id}/checks/run/{key}": { + "get": { + "description": "Evaluate one check key for a bot", + "tags": [ + "bots" + ], + "summary": "Run a single bot check", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Check key", + "name": "key", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/bots.BotCheck" + } + } + } + } + }, "/bots/{id}/members": { "get": { "description": "List members for a bot", @@ -3890,6 +4553,9 @@ const docTemplate = `{ "bots.Bot": { "type": "object", "properties": { + "allow_guest": { + "type": "boolean" + }, "avatar_url": { "type": "string" }, @@ -3999,6 +4665,17 @@ const docTemplate = `{ } } }, + "bots.ListCheckKeysResponse": { + "type": "object", + "properties": { + "keys": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, "bots.ListChecksResponse": { "type": "object", "properties": { @@ -4531,9 +5208,6 @@ const docTemplate = `{ "github_com_memohai_memoh_internal_mcp.Connection": { "type": "object", "properties": { - "active": { - "type": "boolean" - }, "bot_id": { "type": "string" }, @@ -4547,6 +5221,9 @@ const docTemplate = `{ "id": { "type": "string" }, + "is_active": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -4558,6 +5235,17 @@ const docTemplate = `{ } } }, + "handlers.BatchDeleteRequest": { + "type": "object", + "properties": { + "ids": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, "handlers.ChannelMeta": { "type": "object", "properties": { @@ -4697,14 +5385,14 @@ const docTemplate = `{ "handlers.EmbeddingsUsage": { "type": "object", "properties": { + "duration": { + "type": "integer" + }, "image_tokens": { "type": "integer" }, "input_tokens": { "type": "integer" - }, - "video_tokens": { - "type": "integer" } } }, @@ -4936,6 +5624,89 @@ const docTemplate = `{ } } }, + "handlers.memoryAddPayload": { + "type": "object", + "properties": { + "embedding_enabled": { + "type": "boolean" + }, + "filters": { + "type": "object", + "additionalProperties": {} + }, + "infer": { + "type": "boolean" + }, + "message": { + "type": "string" + }, + "messages": { + "type": "array", + "items": { + "$ref": "#/definitions/memory.Message" + } + }, + "metadata": { + "type": "object", + "additionalProperties": {} + }, + "namespace": { + "type": "string" + }, + "run_id": { + "type": "string" + } + } + }, + "handlers.memoryCompactPayload": { + "type": "object", + "properties": { + "decay_days": { + "type": "integer" + }, + "ratio": { + "type": "number" + } + } + }, + "handlers.memoryDeletePayload": { + "type": "object", + "properties": { + "memory_ids": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "handlers.memorySearchPayload": { + "type": "object", + "properties": { + "embedding_enabled": { + "type": "boolean" + }, + "filters": { + "type": "object", + "additionalProperties": {} + }, + "limit": { + "type": "integer" + }, + "query": { + "type": "string" + }, + "run_id": { + "type": "string" + }, + "sources": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, "handlers.skillsOpResponse": { "type": "object", "properties": { @@ -4977,6 +5748,28 @@ const docTemplate = `{ } } }, + "mcp.ExportResponse": { + "type": "object", + "properties": { + "mcpServers": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/mcp.MCPServerEntry" + } + } + } + }, + "mcp.ImportRequest": { + "type": "object", + "properties": { + "mcpServers": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/mcp.MCPServerEntry" + } + } + } + }, "mcp.ListResponse": { "type": "object", "properties": { @@ -4988,21 +5781,203 @@ const docTemplate = `{ } } }, + "mcp.MCPServerEntry": { + "type": "object", + "properties": { + "args": { + "type": "array", + "items": { + "type": "string" + } + }, + "command": { + "type": "string" + }, + "cwd": { + "type": "string" + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "headers": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "transport": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, "mcp.UpsertRequest": { "type": "object", "properties": { - "active": { - "type": "boolean" + "args": { + "type": "array", + "items": { + "type": "string" + } }, - "config": { + "command": { + "type": "string" + }, + "cwd": { + "type": "string" + }, + "env": { "type": "object", - "additionalProperties": {} + "additionalProperties": { + "type": "string" + } + }, + "headers": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "is_active": { + "type": "boolean" }, "name": { "type": "string" }, - "type": { + "transport": { "type": "string" + }, + "url": { + "type": "string" + } + } + }, + "memory.CompactResult": { + "type": "object", + "properties": { + "after_count": { + "type": "integer" + }, + "before_count": { + "type": "integer" + }, + "ratio": { + "type": "number" + }, + "results": { + "type": "array", + "items": { + "$ref": "#/definitions/memory.MemoryItem" + } + } + } + }, + "memory.DeleteResponse": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + }, + "memory.MemoryItem": { + "type": "object", + "properties": { + "agent_id": { + "type": "string" + }, + "bot_id": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "hash": { + "type": "string" + }, + "id": { + "type": "string" + }, + "memory": { + "type": "string" + }, + "metadata": { + "type": "object", + "additionalProperties": {} + }, + "run_id": { + "type": "string" + }, + "score": { + "type": "number" + }, + "updated_at": { + "type": "string" + } + } + }, + "memory.Message": { + "type": "object", + "properties": { + "content": { + "type": "string" + }, + "role": { + "type": "string" + } + } + }, + "memory.RebuildResult": { + "type": "object", + "properties": { + "fs_count": { + "type": "integer" + }, + "missing_count": { + "type": "integer" + }, + "qdrant_count": { + "type": "integer" + }, + "restored_count": { + "type": "integer" + } + } + }, + "memory.SearchResponse": { + "type": "object", + "properties": { + "relations": { + "type": "array", + "items": {} + }, + "results": { + "type": "array", + "items": { + "$ref": "#/definitions/memory.MemoryItem" + } + } + } + }, + "memory.UsageResponse": { + "type": "object", + "properties": { + "avg_text_bytes": { + "type": "integer" + }, + "count": { + "type": "integer" + }, + "estimated_storage_bytes": { + "type": "integer" + }, + "total_text_bytes": { + "type": "integer" } } }, diff --git a/spec/swagger.json b/spec/swagger.json index f8d841cd..0ceb263a 100644 --- a/spec/swagger.json +++ b/spec/swagger.json @@ -635,6 +635,49 @@ } } }, + "/bots/{bot_id}/mcp-ops/batch-delete": { + "post": { + "description": "Delete multiple MCP connections by IDs.", + "tags": [ + "mcp" + ], + "summary": "Batch delete MCP connections", + "parameters": [ + { + "description": "IDs to delete", + "name": "payload", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/handlers.BatchDeleteRequest" + } + } + ], + "responses": { + "204": { + "description": "No Content" + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + } + }, "/bots/{bot_id}/mcp-stdio": { "post": { "description": "Start a stdio MCP process in the bot container and expose it as MCP HTTP endpoint.", @@ -748,6 +791,87 @@ } } }, + "/bots/{bot_id}/mcp/export": { + "get": { + "description": "Export all MCP connections for a bot in standard mcpServers format.", + "tags": [ + "mcp" + ], + "summary": "Export MCP connections", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/mcp.ExportResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + } + }, + "/bots/{bot_id}/mcp/import": { + "put": { + "description": "Batch import MCP connections from standard mcpServers format. Existing connections (matched by name) get config updated with is_active preserved. New connections are created as active.", + "tags": [ + "mcp" + ], + "summary": "Import MCP connections", + "parameters": [ + { + "description": "mcpServers dict", + "name": "payload", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/mcp.ImportRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/mcp.ListResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + } + }, "/bots/{bot_id}/mcp/{id}": { "get": { "description": "Get a MCP connection by ID", @@ -900,6 +1024,486 @@ } } }, + "/bots/{bot_id}/memory": { + "get": { + "description": "List all memories in the bot-shared namespace", + "produces": [ + "application/json" + ], + "tags": [ + "memory" + ], + "summary": "Get all memories", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "bot_id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/memory.SearchResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + }, + "post": { + "description": "Add memory into the bot-shared namespace", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "memory" + ], + "summary": "Add memory", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "bot_id", + "in": "path", + "required": true + }, + { + "description": "Memory add payload", + "name": "payload", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/handlers.memoryAddPayload" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/memory.SearchResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + }, + "delete": { + "description": "Delete specific memories by IDs, or delete all memories if no IDs are provided", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "memory" + ], + "summary": "Delete memories", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "bot_id", + "in": "path", + "required": true + }, + { + "description": "Optional: specify memory_ids to delete; if omitted, deletes all", + "name": "payload", + "in": "body", + "schema": { + "$ref": "#/definitions/handlers.memoryDeletePayload" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/memory.DeleteResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + } + }, + "/bots/{bot_id}/memory/compact": { + "post": { + "description": "Consolidate memories by merging similar/redundant entries using LLM.\n\n**ratio** (required, range (0,1]):\n- 0.8 = light compression, mostly dedup, keep ~80% of entries\n- 0.5 = moderate compression, merge similar facts, keep ~50%\n- 0.3 = aggressive compression, heavily consolidate, keep ~30%\n\n**decay_days** (optional): enable time decay — memories older than N days are treated as low priority and more likely to be merged/dropped.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "memory" + ], + "summary": "Compact memories", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "bot_id", + "in": "path", + "required": true + }, + { + "description": "ratio (0,1] required; decay_days optional", + "name": "payload", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/handlers.memoryCompactPayload" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/memory.CompactResult" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + } + }, + "/bots/{bot_id}/memory/rebuild": { + "post": { + "description": "Read memory files from the container filesystem (source of truth) and restore missing entries to Qdrant", + "produces": [ + "application/json" + ], + "tags": [ + "memory" + ], + "summary": "Rebuild memories from filesystem", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "bot_id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/memory.RebuildResult" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + } + }, + "/bots/{bot_id}/memory/search": { + "post": { + "description": "Search memory in the bot-shared namespace", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "memory" + ], + "summary": "Search memory", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "bot_id", + "in": "path", + "required": true + }, + { + "description": "Memory search payload", + "name": "payload", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/handlers.memorySearchPayload" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/memory.SearchResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + } + }, + "/bots/{bot_id}/memory/usage": { + "get": { + "description": "Query the estimated storage usage of current memories", + "produces": [ + "application/json" + ], + "tags": [ + "memory" + ], + "summary": "Get memory usage", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "bot_id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/memory.UsageResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + } + }, + "/bots/{bot_id}/memory/{id}": { + "delete": { + "description": "Delete a single memory by its ID", + "produces": [ + "application/json" + ], + "tags": [ + "memory" + ], + "summary": "Delete a single memory", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "bot_id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Memory ID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/memory.DeleteResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/handlers.ErrorResponse" + } + } + } + } + }, "/bots/{bot_id}/schedule": { "get": { "description": "List schedules for current user", @@ -2180,6 +2784,65 @@ } } }, + "/bots/{id}/checks/keys": { + "get": { + "description": "Returns all check keys available for a bot (builtin + MCP connections)", + "tags": [ + "bots" + ], + "summary": "List available check keys", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/bots.ListCheckKeysResponse" + } + } + } + } + }, + "/bots/{id}/checks/run/{key}": { + "get": { + "description": "Evaluate one check key for a bot", + "tags": [ + "bots" + ], + "summary": "Run a single bot check", + "parameters": [ + { + "type": "string", + "description": "Bot ID", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Check key", + "name": "key", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/bots.BotCheck" + } + } + } + } + }, "/bots/{id}/members": { "get": { "description": "List members for a bot", @@ -3881,6 +4544,9 @@ "bots.Bot": { "type": "object", "properties": { + "allow_guest": { + "type": "boolean" + }, "avatar_url": { "type": "string" }, @@ -3990,6 +4656,17 @@ } } }, + "bots.ListCheckKeysResponse": { + "type": "object", + "properties": { + "keys": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, "bots.ListChecksResponse": { "type": "object", "properties": { @@ -4522,9 +5199,6 @@ "github_com_memohai_memoh_internal_mcp.Connection": { "type": "object", "properties": { - "active": { - "type": "boolean" - }, "bot_id": { "type": "string" }, @@ -4538,6 +5212,9 @@ "id": { "type": "string" }, + "is_active": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -4549,6 +5226,17 @@ } } }, + "handlers.BatchDeleteRequest": { + "type": "object", + "properties": { + "ids": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, "handlers.ChannelMeta": { "type": "object", "properties": { @@ -4688,14 +5376,14 @@ "handlers.EmbeddingsUsage": { "type": "object", "properties": { + "duration": { + "type": "integer" + }, "image_tokens": { "type": "integer" }, "input_tokens": { "type": "integer" - }, - "video_tokens": { - "type": "integer" } } }, @@ -4927,6 +5615,89 @@ } } }, + "handlers.memoryAddPayload": { + "type": "object", + "properties": { + "embedding_enabled": { + "type": "boolean" + }, + "filters": { + "type": "object", + "additionalProperties": {} + }, + "infer": { + "type": "boolean" + }, + "message": { + "type": "string" + }, + "messages": { + "type": "array", + "items": { + "$ref": "#/definitions/memory.Message" + } + }, + "metadata": { + "type": "object", + "additionalProperties": {} + }, + "namespace": { + "type": "string" + }, + "run_id": { + "type": "string" + } + } + }, + "handlers.memoryCompactPayload": { + "type": "object", + "properties": { + "decay_days": { + "type": "integer" + }, + "ratio": { + "type": "number" + } + } + }, + "handlers.memoryDeletePayload": { + "type": "object", + "properties": { + "memory_ids": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "handlers.memorySearchPayload": { + "type": "object", + "properties": { + "embedding_enabled": { + "type": "boolean" + }, + "filters": { + "type": "object", + "additionalProperties": {} + }, + "limit": { + "type": "integer" + }, + "query": { + "type": "string" + }, + "run_id": { + "type": "string" + }, + "sources": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, "handlers.skillsOpResponse": { "type": "object", "properties": { @@ -4968,6 +5739,28 @@ } } }, + "mcp.ExportResponse": { + "type": "object", + "properties": { + "mcpServers": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/mcp.MCPServerEntry" + } + } + } + }, + "mcp.ImportRequest": { + "type": "object", + "properties": { + "mcpServers": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/mcp.MCPServerEntry" + } + } + } + }, "mcp.ListResponse": { "type": "object", "properties": { @@ -4979,21 +5772,203 @@ } } }, + "mcp.MCPServerEntry": { + "type": "object", + "properties": { + "args": { + "type": "array", + "items": { + "type": "string" + } + }, + "command": { + "type": "string" + }, + "cwd": { + "type": "string" + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "headers": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "transport": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, "mcp.UpsertRequest": { "type": "object", "properties": { - "active": { - "type": "boolean" + "args": { + "type": "array", + "items": { + "type": "string" + } }, - "config": { + "command": { + "type": "string" + }, + "cwd": { + "type": "string" + }, + "env": { "type": "object", - "additionalProperties": {} + "additionalProperties": { + "type": "string" + } + }, + "headers": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "is_active": { + "type": "boolean" }, "name": { "type": "string" }, - "type": { + "transport": { "type": "string" + }, + "url": { + "type": "string" + } + } + }, + "memory.CompactResult": { + "type": "object", + "properties": { + "after_count": { + "type": "integer" + }, + "before_count": { + "type": "integer" + }, + "ratio": { + "type": "number" + }, + "results": { + "type": "array", + "items": { + "$ref": "#/definitions/memory.MemoryItem" + } + } + } + }, + "memory.DeleteResponse": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + }, + "memory.MemoryItem": { + "type": "object", + "properties": { + "agent_id": { + "type": "string" + }, + "bot_id": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "hash": { + "type": "string" + }, + "id": { + "type": "string" + }, + "memory": { + "type": "string" + }, + "metadata": { + "type": "object", + "additionalProperties": {} + }, + "run_id": { + "type": "string" + }, + "score": { + "type": "number" + }, + "updated_at": { + "type": "string" + } + } + }, + "memory.Message": { + "type": "object", + "properties": { + "content": { + "type": "string" + }, + "role": { + "type": "string" + } + } + }, + "memory.RebuildResult": { + "type": "object", + "properties": { + "fs_count": { + "type": "integer" + }, + "missing_count": { + "type": "integer" + }, + "qdrant_count": { + "type": "integer" + }, + "restored_count": { + "type": "integer" + } + } + }, + "memory.SearchResponse": { + "type": "object", + "properties": { + "relations": { + "type": "array", + "items": {} + }, + "results": { + "type": "array", + "items": { + "$ref": "#/definitions/memory.MemoryItem" + } + } + } + }, + "memory.UsageResponse": { + "type": "object", + "properties": { + "avg_text_bytes": { + "type": "integer" + }, + "count": { + "type": "integer" + }, + "estimated_storage_bytes": { + "type": "integer" + }, + "total_text_bytes": { + "type": "integer" } } }, diff --git a/spec/swagger.yaml b/spec/swagger.yaml index 3af56577..ea44b4ee 100644 --- a/spec/swagger.yaml +++ b/spec/swagger.yaml @@ -78,6 +78,8 @@ definitions: type: object bots.Bot: properties: + allow_guest: + type: boolean avatar_url: type: string check_issue_count: @@ -150,6 +152,13 @@ definitions: $ref: '#/definitions/bots.Bot' type: array type: object + bots.ListCheckKeysResponse: + properties: + keys: + items: + type: string + type: array + type: object bots.ListChecksResponse: properties: items: @@ -514,8 +523,6 @@ definitions: type: object github_com_memohai_memoh_internal_mcp.Connection: properties: - active: - type: boolean bot_id: type: string config: @@ -525,6 +532,8 @@ definitions: type: string id: type: string + is_active: + type: boolean name: type: string type: @@ -532,6 +541,13 @@ definitions: updated_at: type: string type: object + handlers.BatchDeleteRequest: + properties: + ids: + items: + type: string + type: array + type: object handlers.ChannelMeta: properties: capabilities: @@ -622,12 +638,12 @@ definitions: type: object handlers.EmbeddingsUsage: properties: + duration: + type: integer image_tokens: type: integer input_tokens: type: integer - video_tokens: - type: integer type: object handlers.ErrorResponse: properties: @@ -777,6 +793,61 @@ definitions: user_id: type: string type: object + handlers.memoryAddPayload: + properties: + embedding_enabled: + type: boolean + filters: + additionalProperties: {} + type: object + infer: + type: boolean + message: + type: string + messages: + items: + $ref: '#/definitions/memory.Message' + type: array + metadata: + additionalProperties: {} + type: object + namespace: + type: string + run_id: + type: string + type: object + handlers.memoryCompactPayload: + properties: + decay_days: + type: integer + ratio: + type: number + type: object + handlers.memoryDeletePayload: + properties: + memory_ids: + items: + type: string + type: array + type: object + handlers.memorySearchPayload: + properties: + embedding_enabled: + type: boolean + filters: + additionalProperties: {} + type: object + limit: + type: integer + query: + type: string + run_id: + type: string + sources: + items: + type: string + type: array + type: object handlers.skillsOpResponse: properties: ok: @@ -804,6 +875,20 @@ definitions: user_id: type: string type: object + mcp.ExportResponse: + properties: + mcpServers: + additionalProperties: + $ref: '#/definitions/mcp.MCPServerEntry' + type: object + type: object + mcp.ImportRequest: + properties: + mcpServers: + additionalProperties: + $ref: '#/definitions/mcp.MCPServerEntry' + type: object + type: object mcp.ListResponse: properties: items: @@ -811,17 +896,136 @@ definitions: $ref: '#/definitions/github_com_memohai_memoh_internal_mcp.Connection' type: array type: object + mcp.MCPServerEntry: + properties: + args: + items: + type: string + type: array + command: + type: string + cwd: + type: string + env: + additionalProperties: + type: string + type: object + headers: + additionalProperties: + type: string + type: object + transport: + type: string + url: + type: string + type: object mcp.UpsertRequest: properties: - active: - type: boolean - config: - additionalProperties: {} + args: + items: + type: string + type: array + command: + type: string + cwd: + type: string + env: + additionalProperties: + type: string type: object + headers: + additionalProperties: + type: string + type: object + is_active: + type: boolean name: type: string - type: + transport: type: string + url: + type: string + type: object + memory.CompactResult: + properties: + after_count: + type: integer + before_count: + type: integer + ratio: + type: number + results: + items: + $ref: '#/definitions/memory.MemoryItem' + type: array + type: object + memory.DeleteResponse: + properties: + message: + type: string + type: object + memory.MemoryItem: + properties: + agent_id: + type: string + bot_id: + type: string + created_at: + type: string + hash: + type: string + id: + type: string + memory: + type: string + metadata: + additionalProperties: {} + type: object + run_id: + type: string + score: + type: number + updated_at: + type: string + type: object + memory.Message: + properties: + content: + type: string + role: + type: string + type: object + memory.RebuildResult: + properties: + fs_count: + type: integer + missing_count: + type: integer + qdrant_count: + type: integer + restored_count: + type: integer + type: object + memory.SearchResponse: + properties: + relations: + items: {} + type: array + results: + items: + $ref: '#/definitions/memory.MemoryItem' + type: array + type: object + memory.UsageResponse: + properties: + avg_text_bytes: + type: integer + count: + type: integer + estimated_storage_bytes: + type: integer + total_text_bytes: + type: integer type: object models.AddRequest: properties: @@ -1602,6 +1806,34 @@ paths: summary: Create MCP connection tags: - mcp + /bots/{bot_id}/mcp-ops/batch-delete: + post: + description: Delete multiple MCP connections by IDs. + parameters: + - description: IDs to delete + in: body + name: payload + required: true + schema: + $ref: '#/definitions/handlers.BatchDeleteRequest' + responses: + "204": + description: No Content + "400": + description: Bad Request + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/handlers.ErrorResponse' + summary: Batch delete MCP connections + tags: + - mcp /bots/{bot_id}/mcp-stdio: post: description: Start a stdio MCP process in the bot container and expose it as @@ -1779,6 +2011,390 @@ paths: summary: Update MCP connection tags: - mcp + /bots/{bot_id}/mcp/export: + get: + description: Export all MCP connections for a bot in standard mcpServers format. + responses: + "200": + description: OK + schema: + $ref: '#/definitions/mcp.ExportResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/handlers.ErrorResponse' + summary: Export MCP connections + tags: + - mcp + /bots/{bot_id}/mcp/import: + put: + description: Batch import MCP connections from standard mcpServers format. Existing + connections (matched by name) get config updated with is_active preserved. + New connections are created as active. + parameters: + - description: mcpServers dict + in: body + name: payload + required: true + schema: + $ref: '#/definitions/mcp.ImportRequest' + responses: + "200": + description: OK + schema: + $ref: '#/definitions/mcp.ListResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/handlers.ErrorResponse' + summary: Import MCP connections + tags: + - mcp + /bots/{bot_id}/memory: + delete: + consumes: + - application/json + description: Delete specific memories by IDs, or delete all memories if no IDs + are provided + parameters: + - description: Bot ID + in: path + name: bot_id + required: true + type: string + - description: 'Optional: specify memory_ids to delete; if omitted, deletes + all' + in: body + name: payload + schema: + $ref: '#/definitions/handlers.memoryDeletePayload' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/memory.DeleteResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/handlers.ErrorResponse' + summary: Delete memories + tags: + - memory + get: + description: List all memories in the bot-shared namespace + parameters: + - description: Bot ID + in: path + name: bot_id + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/memory.SearchResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/handlers.ErrorResponse' + summary: Get all memories + tags: + - memory + post: + consumes: + - application/json + description: Add memory into the bot-shared namespace + parameters: + - description: Bot ID + in: path + name: bot_id + required: true + type: string + - description: Memory add payload + in: body + name: payload + required: true + schema: + $ref: '#/definitions/handlers.memoryAddPayload' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/memory.SearchResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/handlers.ErrorResponse' + summary: Add memory + tags: + - memory + /bots/{bot_id}/memory/{id}: + delete: + description: Delete a single memory by its ID + parameters: + - description: Bot ID + in: path + name: bot_id + required: true + type: string + - description: Memory ID + in: path + name: id + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/memory.DeleteResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/handlers.ErrorResponse' + summary: Delete a single memory + tags: + - memory + /bots/{bot_id}/memory/compact: + post: + consumes: + - application/json + description: |- + Consolidate memories by merging similar/redundant entries using LLM. + + **ratio** (required, range (0,1]): + - 0.8 = light compression, mostly dedup, keep ~80% of entries + - 0.5 = moderate compression, merge similar facts, keep ~50% + - 0.3 = aggressive compression, heavily consolidate, keep ~30% + + **decay_days** (optional): enable time decay — memories older than N days are treated as low priority and more likely to be merged/dropped. + parameters: + - description: Bot ID + in: path + name: bot_id + required: true + type: string + - description: ratio (0,1] required; decay_days optional + in: body + name: payload + required: true + schema: + $ref: '#/definitions/handlers.memoryCompactPayload' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/memory.CompactResult' + "400": + description: Bad Request + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/handlers.ErrorResponse' + summary: Compact memories + tags: + - memory + /bots/{bot_id}/memory/rebuild: + post: + description: Read memory files from the container filesystem (source of truth) + and restore missing entries to Qdrant + parameters: + - description: Bot ID + in: path + name: bot_id + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/memory.RebuildResult' + "400": + description: Bad Request + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/handlers.ErrorResponse' + summary: Rebuild memories from filesystem + tags: + - memory + /bots/{bot_id}/memory/search: + post: + consumes: + - application/json + description: Search memory in the bot-shared namespace + parameters: + - description: Bot ID + in: path + name: bot_id + required: true + type: string + - description: Memory search payload + in: body + name: payload + required: true + schema: + $ref: '#/definitions/handlers.memorySearchPayload' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/memory.SearchResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "404": + description: Not Found + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/handlers.ErrorResponse' + summary: Search memory + tags: + - memory + /bots/{bot_id}/memory/usage: + get: + description: Query the estimated storage usage of current memories + parameters: + - description: Bot ID + in: path + name: bot_id + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/memory.UsageResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/handlers.ErrorResponse' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/handlers.ErrorResponse' + summary: Get memory usage + tags: + - memory /bots/{bot_id}/schedule: get: description: List schedules for current user @@ -2628,6 +3244,45 @@ paths: summary: List bot runtime checks tags: - bots + /bots/{id}/checks/keys: + get: + description: Returns all check keys available for a bot (builtin + MCP connections) + parameters: + - description: Bot ID + in: path + name: id + required: true + type: string + responses: + "200": + description: OK + schema: + $ref: '#/definitions/bots.ListCheckKeysResponse' + summary: List available check keys + tags: + - bots + /bots/{id}/checks/run/{key}: + get: + description: Evaluate one check key for a bot + parameters: + - description: Bot ID + in: path + name: id + required: true + type: string + - description: Check key + in: path + name: key + required: true + type: string + responses: + "200": + description: OK + schema: + $ref: '#/definitions/bots.BotCheck' + summary: Run a single bot check + tags: + - bots /bots/{id}/members: get: description: List members for a bot