feat: memory search/compact/rebuild api

This commit is contained in:
Ran
2026-02-13 06:14:57 +08:00
parent 2614763547
commit 0406f42e86
17 changed files with 3725 additions and 142 deletions
+21 -1
View File
@@ -119,7 +119,7 @@ func main() {
// http handlers (group:"server_handlers")
provideServerHandler(handlers.NewPingHandler),
provideServerHandler(provideAuthHandler),
provideServerHandler(handlers.NewMemoryHandler),
provideServerHandler(provideMemoryHandler),
provideServerHandler(handlers.NewEmbeddingsHandler),
provideServerHandler(provideMessageHandler),
provideServerHandler(handlers.NewSwaggerHandler),
@@ -371,6 +371,18 @@ func provideToolGatewayService(log *slog.Logger, cfg config.Config, channelManag
// handler providers (interface adaptation / config extraction)
// ---------------------------------------------------------------------------
func provideMemoryHandler(log *slog.Logger, service *memory.Service, chatService *conversation.Service, accountService *accounts.Service, cfg config.Config, manager *mcp.Manager) *handlers.MemoryHandler {
h := handlers.NewMemoryHandler(log, service, chatService, accountService)
if manager != nil {
execWorkDir := cfg.MCP.DataMount
if strings.TrimSpace(execWorkDir) == "" {
execWorkDir = config.DefaultDataMount
}
h.SetMemoryFS(memory.NewMemoryFS(log, manager, execWorkDir))
}
return h
}
func provideAuthHandler(log *slog.Logger, accountService *accounts.Service, rc *boot.RuntimeConfig) *handlers.AuthHandler {
return handlers.NewAuthHandler(log, accountService, rc.JwtSecret, rc.JwtExpiresIn)
}
@@ -594,6 +606,14 @@ func (c *lazyLLMClient) Decide(ctx context.Context, req memory.DecideRequest) (m
return client.Decide(ctx, req)
}
func (c *lazyLLMClient) Compact(ctx context.Context, req memory.CompactRequest) (memory.CompactResponse, error) {
client, err := c.resolve(ctx)
if err != nil {
return memory.CompactResponse{}, err
}
return client.Compact(ctx, req)
}
func (c *lazyLLMClient) DetectLanguage(ctx context.Context, text string) (string, error) {
client, err := c.resolve(ctx)
if err != nil {
-20
View File
@@ -25,7 +25,6 @@ require (
github.com/robfig/cron/v3 v3.0.1
github.com/stretchr/testify v1.11.1
github.com/swaggo/swag v1.16.6
github.com/tmc/langchaingo v0.1.14
go.uber.org/fx v1.24.0
golang.org/x/crypto v0.48.0
google.golang.org/grpc v1.78.0
@@ -33,15 +32,6 @@ require (
)
require (
cloud.google.com/go v0.116.0 // indirect
cloud.google.com/go/ai v0.7.0 // indirect
cloud.google.com/go/aiplatform v1.69.0 // indirect
cloud.google.com/go/auth v0.14.0 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
cloud.google.com/go/iam v1.2.2 // indirect
cloud.google.com/go/longrunning v0.6.2 // indirect
cloud.google.com/go/vertexai v0.12.0 // indirect
cyphar.com/go-pathrs v0.2.3 // indirect
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
@@ -67,7 +57,6 @@ require (
github.com/cyphar/filepath-securejoin v0.6.1 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/dlclark/regexp2 v1.10.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
@@ -83,12 +72,8 @@ require (
github.com/go-openapi/swag/yamlutils v0.25.4 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
github.com/google/generative-ai-go v0.15.1 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/jsonschema-go v0.4.2 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
github.com/googleapis/gax-go/v2 v2.14.1 // indirect
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
@@ -109,7 +94,6 @@ require (
github.com/opencontainers/selinux v1.13.1 // indirect
github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pkoukk/tiktoken-go v0.1.6 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/sasha-s/go-deadlock v0.3.6 // indirect
github.com/sirupsen/logrus v1.9.4 // indirect
@@ -118,7 +102,6 @@ require (
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect
go.opentelemetry.io/otel v1.40.0 // indirect
go.opentelemetry.io/otel/metric v1.40.0 // indirect
@@ -135,9 +118,6 @@ require (
golang.org/x/text v0.34.0 // indirect
golang.org/x/time v0.14.0 // indirect
golang.org/x/tools v0.42.0 // indirect
google.golang.org/api v0.218.0 // indirect
google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20251029180050-ab9386a59fda // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect
google.golang.org/protobuf v1.36.11 // indirect
)
+1 -54
View File
@@ -1,22 +1,4 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE=
cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U=
cloud.google.com/go/ai v0.7.0 h1:P6+b5p4gXlza5E+u7uvcgYlzZ7103ACg70YdZeC6oGE=
cloud.google.com/go/ai v0.7.0/go.mod h1:7ozuEcraovh4ABsPbrec3o4LmFl9HigNI3D5haxYeQo=
cloud.google.com/go/aiplatform v1.69.0 h1:XvBzK8e6/6ufbi/i129Vmn/gVqFwbNPmRQ89K+MGlgc=
cloud.google.com/go/aiplatform v1.69.0/go.mod h1:nUsIqzS3khlnWvpjfJbP+2+h+VrFyYsTm7RNCAViiY8=
cloud.google.com/go/auth v0.14.0 h1:A5C4dKV/Spdvxcl0ggWwWEzzP7AZMJSEIgrkngwhGYM=
cloud.google.com/go/auth v0.14.0/go.mod h1:CYsoRL1PdiDuqeQpZE0bP2pnPrGqFcOkI0nldEQis+A=
cloud.google.com/go/auth/oauth2adapt v0.2.7 h1:/Lc7xODdqcEw8IrZ9SvwnlLX6j9FHQM74z6cBk9Rw6M=
cloud.google.com/go/auth/oauth2adapt v0.2.7/go.mod h1:NTbTTzfvPl1Y3V1nPpOgl2w6d/FjO7NNUQaWSox6ZMc=
cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
cloud.google.com/go/iam v1.2.2 h1:ozUSofHUGf/F4tCNy/mu9tHLTaxZFLOUiKzjcgWHGIA=
cloud.google.com/go/iam v1.2.2/go.mod h1:0Ys8ccaZHdI1dEUilwzqng/6ps2YB6vRsjIe00/+6JY=
cloud.google.com/go/longrunning v0.6.2 h1:xjDfh1pQcWPEvnfjZmwjKQEcHnpz6lHjfy7Fo0MK+hc=
cloud.google.com/go/longrunning v0.6.2/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI=
cloud.google.com/go/vertexai v0.12.0 h1:zTadEo/CtsoyRXNx3uGCncoWAP1H2HakGqwznt+iMo8=
cloud.google.com/go/vertexai v0.12.0/go.mod h1:8u+d0TsvBfAAd2x5R6GMgbYhsLgo3J7lmP4bR8g2ig8=
cyphar.com/go-pathrs v0.2.3 h1:0pH8gep37wB0BgaXrEaN1OtZhUMeS7VvaejSr6i822o=
cyphar.com/go-pathrs v0.2.3/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
@@ -53,8 +35,6 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0=
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4=
github.com/containerd/cgroups/v3 v3.1.2 h1:OSosXMtkhI6Qove637tg1XgK4q+DhR0mX8Wi8EhrHa4=
github.com/containerd/cgroups/v3 v3.1.2/go.mod h1:PKZ2AcWmSBsY/tJUVhtS/rluX0b1uq1GmPO1ElCmbOw=
github.com/containerd/containerd/api v1.10.0 h1:5n0oHYVBwN4VhoX9fFykCV9dF1/BvAXeg2F8W6UYq1o=
@@ -91,17 +71,10 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM=
github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo=
github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8=
github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -115,7 +88,7 @@ github.com/go-openapi/jsonreference v0.21.4 h1:24qaE2y9bx/q3uRK/qN+TDwbok1NhbSmG
github.com/go-openapi/jsonreference v0.21.4/go.mod h1:rIENPTjDbLpzQmQWCj5kKj3ZlmEh+EFVbz3RTUh30/4=
github.com/go-openapi/spec v0.22.3 h1:qRSmj6Smz2rEBxMnLRBMeBWxbbOvuOoElvSvObIgwQc=
github.com/go-openapi/spec v0.22.3/go.mod h1:iIImLODL2loCh3Vnox8TY2YWYJZjMAKYyLH2Mu8lOZs=
github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4=
github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU=
github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI=
@@ -160,8 +133,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/generative-ai-go v0.15.1 h1:n8aQUpvhPOlGVuM2DRkJ2jvx04zpp42B778AROJa+pQ=
github.com/google/generative-ai-go v0.15.1/go.mod h1:AAucpWZjXsDKhQYWvCYuP6d0yB1kX998pJlOW1rAesw=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -175,15 +146,9 @@ github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbc
github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE=
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k=
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw=
github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA=
github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q=
github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA=
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
@@ -254,10 +219,6 @@ github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 h1:KPpdlQLZcHfTMQ
github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw=
github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -288,8 +249,6 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI=
github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg=
github.com/tmc/langchaingo v0.1.14 h1:o1qWBPigAIuFvrG6cjTFo0cZPFEZ47ZqpOYMjM15yZc=
github.com/tmc/langchaingo v0.1.14/go.mod h1:aKKYXYoqhIDEv7WKdpnnCLRaqXic69cX9MnDUk72378=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
@@ -304,8 +263,6 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0=
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
@@ -328,8 +285,6 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -398,17 +353,11 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/api v0.218.0 h1:x6JCjEWeZ9PFCRe9z0FBrNwj7pB7DOAqT35N+IPnAUA=
google.golang.org/api v0.218.0/go.mod h1:5VGHBAkxrA/8EFjLVEYmMUJ8/8+gWWQ3s4cFH0FxG2M=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 h1:ToEetK57OidYuqD4Q5w+vfEnPvPpuTwedCNVohYJfNk=
google.golang.org/genproto v0.0.0-20241118233622-e639e219e697/go.mod h1:JJrvXBWRZaFMxBufik1a4RpFw4HhgVtBBWQeQgUj2cc=
google.golang.org/genproto/googleapis/api v0.0.0-20251029180050-ab9386a59fda h1:+2XxjfsAu6vqFxwGBRcHiMaDCuZiqXGDUDVWVtrFAnE=
google.golang.org/genproto/googleapis/api v0.0.0-20251029180050-ab9386a59fda/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
@@ -437,5 +386,3 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=
+385 -10
View File
@@ -6,6 +6,7 @@ import (
"net/http"
"sort"
"strings"
"time"
"github.com/labstack/echo/v4"
@@ -19,6 +20,7 @@ type MemoryHandler struct {
service *memory.Service
chatService *conversation.Service
accountService *accounts.Service
memoryFS *memory.MemoryFS
logger *slog.Logger
}
@@ -42,6 +44,15 @@ type memorySearchPayload struct {
EmbeddingEnabled *bool `json:"embedding_enabled,omitempty"`
}
type memoryDeletePayload struct {
MemoryIDs []string `json:"memory_ids,omitempty"`
}
type memoryCompactPayload struct {
Ratio float64 `json:"ratio"`
DecayDays *int `json:"decay_days,omitempty"`
}
// namespaceScope holds namespace + scopeId for a single memory scope.
type namespaceScope struct {
Namespace string
@@ -60,13 +71,22 @@ func NewMemoryHandler(log *slog.Logger, service *memory.Service, chatService *co
}
}
// SetMemoryFS sets the optional filesystem persistence layer.
func (h *MemoryHandler) SetMemoryFS(fs *memory.MemoryFS) {
h.memoryFS = fs
}
// Register registers chat-level memory routes.
func (h *MemoryHandler) Register(e *echo.Echo) {
chatGroup := e.Group("/bots/:bot_id/memory")
chatGroup.POST("", h.ChatAdd)
chatGroup.POST("/search", h.ChatSearch)
chatGroup.POST("/compact", h.ChatCompact)
chatGroup.POST("/rebuild", h.ChatRebuild)
chatGroup.GET("", h.ChatGetAll)
chatGroup.DELETE("", h.ChatDeleteAll)
chatGroup.GET("/usage", h.ChatUsage)
chatGroup.DELETE("", h.ChatDelete)
chatGroup.DELETE("/:memory_id", h.ChatDeleteOne)
}
func (h *MemoryHandler) checkService() error {
@@ -78,7 +98,20 @@ func (h *MemoryHandler) checkService() error {
// --- Chat-level memory endpoints ---
// ChatAdd adds memory into the bot-shared namespace.
// ChatAdd godoc
// @Summary Add memory
// @Description Add memory into the bot-shared namespace
// @Tags memory
// @Accept json
// @Produce json
// @Param bot_id path string true "Bot ID"
// @Param payload body memoryAddPayload true "Memory add payload"
// @Success 200 {object} memory.SearchResponse
// @Failure 400 {object} ErrorResponse
// @Failure 403 {object} ErrorResponse
// @Failure 500 {object} ErrorResponse
// @Failure 503 {object} ErrorResponse
// @Router /bots/{bot_id}/memory [post]
func (h *MemoryHandler) ChatAdd(c echo.Context) error {
if err := h.checkService(); err != nil {
return err
@@ -126,10 +159,37 @@ func (h *MemoryHandler) ChatAdd(c echo.Context) error {
if err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
}
// Async persist to filesystem.
if h.memoryFS != nil && len(resp.Results) > 0 {
items := resp.Results
go func() {
bgCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := h.memoryFS.PersistMemories(bgCtx, botID, items, filters); err != nil {
h.logger.Warn("async memory persist failed", slog.Any("error", err))
}
}()
}
return c.JSON(http.StatusOK, resp)
}
// ChatSearch searches memory in the bot-shared namespace.
// ChatSearch godoc
// @Summary Search memory
// @Description Search memory in the bot-shared namespace
// @Tags memory
// @Accept json
// @Produce json
// @Param bot_id path string true "Bot ID"
// @Param payload body memorySearchPayload true "Memory search payload"
// @Success 200 {object} memory.SearchResponse
// @Failure 400 {object} ErrorResponse
// @Failure 403 {object} ErrorResponse
// @Failure 404 {object} ErrorResponse
// @Failure 500 {object} ErrorResponse
// @Failure 503 {object} ErrorResponse
// @Router /bots/{bot_id}/memory/search [post]
func (h *MemoryHandler) ChatSearch(c echo.Context) error {
if err := h.checkService(); err != nil {
return err
@@ -197,7 +257,18 @@ func (h *MemoryHandler) ChatSearch(c echo.Context) error {
return c.JSON(http.StatusOK, memory.SearchResponse{Results: allResults})
}
// ChatGetAll lists all memories in the bot-shared namespace.
// ChatGetAll godoc
// @Summary Get all memories
// @Description List all memories in the bot-shared namespace
// @Tags memory
// @Produce json
// @Param bot_id path string true "Bot ID"
// @Success 200 {object} memory.SearchResponse
// @Failure 400 {object} ErrorResponse
// @Failure 403 {object} ErrorResponse
// @Failure 500 {object} ErrorResponse
// @Failure 503 {object} ErrorResponse
// @Router /bots/{bot_id}/memory [get]
func (h *MemoryHandler) ChatGetAll(c echo.Context) error {
if err := h.checkService(); err != nil {
return err
@@ -236,8 +307,212 @@ func (h *MemoryHandler) ChatGetAll(c echo.Context) error {
return c.JSON(http.StatusOK, memory.SearchResponse{Results: allResults})
}
// ChatDeleteAll deletes all memories in the bot-shared namespace.
func (h *MemoryHandler) ChatDeleteAll(c echo.Context) error {
// ChatDelete godoc
// @Summary Delete memories
// @Description Delete specific memories by IDs, or delete all memories if no IDs are provided
// @Tags memory
// @Accept json
// @Produce json
// @Param bot_id path string true "Bot ID"
// @Param payload body memoryDeletePayload false "Optional: specify memory_ids to delete; if omitted, deletes all"
// @Success 200 {object} memory.DeleteResponse
// @Failure 400 {object} ErrorResponse
// @Failure 403 {object} ErrorResponse
// @Failure 500 {object} ErrorResponse
// @Failure 503 {object} ErrorResponse
// @Router /bots/{bot_id}/memory [delete]
func (h *MemoryHandler) ChatDelete(c echo.Context) error {
if err := h.checkService(); err != nil {
return err
}
channelIdentityID, err := h.requireChannelIdentityID(c)
if err != nil {
return err
}
containerID, err := h.resolveBotContainerID(c)
if err != nil {
return err
}
if err := h.requireChatParticipant(c.Request().Context(), containerID, channelIdentityID); err != nil {
return err
}
var payload memoryDeletePayload
// Body is optional; ignore bind errors for empty body.
_ = c.Bind(&payload)
// If memory_ids provided, delete specific memories.
if len(payload.MemoryIDs) > 0 {
resp, err := h.service.DeleteBatch(c.Request().Context(), payload.MemoryIDs)
if err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
}
// Sync remove from filesystem.
if h.memoryFS != nil {
if err := h.memoryFS.RemoveMemories(c.Request().Context(), containerID, payload.MemoryIDs); err != nil {
h.logger.Warn("delete memory fs remove failed", slog.Any("error", err))
}
}
return c.JSON(http.StatusOK, resp)
}
// Otherwise delete all memories in the bot-shared namespace.
scopes, err := h.resolveEnabledScopes(c.Request().Context(), containerID)
if err != nil {
return err
}
for _, scope := range scopes {
req := memory.DeleteAllRequest{
Filters: buildNamespaceFilters(scope.Namespace, scope.ScopeID, nil),
}
if _, err := h.service.DeleteAll(c.Request().Context(), req); err != nil {
h.logger.Warn("deleteall namespace failed", slog.String("namespace", scope.Namespace), slog.Any("error", err))
}
}
// Sync remove all from filesystem.
if h.memoryFS != nil {
if err := h.memoryFS.RemoveAllMemories(c.Request().Context(), containerID); err != nil {
h.logger.Warn("deleteall memory fs remove failed", slog.Any("error", err))
}
}
return c.JSON(http.StatusOK, memory.DeleteResponse{Message: "All memories deleted successfully!"})
}
// ChatDeleteOne godoc
// @Summary Delete a single memory
// @Description Delete a single memory by its ID
// @Tags memory
// @Produce json
// @Param bot_id path string true "Bot ID"
// @Param id path string true "Memory ID"
// @Success 200 {object} memory.DeleteResponse
// @Failure 400 {object} ErrorResponse
// @Failure 403 {object} ErrorResponse
// @Failure 500 {object} ErrorResponse
// @Failure 503 {object} ErrorResponse
// @Router /bots/{bot_id}/memory/{id} [delete]
func (h *MemoryHandler) ChatDeleteOne(c echo.Context) error {
if err := h.checkService(); err != nil {
return err
}
channelIdentityID, err := h.requireChannelIdentityID(c)
if err != nil {
return err
}
containerID, err := h.resolveBotContainerID(c)
if err != nil {
return err
}
if err := h.requireChatParticipant(c.Request().Context(), containerID, channelIdentityID); err != nil {
return err
}
memoryID := strings.TrimSpace(c.Param("memory_id"))
if memoryID == "" {
return echo.NewHTTPError(http.StatusBadRequest, "memory_id is required")
}
resp, err := h.service.Delete(c.Request().Context(), memoryID)
if err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
}
// Sync remove from filesystem.
if h.memoryFS != nil {
if err := h.memoryFS.RemoveMemories(c.Request().Context(), containerID, []string{memoryID}); err != nil {
h.logger.Warn("delete one memory fs remove failed", slog.Any("error", err))
}
}
return c.JSON(http.StatusOK, resp)
}
// ChatCompact godoc
// @Summary Compact memories
// @Description Consolidate memories by merging similar/redundant entries using LLM.
// @Description
// @Description **ratio** (required, range (0,1]):
// @Description - 0.8 = light compression, mostly dedup, keep ~80% of entries
// @Description - 0.5 = moderate compression, merge similar facts, keep ~50%
// @Description - 0.3 = aggressive compression, heavily consolidate, keep ~30%
// @Description
// @Description **decay_days** (optional): enable time decay — memories older than N days are treated as low priority and more likely to be merged/dropped.
// @Tags memory
// @Accept json
// @Produce json
// @Param bot_id path string true "Bot ID"
// @Param payload body memoryCompactPayload true "ratio (0,1] required; decay_days optional"
// @Success 200 {object} memory.CompactResult
// @Failure 400 {object} ErrorResponse
// @Failure 403 {object} ErrorResponse
// @Failure 500 {object} ErrorResponse
// @Failure 503 {object} ErrorResponse
// @Router /bots/{bot_id}/memory/compact [post]
func (h *MemoryHandler) ChatCompact(c echo.Context) error {
if err := h.checkService(); err != nil {
return err
}
channelIdentityID, err := h.requireChannelIdentityID(c)
if err != nil {
return err
}
containerID, err := h.resolveBotContainerID(c)
if err != nil {
return err
}
if err := h.requireChatParticipant(c.Request().Context(), containerID, channelIdentityID); err != nil {
return err
}
var payload memoryCompactPayload
if err := c.Bind(&payload); err != nil {
return echo.NewHTTPError(http.StatusBadRequest, err.Error())
}
if payload.Ratio <= 0 || payload.Ratio > 1 {
return echo.NewHTTPError(http.StatusBadRequest, "ratio is required and must be in range (0, 1]")
}
ratio := payload.Ratio
var decayDays int
if payload.DecayDays != nil && *payload.DecayDays > 0 {
decayDays = *payload.DecayDays
}
scopes, err := h.resolveEnabledScopes(c.Request().Context(), containerID)
if err != nil {
return err
}
if len(scopes) == 0 {
return echo.NewHTTPError(http.StatusBadRequest, "no memory scopes found")
}
// Compact the first (primary) scope.
scope := scopes[0]
filters := buildNamespaceFilters(scope.Namespace, scope.ScopeID, nil)
result, err := h.service.Compact(c.Request().Context(), filters, ratio, decayDays)
if err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
}
// Sync rebuild filesystem.
if h.memoryFS != nil {
if err := h.memoryFS.RebuildFiles(c.Request().Context(), containerID, result.Results, filters); err != nil {
h.logger.Warn("compact memory fs rebuild failed", slog.Any("error", err))
}
}
return c.JSON(http.StatusOK, result)
}
// ChatUsage godoc
// @Summary Get memory usage
// @Description Query the estimated storage usage of current memories
// @Tags memory
// @Produce json
// @Param bot_id path string true "Bot ID"
// @Success 200 {object} memory.UsageResponse
// @Failure 400 {object} ErrorResponse
// @Failure 403 {object} ErrorResponse
// @Failure 500 {object} ErrorResponse
// @Failure 503 {object} ErrorResponse
// @Router /bots/{bot_id}/memory/usage [get]
func (h *MemoryHandler) ChatUsage(c echo.Context) error {
if err := h.checkService(); err != nil {
return err
}
@@ -258,15 +533,115 @@ func (h *MemoryHandler) ChatDeleteAll(c echo.Context) error {
return err
}
var totalUsage memory.UsageResponse
for _, scope := range scopes {
req := memory.DeleteAllRequest{
filters := buildNamespaceFilters(scope.Namespace, scope.ScopeID, nil)
usage, err := h.service.Usage(c.Request().Context(), filters)
if err != nil {
h.logger.Warn("usage namespace failed", slog.String("namespace", scope.Namespace), slog.Any("error", err))
continue
}
totalUsage.Count += usage.Count
totalUsage.TotalTextBytes += usage.TotalTextBytes
totalUsage.EstimatedStorageBytes += usage.EstimatedStorageBytes
}
if totalUsage.Count > 0 {
totalUsage.AvgTextBytes = totalUsage.TotalTextBytes / int64(totalUsage.Count)
}
return c.JSON(http.StatusOK, totalUsage)
}
// ChatRebuild godoc
// @Summary Rebuild memories from filesystem
// @Description Read memory files from the container filesystem (source of truth) and restore missing entries to Qdrant
// @Tags memory
// @Produce json
// @Param bot_id path string true "Bot ID"
// @Success 200 {object} memory.RebuildResult
// @Failure 400 {object} ErrorResponse
// @Failure 403 {object} ErrorResponse
// @Failure 500 {object} ErrorResponse
// @Failure 503 {object} ErrorResponse
// @Router /bots/{bot_id}/memory/rebuild [post]
func (h *MemoryHandler) ChatRebuild(c echo.Context) error {
if err := h.checkService(); err != nil {
return err
}
if h.memoryFS == nil {
return echo.NewHTTPError(http.StatusServiceUnavailable, "memory filesystem not configured")
}
channelIdentityID, err := h.requireChannelIdentityID(c)
if err != nil {
return err
}
containerID, err := h.resolveBotContainerID(c)
if err != nil {
return err
}
if err := h.requireChatParticipant(c.Request().Context(), containerID, channelIdentityID); err != nil {
return err
}
// Read filesystem entries.
fsItems, err := h.memoryFS.ReadAllMemoryFiles(c.Request().Context(), containerID)
if err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, "read memory files failed: "+err.Error())
}
// Read manifest for filters.
manifest, _ := h.memoryFS.ReadManifest(c.Request().Context(), containerID)
// Get current Qdrant entries.
scopes, err := h.resolveEnabledScopes(c.Request().Context(), containerID)
if err != nil {
return err
}
existingIDs := map[string]struct{}{}
for _, scope := range scopes {
req := memory.GetAllRequest{
Filters: buildNamespaceFilters(scope.Namespace, scope.ScopeID, nil),
}
if _, err := h.service.DeleteAll(c.Request().Context(), req); err != nil {
h.logger.Warn("deleteall namespace failed", slog.String("namespace", scope.Namespace), slog.Any("error", err))
resp, err := h.service.GetAll(c.Request().Context(), req)
if err != nil {
h.logger.Warn("rebuild getall failed", slog.String("namespace", scope.Namespace), slog.Any("error", err))
continue
}
for _, item := range resp.Results {
existingIDs[item.ID] = struct{}{}
}
}
return c.JSON(http.StatusOK, memory.DeleteResponse{Message: "Memory deleted successfully!"})
// Find and restore missing entries.
var restoredCount int
for _, fsItem := range fsItems {
if _, exists := existingIDs[fsItem.ID]; exists {
continue
}
// Resolve filters from manifest, fallback to first scope.
var filters map[string]any
if manifest != nil {
if entry, ok := manifest.Entries[fsItem.ID]; ok && len(entry.Filters) > 0 {
filters = entry.Filters
}
}
if len(filters) == 0 && len(scopes) > 0 {
filters = buildNamespaceFilters(scopes[0].Namespace, scopes[0].ScopeID, nil)
}
if _, err := h.service.RebuildAdd(c.Request().Context(), fsItem.ID, fsItem.Memory, filters); err != nil {
h.logger.Warn("rebuild add failed", slog.String("id", fsItem.ID), slog.Any("error", err))
continue
}
restoredCount++
}
return c.JSON(http.StatusOK, memory.RebuildResult{
FsCount: len(fsItems),
QdrantCount: len(existingIDs),
MissingCount: len(fsItems) - len(existingIDs),
RestoredCount: restoredCount,
})
}
// --- helpers ---
+17 -16
View File
@@ -13,7 +13,8 @@ import (
mcpgw "github.com/memohai/memoh/internal/mcp"
)
type fileEntry struct {
// FileEntry represents a filesystem entry returned by ExecList.
type FileEntry struct {
Path string
IsDir bool
Size int64
@@ -21,11 +22,11 @@ type fileEntry struct {
ModTime time.Time
}
// execRead reads a file inside the container via cat.
func execRead(ctx context.Context, runner ExecRunner, botID, workDir, filePath string) (string, error) {
// ExecRead reads a file inside the container via cat.
func ExecRead(ctx context.Context, runner ExecRunner, botID, workDir, filePath string) (string, error) {
result, err := runner.ExecWithCapture(ctx, mcpgw.ExecRequest{
BotID: botID,
Command: []string{"/bin/sh", "-c", "cat " + shellQuote(filePath)},
Command: []string{"/bin/sh", "-c", "cat " + ShellQuote(filePath)},
WorkDir: workDir,
})
if err != nil {
@@ -37,13 +38,13 @@ func execRead(ctx context.Context, runner ExecRunner, botID, workDir, filePath s
return result.Stdout, nil
}
// execWrite writes content to a file inside the container using base64 encoding
// ExecWrite writes content to a file inside the container using base64 encoding
// to avoid shell escaping issues.
func execWrite(ctx context.Context, runner ExecRunner, botID, workDir, filePath, content string) error {
func ExecWrite(ctx context.Context, runner ExecRunner, botID, workDir, filePath, content string) error {
encoded := base64.StdEncoding.EncodeToString([]byte(content))
dir := path.Dir(filePath)
script := fmt.Sprintf("mkdir -p %s && echo %s | base64 -d > %s",
shellQuote(dir), shellQuote(encoded), shellQuote(filePath))
ShellQuote(dir), ShellQuote(encoded), ShellQuote(filePath))
result, err := runner.ExecWithCapture(ctx, mcpgw.ExecRequest{
BotID: botID,
Command: []string{"/bin/sh", "-c", script},
@@ -58,9 +59,9 @@ func execWrite(ctx context.Context, runner ExecRunner, botID, workDir, filePath,
return nil
}
// execList lists directory entries inside the container via find + stat.
// ExecList lists directory entries inside the container via find + stat.
// Output format per line: <name>|<type>|<size>|<mode>|<mtime_epoch>
func execList(ctx context.Context, runner ExecRunner, botID, workDir, dirPath string, recursive bool) ([]fileEntry, error) {
func ExecList(ctx context.Context, runner ExecRunner, botID, workDir, dirPath string, recursive bool) ([]FileEntry, error) {
depthFlag := "-maxdepth 1"
if recursive {
depthFlag = ""
@@ -69,7 +70,7 @@ func execList(ctx context.Context, runner ExecRunner, botID, workDir, dirPath st
// busybox stat -c format: %n=name, %F=type, %s=size, %a=octal mode, %Y=mtime epoch
script := fmt.Sprintf(
`find %s %s ! -path %s -exec stat -c '%%n|%%F|%%s|%%a|%%Y' {} \;`,
shellQuote(dirPath), depthFlag, shellQuote(dirPath),
ShellQuote(dirPath), depthFlag, ShellQuote(dirPath),
)
result, err := runner.ExecWithCapture(ctx, mcpgw.ExecRequest{
BotID: botID,
@@ -85,10 +86,10 @@ func execList(ctx context.Context, runner ExecRunner, botID, workDir, dirPath st
return parseStatOutput(result.Stdout, dirPath), nil
}
// parseStatOutput parses lines of "fullpath|type|size|mode|mtime" into fileEntry slices.
func parseStatOutput(output, basePath string) []fileEntry {
// parseStatOutput parses lines of "fullpath|type|size|mode|mtime" into FileEntry slices.
func parseStatOutput(output, basePath string) []FileEntry {
lines := strings.Split(strings.TrimSpace(output), "\n")
entries := make([]fileEntry, 0, len(lines))
entries := make([]FileEntry, 0, len(lines))
// Normalize base path for computing relative paths.
base := strings.TrimSuffix(basePath, "/")
if base == "" || base == "." {
@@ -124,7 +125,7 @@ func parseStatOutput(output, basePath string) []fileEntry {
mtimeEpoch, _ := strconv.ParseInt(mtimeStr, 10, 64)
modTime := time.Unix(mtimeEpoch, 0)
entries = append(entries, fileEntry{
entries = append(entries, FileEntry{
Path: rel,
IsDir: isDir,
Size: size,
@@ -171,8 +172,8 @@ func applyEdit(raw, filePath, oldText, newText string) (string, error) {
return bom + restoreLineEndings(updated, originalEnding), nil
}
// shellQuote wraps a string in single quotes, escaping embedded single quotes.
func shellQuote(s string) string {
// ShellQuote wraps a string in single quotes, escaping embedded single quotes.
func ShellQuote(s string) string {
if s == "" {
return "''"
}
@@ -13,9 +13,9 @@ func TestShellQuote(t *testing.T) {
{"a b", "'a b'"},
}
for _, tt := range tests {
got := shellQuote(tt.in)
got := ShellQuote(tt.in)
if got != tt.want {
t.Errorf("shellQuote(%q) = %q, want %q", tt.in, got, tt.want)
t.Errorf("ShellQuote(%q) = %q, want %q", tt.in, got, tt.want)
}
}
}
+5 -5
View File
@@ -154,7 +154,7 @@ func (p *Executor) CallTool(ctx context.Context, session mcpgw.ToolSessionContex
if filePath == "" {
return mcpgw.BuildToolErrorResult("path is required"), nil
}
content, err := execRead(ctx, p.execRunner, botID, p.execWorkDir, filePath)
content, err := ExecRead(ctx, p.execRunner, botID, p.execWorkDir, filePath)
if err != nil {
return mcpgw.BuildToolErrorResult(err.Error()), nil
}
@@ -166,7 +166,7 @@ func (p *Executor) CallTool(ctx context.Context, session mcpgw.ToolSessionContex
if filePath == "" {
return mcpgw.BuildToolErrorResult("path is required"), nil
}
if err := execWrite(ctx, p.execRunner, botID, p.execWorkDir, filePath, content); err != nil {
if err := ExecWrite(ctx, p.execRunner, botID, p.execWorkDir, filePath, content); err != nil {
return mcpgw.BuildToolErrorResult(err.Error()), nil
}
return mcpgw.BuildToolSuccessResult(map[string]any{"ok": true}), nil
@@ -177,7 +177,7 @@ func (p *Executor) CallTool(ctx context.Context, session mcpgw.ToolSessionContex
dirPath = "."
}
recursive, _, _ := mcpgw.BoolArg(arguments, "recursive")
entries, err := execList(ctx, p.execRunner, botID, p.execWorkDir, dirPath, recursive)
entries, err := ExecList(ctx, p.execRunner, botID, p.execWorkDir, dirPath, recursive)
if err != nil {
return mcpgw.BuildToolErrorResult(err.Error()), nil
}
@@ -201,7 +201,7 @@ func (p *Executor) CallTool(ctx context.Context, session mcpgw.ToolSessionContex
return mcpgw.BuildToolErrorResult("path, old_text and new_text are required"), nil
}
// Step 1: read via exec
raw, err := execRead(ctx, p.execRunner, botID, p.execWorkDir, filePath)
raw, err := ExecRead(ctx, p.execRunner, botID, p.execWorkDir, filePath)
if err != nil {
return mcpgw.BuildToolErrorResult(err.Error()), nil
}
@@ -211,7 +211,7 @@ func (p *Executor) CallTool(ctx context.Context, session mcpgw.ToolSessionContex
return mcpgw.BuildToolErrorResult(err.Error()), nil
}
// Step 3: write back via exec
if err := execWrite(ctx, p.execRunner, botID, p.execWorkDir, filePath, updated); err != nil {
if err := ExecWrite(ctx, p.execRunner, botID, p.execWorkDir, filePath, updated); err != nil {
return mcpgw.BuildToolErrorResult(err.Error()), nil
}
return mcpgw.BuildToolSuccessResult(map[string]any{"ok": true}), nil
+30
View File
@@ -130,6 +130,36 @@ func (c *LLMClient) Decide(ctx context.Context, req DecideRequest) (DecideRespon
return DecideResponse{Actions: actions}, nil
}
func (c *LLMClient) Compact(ctx context.Context, req CompactRequest) (CompactResponse, error) {
if len(req.Memories) == 0 {
return CompactResponse{}, fmt.Errorf("memories is required")
}
memories := make([]map[string]string, 0, len(req.Memories))
for _, m := range req.Memories {
entry := map[string]string{
"id": m.ID,
"text": m.Memory,
}
if m.CreatedAt != "" {
entry["created_at"] = m.CreatedAt
}
memories = append(memories, entry)
}
systemPrompt, userPrompt := getCompactMemoryMessages(memories, req.TargetCount, req.DecayDays)
content, err := c.callChat(ctx, []chatMessage{
{Role: "system", Content: systemPrompt},
{Role: "user", Content: userPrompt},
})
if err != nil {
return CompactResponse{}, err
}
var parsed CompactResponse
if err := json.Unmarshal([]byte(removeCodeBlocks(content)), &parsed); err != nil {
return CompactResponse{}, fmt.Errorf("failed to parse compact response: %w", err)
}
return parsed, nil
}
func (c *LLMClient) DetectLanguage(ctx context.Context, text string) (string, error) {
if strings.TrimSpace(text) == "" {
return "", fmt.Errorf("text is required")
+327
View File
@@ -0,0 +1,327 @@
package memory
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"strings"
"sync"
"time"
mcpgw "github.com/memohai/memoh/internal/mcp"
"github.com/memohai/memoh/internal/mcp/providers/container"
)
const (
manifestPath = "index/manifest.json"
memoryDirPath = "memory"
manifestVer = 1
)
// MemoryFS persists memory entries as files inside the bot container via ExecRunner.
type MemoryFS struct {
execRunner container.ExecRunner
workDir string // e.g. "/data"
logger *slog.Logger
mu sync.Mutex // serialize manifest updates
}
// Manifest is the index file that records everything needed to rebuild memories.
type Manifest struct {
Version int `json:"version"`
UpdatedAt string `json:"updated_at"`
Entries map[string]ManifestEntry `json:"entries"`
}
// ManifestEntry records metadata for a single memory entry.
type ManifestEntry struct {
Hash string `json:"hash"`
CreatedAt string `json:"created_at"`
Lang string `json:"lang,omitempty"`
Filters map[string]any `json:"filters,omitempty"`
}
// NewMemoryFS creates a MemoryFS that writes through the given ExecRunner.
func NewMemoryFS(log *slog.Logger, runner container.ExecRunner, workDir string) *MemoryFS {
if log == nil {
log = slog.Default()
}
if strings.TrimSpace(workDir) == "" {
workDir = "/data"
}
return &MemoryFS{
execRunner: runner,
workDir: workDir,
logger: log.With(slog.String("component", "memoryfs")),
}
}
// ----- write operations -----
// PersistMemories writes .md files for new items and incrementally updates the manifest.
// Used after Add — does NOT delete existing files.
func (fs *MemoryFS) PersistMemories(ctx context.Context, botID string, items []MemoryItem, filters map[string]any) error {
if len(items) == 0 {
return nil
}
fs.mu.Lock()
defer fs.mu.Unlock()
// Read existing manifest (or create new one).
manifest, _ := fs.readManifestLocked(ctx, botID)
if manifest == nil {
manifest = &Manifest{
Version: manifestVer,
Entries: map[string]ManifestEntry{},
}
}
for _, item := range items {
if strings.TrimSpace(item.ID) == "" || strings.TrimSpace(item.Memory) == "" {
continue
}
// Write individual .md file.
if err := fs.writeMemoryFile(ctx, botID, item); err != nil {
fs.logger.Warn("write memory file failed", slog.String("id", item.ID), slog.Any("error", err))
continue
}
// Update manifest entry.
manifest.Entries[item.ID] = ManifestEntry{
Hash: item.Hash,
CreatedAt: item.CreatedAt,
Filters: filters,
}
}
manifest.UpdatedAt = time.Now().UTC().Format(time.RFC3339)
return fs.writeManifest(ctx, botID, manifest)
}
// RebuildFiles does a full replace: deletes all old memory/*.md files, writes new ones,
// and rewrites manifest from scratch. Used after Compact.
func (fs *MemoryFS) RebuildFiles(ctx context.Context, botID string, items []MemoryItem, filters map[string]any) error {
fs.mu.Lock()
defer fs.mu.Unlock()
// Delete old memory dir contents.
fs.execDeleteDir(ctx, botID, memoryDirPath)
manifest := &Manifest{
Version: manifestVer,
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
Entries: make(map[string]ManifestEntry, len(items)),
}
for _, item := range items {
if strings.TrimSpace(item.ID) == "" || strings.TrimSpace(item.Memory) == "" {
continue
}
if err := fs.writeMemoryFile(ctx, botID, item); err != nil {
fs.logger.Warn("rebuild write memory file failed", slog.String("id", item.ID), slog.Any("error", err))
continue
}
manifest.Entries[item.ID] = ManifestEntry{
Hash: item.Hash,
CreatedAt: item.CreatedAt,
Filters: filters,
}
}
return fs.writeManifest(ctx, botID, manifest)
}
// RemoveMemories removes specific memory files from the FS and updates the manifest.
func (fs *MemoryFS) RemoveMemories(ctx context.Context, botID string, ids []string) error {
if len(ids) == 0 {
return nil
}
fs.mu.Lock()
defer fs.mu.Unlock()
manifest, _ := fs.readManifestLocked(ctx, botID)
if manifest == nil {
manifest = &Manifest{Version: manifestVer, Entries: map[string]ManifestEntry{}}
}
for _, id := range ids {
id = strings.TrimSpace(id)
if id == "" {
continue
}
fs.execDeleteFile(ctx, botID, fmt.Sprintf("%s/%s.md", memoryDirPath, id))
delete(manifest.Entries, id)
}
manifest.UpdatedAt = time.Now().UTC().Format(time.RFC3339)
return fs.writeManifest(ctx, botID, manifest)
}
// RemoveAllMemories deletes all memory files and the manifest.
func (fs *MemoryFS) RemoveAllMemories(ctx context.Context, botID string) error {
fs.mu.Lock()
defer fs.mu.Unlock()
fs.execDeleteDir(ctx, botID, memoryDirPath)
emptyManifest := &Manifest{
Version: manifestVer,
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
Entries: map[string]ManifestEntry{},
}
return fs.writeManifest(ctx, botID, emptyManifest)
}
// ----- read operations -----
// ReadManifest reads and parses the manifest.json file.
func (fs *MemoryFS) ReadManifest(ctx context.Context, botID string) (*Manifest, error) {
fs.mu.Lock()
defer fs.mu.Unlock()
return fs.readManifestLocked(ctx, botID)
}
func (fs *MemoryFS) readManifestLocked(ctx context.Context, botID string) (*Manifest, error) {
content, err := container.ExecRead(ctx, fs.execRunner, botID, fs.workDir, manifestPath)
if err != nil {
return nil, err
}
var manifest Manifest
if err := json.Unmarshal([]byte(content), &manifest); err != nil {
return nil, fmt.Errorf("parse manifest: %w", err)
}
return &manifest, nil
}
// ReadAllMemoryFiles lists and reads all .md files under memory/ and parses their frontmatter.
func (fs *MemoryFS) ReadAllMemoryFiles(ctx context.Context, botID string) ([]MemoryItem, error) {
entries, err := container.ExecList(ctx, fs.execRunner, botID, fs.workDir, memoryDirPath, false)
if err != nil {
return nil, fmt.Errorf("list memory dir: %w", err)
}
var items []MemoryItem
for _, entry := range entries {
if entry.IsDir || !strings.HasSuffix(entry.Path, ".md") {
continue
}
filePath := memoryDirPath + "/" + entry.Path
content, err := container.ExecRead(ctx, fs.execRunner, botID, fs.workDir, filePath)
if err != nil {
fs.logger.Warn("read memory file failed", slog.String("path", filePath), slog.Any("error", err))
continue
}
item, err := parseMemoryMD(content)
if err != nil {
fs.logger.Warn("parse memory file failed", slog.String("path", filePath), slog.Any("error", err))
continue
}
items = append(items, item)
}
return items, nil
}
// ----- internal helpers -----
func (fs *MemoryFS) writeMemoryFile(ctx context.Context, botID string, item MemoryItem) error {
content := formatMemoryMD(item)
filePath := fmt.Sprintf("%s/%s.md", memoryDirPath, item.ID)
return container.ExecWrite(ctx, fs.execRunner, botID, fs.workDir, filePath, content)
}
func (fs *MemoryFS) writeManifest(ctx context.Context, botID string, manifest *Manifest) error {
data, err := json.MarshalIndent(manifest, "", " ")
if err != nil {
return fmt.Errorf("marshal manifest: %w", err)
}
return container.ExecWrite(ctx, fs.execRunner, botID, fs.workDir, manifestPath, string(data))
}
// execDeleteDir removes all files inside a directory (but keeps the directory itself).
func (fs *MemoryFS) execDeleteDir(ctx context.Context, botID, dirPath string) {
// Use find + rm to avoid shell quoting issues with glob wildcards.
script := fmt.Sprintf("find %s -type f -delete 2>/dev/null; true", container.ShellQuote(dirPath))
_, err := fs.execRunner.ExecWithCapture(ctx, mcpgw.ExecRequest{
BotID: botID,
Command: []string{"/bin/sh", "-c", script},
WorkDir: fs.workDir,
})
if err != nil {
fs.logger.Warn("exec delete dir failed", slog.String("path", dirPath), slog.Any("error", err))
}
}
// execDeleteFile removes a single file.
func (fs *MemoryFS) execDeleteFile(ctx context.Context, botID, filePath string) {
script := fmt.Sprintf("rm -f %s", container.ShellQuote(filePath))
_, err := fs.execRunner.ExecWithCapture(ctx, mcpgw.ExecRequest{
BotID: botID,
Command: []string{"/bin/sh", "-c", script},
WorkDir: fs.workDir,
})
if err != nil {
fs.logger.Warn("exec delete file failed", slog.String("path", filePath), slog.Any("error", err))
}
}
// ----- .md formatting / parsing -----
func formatMemoryMD(item MemoryItem) string {
var b strings.Builder
b.WriteString("---\n")
b.WriteString(fmt.Sprintf("id: %s\n", item.ID))
if item.Hash != "" {
b.WriteString(fmt.Sprintf("hash: %s\n", item.Hash))
}
if item.CreatedAt != "" {
b.WriteString(fmt.Sprintf("created_at: %s\n", item.CreatedAt))
}
if item.UpdatedAt != "" {
b.WriteString(fmt.Sprintf("updated_at: %s\n", item.UpdatedAt))
}
b.WriteString("---\n")
b.WriteString(item.Memory)
b.WriteString("\n")
return b.String()
}
func parseMemoryMD(content string) (MemoryItem, error) {
content = strings.TrimSpace(content)
if !strings.HasPrefix(content, "---") {
return MemoryItem{}, fmt.Errorf("missing frontmatter")
}
// Split on "---" delimiters.
parts := strings.SplitN(content[3:], "---", 2)
if len(parts) < 2 {
return MemoryItem{}, fmt.Errorf("incomplete frontmatter")
}
frontmatter := strings.TrimSpace(parts[0])
body := strings.TrimSpace(parts[1])
item := MemoryItem{Memory: body}
for _, line := range strings.Split(frontmatter, "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
key, value, found := strings.Cut(line, ":")
if !found {
continue
}
key = strings.TrimSpace(key)
value = strings.TrimSpace(value)
switch key {
case "id":
item.ID = value
case "hash":
item.Hash = value
case "created_at":
item.CreatedAt = value
case "updated_at":
item.UpdatedAt = value
}
}
if item.ID == "" {
return MemoryItem{}, fmt.Errorf("missing id in frontmatter")
}
return item, nil
}
+36
View File
@@ -106,6 +106,42 @@ Follow the instruction mentioned below:
Do not return anything except the JSON format.`, toJSON(retrievedOldMemory), toJSON(newRetrievedFacts), "```json", "```")
}
func getCompactMemoryMessages(memories []map[string]string, targetCount int, decayDays int) (string, string) {
decayInstruction := ""
if decayDays > 0 {
decayInstruction = fmt.Sprintf(`
10. TIME DECAY: Today's date is %s. Memories older than %d days are LOW PRIORITY.
- When deciding which facts to merge or drop, prefer dropping/merging older low-priority memories over newer ones.
- If an older memory and a newer memory convey similar information, keep the newer one.
- Very old memories should only be kept if they contain unique, still-relevant information (e.g. name, identity, long-term preferences).
`, time.Now().UTC().Format("2006-01-02"), decayDays)
}
systemPrompt := fmt.Sprintf(`You are a Memory Compactor. Your job is to consolidate a list of memory entries into a smaller, more concise set.
Guidelines:
1. Merge similar or redundant entries into single, concise facts.
2. If two entries contradict each other, keep only the more recent or more specific one.
3. Preserve all unique, non-redundant information do not lose important facts.
4. Each output fact should be a single, self-contained statement.
5. Target approximately %d output facts (but use fewer if the information naturally consolidates to less, and never produce more than the input count).
6. Keep the same language as the original memories. Do not translate.
7. Return a JSON object with a single key "facts" containing an array of strings.
8. DO NOT RETURN ANYTHING ELSE OTHER THAN THE JSON FORMAT.
9. DO NOT ADD ANY ADDITIONAL TEXT OR CODEBLOCK IN THE JSON FIELDS WHICH MAKE IT INVALID SUCH AS "%s" OR "%s".%s
Example:
Input memories:
[{"id":"1","text":"User likes dark mode","created_at":"2026-01-01"},{"id":"2","text":"User prefers dark theme for all apps","created_at":"2026-02-10"},{"id":"3","text":"User is a software engineer","created_at":"2026-01-15"},{"id":"4","text":"User works as a developer","created_at":"2026-02-01"}]
Target: 2
Output: {"facts": ["User prefers dark theme for all apps", "User is a software engineer"]}
`, targetCount, "```json", "```", decayInstruction)
userPrompt := fmt.Sprintf("Consolidate the following memories into approximately %d concise facts:\n\n%s", targetCount, toJSON(memories))
return systemPrompt, userPrompt
}
func getLanguageDetectionMessages(text string) (string, string) {
systemPrompt := `You are a language classifier for the given input text.
Return a JSON object with a single key "language" whose value is one of the allowed codes.
+29
View File
@@ -329,6 +329,22 @@ func (s *QdrantStore) Delete(ctx context.Context, id string) error {
return err
}
func (s *QdrantStore) DeleteBatch(ctx context.Context, ids []string) error {
if len(ids) == 0 {
return nil
}
pointIDs := make([]*qdrant.PointId, 0, len(ids))
for _, id := range ids {
pointIDs = append(pointIDs, qdrant.NewIDUUID(id))
}
_, err := s.client.Delete(ctx, &qdrant.DeletePoints{
CollectionName: s.collection,
Wait: qdrant.PtrOf(true),
Points: qdrant.NewPointsSelectorIDs(pointIDs),
})
return err
}
func (s *QdrantStore) List(ctx context.Context, limit int, filters map[string]any) ([]qdrantPoint, error) {
if limit <= 0 {
limit = 100
@@ -379,6 +395,19 @@ func (s *QdrantStore) Scroll(ctx context.Context, limit int, filters map[string]
return result, nextOffset, nil
}
func (s *QdrantStore) Count(ctx context.Context, filters map[string]any) (uint64, error) {
filter := buildQdrantFilter(filters)
result, err := s.client.Count(ctx, &qdrant.CountPoints{
CollectionName: s.collection,
Filter: filter,
Exact: qdrant.PtrOf(true),
})
if err != nil {
return 0, err
}
return result, nil
}
func (s *QdrantStore) DeleteAll(ctx context.Context, filters map[string]any) error {
filter := buildQdrantFilter(filters)
if filter == nil {
+193
View File
@@ -6,6 +6,7 @@ import (
"encoding/hex"
"fmt"
"log/slog"
"math"
"sort"
"strings"
"time"
@@ -448,6 +449,26 @@ func (s *Service) Delete(ctx context.Context, memoryID string) (DeleteResponse,
return DeleteResponse{Message: "Memory deleted successfully!"}, nil
}
func (s *Service) DeleteBatch(ctx context.Context, memoryIDs []string) (DeleteResponse, error) {
if len(memoryIDs) == 0 {
return DeleteResponse{}, fmt.Errorf("memory_ids is required")
}
cleaned := make([]string, 0, len(memoryIDs))
for _, id := range memoryIDs {
id = strings.TrimSpace(id)
if id != "" {
cleaned = append(cleaned, id)
}
}
if len(cleaned) == 0 {
return DeleteResponse{}, fmt.Errorf("memory_ids is required")
}
if err := s.store.DeleteBatch(ctx, cleaned); err != nil {
return DeleteResponse{}, err
}
return DeleteResponse{Message: fmt.Sprintf("%d memories deleted successfully!", len(cleaned))}, nil
}
func (s *Service) DeleteAll(ctx context.Context, req DeleteAllRequest) (DeleteResponse, error) {
filters := map[string]any{}
for k, v := range req.Filters {
@@ -471,6 +492,142 @@ func (s *Service) DeleteAll(ctx context.Context, req DeleteAllRequest) (DeleteRe
return DeleteResponse{Message: "Memories deleted successfully!"}, nil
}
func (s *Service) Compact(ctx context.Context, filters map[string]any, ratio float64, decayDays int) (CompactResult, error) {
if s.llm == nil {
return CompactResult{}, fmt.Errorf("llm not configured")
}
if s.store == nil {
return CompactResult{}, fmt.Errorf("qdrant store not configured")
}
if ratio <= 0 || ratio > 1 {
ratio = 0.5
}
// Fetch all existing memories.
points, err := s.store.List(ctx, 0, filters)
if err != nil {
return CompactResult{}, err
}
beforeCount := len(points)
if beforeCount <= 1 {
// Nothing to compact.
items := make([]MemoryItem, 0, len(points))
for _, p := range points {
items = append(items, payloadToMemoryItem(p.ID, p.Payload))
}
return CompactResult{
BeforeCount: beforeCount,
AfterCount: beforeCount,
Ratio: 1.0,
Results: items,
}, nil
}
// Build candidate list and compute target.
candidates := make([]CandidateMemory, 0, beforeCount)
for _, p := range points {
candidates = append(candidates, CandidateMemory{
ID: p.ID,
Memory: fmt.Sprint(p.Payload["data"]),
CreatedAt: fmt.Sprint(p.Payload["created_at"]),
})
}
targetCount := int(math.Round(float64(beforeCount) * ratio))
if targetCount < 1 {
targetCount = 1
}
// Ask LLM to consolidate.
compactResp, err := s.llm.Compact(ctx, CompactRequest{
Memories: candidates,
TargetCount: targetCount,
DecayDays: decayDays,
})
if err != nil {
return CompactResult{}, fmt.Errorf("compact llm call failed: %w", err)
}
if len(compactResp.Facts) == 0 {
return CompactResult{}, fmt.Errorf("compact returned no facts")
}
// Delete old memories.
if err := s.store.DeleteAll(ctx, filters); err != nil {
return CompactResult{}, fmt.Errorf("compact delete old failed: %w", err)
}
// Reset BM25 stats for deleted documents.
if s.bm25 != nil {
for _, p := range points {
text := fmt.Sprint(p.Payload["data"])
lang := fmt.Sprint(p.Payload["lang"])
if strings.TrimSpace(text) == "" || strings.TrimSpace(lang) == "" {
continue
}
freq, docLen, err := s.bm25.TermFrequencies(lang, text)
if err != nil {
continue
}
s.bm25.RemoveDocument(lang, freq, docLen)
}
}
// Add compacted facts.
results := make([]MemoryItem, 0, len(compactResp.Facts))
for _, fact := range compactResp.Facts {
if strings.TrimSpace(fact) == "" {
continue
}
item, err := s.applyAdd(ctx, fact, filters, nil, false)
if err != nil {
return CompactResult{}, fmt.Errorf("compact add failed: %w", err)
}
results = append(results, item)
}
afterCount := len(results)
actualRatio := float64(afterCount) / float64(beforeCount)
return CompactResult{
BeforeCount: beforeCount,
AfterCount: afterCount,
Ratio: math.Round(actualRatio*100) / 100,
Results: results,
}, nil
}
const (
// Estimated sparse vector overhead per point: ~200 dims * 8 bytes (4 index + 4 value).
sparseVectorOverheadBytes = 1600
// Estimated payload metadata overhead per point (hash, dates, filters, lang, metadata JSON).
payloadMetadataOverheadBytes = 256
)
func (s *Service) Usage(ctx context.Context, filters map[string]any) (UsageResponse, error) {
if s.store == nil {
return UsageResponse{}, fmt.Errorf("qdrant store not configured")
}
points, err := s.store.List(ctx, 0, filters)
if err != nil {
return UsageResponse{}, err
}
count := len(points)
var totalTextBytes int64
for _, p := range points {
text := fmt.Sprint(p.Payload["data"])
totalTextBytes += int64(len(text))
}
var avgTextBytes int64
if count > 0 {
avgTextBytes = totalTextBytes / int64(count)
}
estimatedStorage := totalTextBytes + int64(count)*(sparseVectorOverheadBytes+payloadMetadataOverheadBytes)
return UsageResponse{
Count: count,
TotalTextBytes: totalTextBytes,
AvgTextBytes: avgTextBytes,
EstimatedStorageBytes: estimatedStorage,
}, nil
}
func (s *Service) WarmupBM25(ctx context.Context, batchSize int) error {
if s.bm25 == nil || s.store == nil {
return nil
@@ -602,6 +759,42 @@ func (s *Service) applyAdd(ctx context.Context, text string, filters map[string]
return payloadToMemoryItem(id, payload), nil
}
// RebuildAdd inserts a memory with a specific ID (from filesystem recovery).
// Like applyAdd but preserves the given ID instead of generating a new UUID.
func (s *Service) RebuildAdd(ctx context.Context, id, text string, filters map[string]any) (MemoryItem, error) {
if s.store == nil {
return MemoryItem{}, fmt.Errorf("qdrant store not configured")
}
if s.bm25 == nil {
return MemoryItem{}, fmt.Errorf("bm25 indexer not configured")
}
if strings.TrimSpace(id) == "" {
return MemoryItem{}, fmt.Errorf("id is required for rebuild")
}
lang, err := s.detectLanguage(ctx, text)
if err != nil {
return MemoryItem{}, err
}
termFreq, docLen, err := s.bm25.TermFrequencies(lang, text)
if err != nil {
return MemoryItem{}, err
}
sparseIndices, sparseValues := s.bm25.AddDocument(lang, termFreq, docLen)
payload := buildPayload(text, filters, nil, "")
payload["lang"] = lang
point := qdrantPoint{
ID: id,
SparseIndices: sparseIndices,
SparseValues: sparseValues,
SparseVectorName: s.store.sparseVectorName,
Payload: payload,
}
if err := s.store.Upsert(ctx, []qdrantPoint{point}); err != nil {
return MemoryItem{}, err
}
return payloadToMemoryItem(id, payload), nil
}
func (s *Service) applyUpdate(ctx context.Context, id, text string, filters map[string]any, metadata map[string]any, embeddingEnabled bool) (MemoryItem, error) {
if strings.TrimSpace(id) == "" {
return MemoryItem{}, fmt.Errorf("update action missing id")
+7
View File
@@ -11,6 +11,7 @@ import (
type MockLLM struct {
ExtractFunc func(ctx context.Context, req ExtractRequest) (ExtractResponse, error)
DecideFunc func(ctx context.Context, req DecideRequest) (DecideResponse, error)
CompactFunc func(ctx context.Context, req CompactRequest) (CompactResponse, error)
DetectLanguageFunc func(ctx context.Context, text string) (string, error)
}
@@ -20,6 +21,12 @@ func (m *MockLLM) Extract(ctx context.Context, req ExtractRequest) (ExtractRespo
func (m *MockLLM) Decide(ctx context.Context, req DecideRequest) (DecideResponse, error) {
return m.DecideFunc(ctx, req)
}
func (m *MockLLM) Compact(ctx context.Context, req CompactRequest) (CompactResponse, error) {
if m.CompactFunc != nil {
return m.CompactFunc(ctx, req)
}
return CompactResponse{}, fmt.Errorf("compact not mocked")
}
func (m *MockLLM) DetectLanguage(ctx context.Context, text string) (string, error) {
return m.DetectLanguageFunc(ctx, text)
}
+36 -3
View File
@@ -6,6 +6,7 @@ import "context"
type LLM interface {
Extract(ctx context.Context, req ExtractRequest) (ExtractResponse, error)
Decide(ctx context.Context, req DecideRequest) (DecideResponse, error)
Compact(ctx context.Context, req CompactRequest) (CompactResponse, error)
DetectLanguage(ctx context.Context, text string) (string, error)
}
@@ -117,9 +118,10 @@ type ExtractResponse struct {
}
type CandidateMemory struct {
ID string `json:"id"`
Memory string `json:"memory"`
Metadata map[string]any `json:"metadata,omitempty"`
ID string `json:"id"`
Memory string `json:"memory"`
CreatedAt string `json:"created_at,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type DecideRequest struct {
@@ -139,3 +141,34 @@ type DecisionAction struct {
type DecideResponse struct {
Actions []DecisionAction `json:"actions"`
}
type CompactRequest struct {
Memories []CandidateMemory `json:"memories"`
TargetCount int `json:"target_count"`
DecayDays int `json:"decay_days,omitempty"`
}
type CompactResponse struct {
Facts []string `json:"facts"`
}
type CompactResult struct {
BeforeCount int `json:"before_count"`
AfterCount int `json:"after_count"`
Ratio float64 `json:"ratio"`
Results []MemoryItem `json:"results"`
}
type UsageResponse struct {
Count int `json:"count"`
TotalTextBytes int64 `json:"total_text_bytes"`
AvgTextBytes int64 `json:"avg_text_bytes"`
EstimatedStorageBytes int64 `json:"estimated_storage_bytes"`
}
type RebuildResult struct {
FsCount int `json:"fs_count"`
QdrantCount int `json:"qdrant_count"`
MissingCount int `json:"missing_count"`
RestoredCount int `json:"restored_count"`
}
+986 -11
View File
File diff suppressed because it is too large Load Diff
+986 -11
View File
File diff suppressed because it is too large Load Diff
+664 -9
View File
@@ -78,6 +78,8 @@ definitions:
type: object
bots.Bot:
properties:
allow_guest:
type: boolean
avatar_url:
type: string
check_issue_count:
@@ -150,6 +152,13 @@ definitions:
$ref: '#/definitions/bots.Bot'
type: array
type: object
bots.ListCheckKeysResponse:
properties:
keys:
items:
type: string
type: array
type: object
bots.ListChecksResponse:
properties:
items:
@@ -514,8 +523,6 @@ definitions:
type: object
github_com_memohai_memoh_internal_mcp.Connection:
properties:
active:
type: boolean
bot_id:
type: string
config:
@@ -525,6 +532,8 @@ definitions:
type: string
id:
type: string
is_active:
type: boolean
name:
type: string
type:
@@ -532,6 +541,13 @@ definitions:
updated_at:
type: string
type: object
handlers.BatchDeleteRequest:
properties:
ids:
items:
type: string
type: array
type: object
handlers.ChannelMeta:
properties:
capabilities:
@@ -622,12 +638,12 @@ definitions:
type: object
handlers.EmbeddingsUsage:
properties:
duration:
type: integer
image_tokens:
type: integer
input_tokens:
type: integer
video_tokens:
type: integer
type: object
handlers.ErrorResponse:
properties:
@@ -777,6 +793,61 @@ definitions:
user_id:
type: string
type: object
handlers.memoryAddPayload:
properties:
embedding_enabled:
type: boolean
filters:
additionalProperties: {}
type: object
infer:
type: boolean
message:
type: string
messages:
items:
$ref: '#/definitions/memory.Message'
type: array
metadata:
additionalProperties: {}
type: object
namespace:
type: string
run_id:
type: string
type: object
handlers.memoryCompactPayload:
properties:
decay_days:
type: integer
ratio:
type: number
type: object
handlers.memoryDeletePayload:
properties:
memory_ids:
items:
type: string
type: array
type: object
handlers.memorySearchPayload:
properties:
embedding_enabled:
type: boolean
filters:
additionalProperties: {}
type: object
limit:
type: integer
query:
type: string
run_id:
type: string
sources:
items:
type: string
type: array
type: object
handlers.skillsOpResponse:
properties:
ok:
@@ -804,6 +875,20 @@ definitions:
user_id:
type: string
type: object
mcp.ExportResponse:
properties:
mcpServers:
additionalProperties:
$ref: '#/definitions/mcp.MCPServerEntry'
type: object
type: object
mcp.ImportRequest:
properties:
mcpServers:
additionalProperties:
$ref: '#/definitions/mcp.MCPServerEntry'
type: object
type: object
mcp.ListResponse:
properties:
items:
@@ -811,17 +896,136 @@ definitions:
$ref: '#/definitions/github_com_memohai_memoh_internal_mcp.Connection'
type: array
type: object
mcp.MCPServerEntry:
properties:
args:
items:
type: string
type: array
command:
type: string
cwd:
type: string
env:
additionalProperties:
type: string
type: object
headers:
additionalProperties:
type: string
type: object
transport:
type: string
url:
type: string
type: object
mcp.UpsertRequest:
properties:
active:
type: boolean
config:
additionalProperties: {}
args:
items:
type: string
type: array
command:
type: string
cwd:
type: string
env:
additionalProperties:
type: string
type: object
headers:
additionalProperties:
type: string
type: object
is_active:
type: boolean
name:
type: string
type:
transport:
type: string
url:
type: string
type: object
memory.CompactResult:
properties:
after_count:
type: integer
before_count:
type: integer
ratio:
type: number
results:
items:
$ref: '#/definitions/memory.MemoryItem'
type: array
type: object
memory.DeleteResponse:
properties:
message:
type: string
type: object
memory.MemoryItem:
properties:
agent_id:
type: string
bot_id:
type: string
created_at:
type: string
hash:
type: string
id:
type: string
memory:
type: string
metadata:
additionalProperties: {}
type: object
run_id:
type: string
score:
type: number
updated_at:
type: string
type: object
memory.Message:
properties:
content:
type: string
role:
type: string
type: object
memory.RebuildResult:
properties:
fs_count:
type: integer
missing_count:
type: integer
qdrant_count:
type: integer
restored_count:
type: integer
type: object
memory.SearchResponse:
properties:
relations:
items: {}
type: array
results:
items:
$ref: '#/definitions/memory.MemoryItem'
type: array
type: object
memory.UsageResponse:
properties:
avg_text_bytes:
type: integer
count:
type: integer
estimated_storage_bytes:
type: integer
total_text_bytes:
type: integer
type: object
models.AddRequest:
properties:
@@ -1602,6 +1806,34 @@ paths:
summary: Create MCP connection
tags:
- mcp
/bots/{bot_id}/mcp-ops/batch-delete:
post:
description: Delete multiple MCP connections by IDs.
parameters:
- description: IDs to delete
in: body
name: payload
required: true
schema:
$ref: '#/definitions/handlers.BatchDeleteRequest'
responses:
"204":
description: No Content
"400":
description: Bad Request
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"403":
description: Forbidden
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/handlers.ErrorResponse'
summary: Batch delete MCP connections
tags:
- mcp
/bots/{bot_id}/mcp-stdio:
post:
description: Start a stdio MCP process in the bot container and expose it as
@@ -1779,6 +2011,390 @@ paths:
summary: Update MCP connection
tags:
- mcp
/bots/{bot_id}/mcp/export:
get:
description: Export all MCP connections for a bot in standard mcpServers format.
responses:
"200":
description: OK
schema:
$ref: '#/definitions/mcp.ExportResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"403":
description: Forbidden
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/handlers.ErrorResponse'
summary: Export MCP connections
tags:
- mcp
/bots/{bot_id}/mcp/import:
put:
description: Batch import MCP connections from standard mcpServers format. Existing
connections (matched by name) get config updated with is_active preserved.
New connections are created as active.
parameters:
- description: mcpServers dict
in: body
name: payload
required: true
schema:
$ref: '#/definitions/mcp.ImportRequest'
responses:
"200":
description: OK
schema:
$ref: '#/definitions/mcp.ListResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"403":
description: Forbidden
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/handlers.ErrorResponse'
summary: Import MCP connections
tags:
- mcp
/bots/{bot_id}/memory:
delete:
consumes:
- application/json
description: Delete specific memories by IDs, or delete all memories if no IDs
are provided
parameters:
- description: Bot ID
in: path
name: bot_id
required: true
type: string
- description: 'Optional: specify memory_ids to delete; if omitted, deletes
all'
in: body
name: payload
schema:
$ref: '#/definitions/handlers.memoryDeletePayload'
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/memory.DeleteResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"403":
description: Forbidden
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"503":
description: Service Unavailable
schema:
$ref: '#/definitions/handlers.ErrorResponse'
summary: Delete memories
tags:
- memory
get:
description: List all memories in the bot-shared namespace
parameters:
- description: Bot ID
in: path
name: bot_id
required: true
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/memory.SearchResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"403":
description: Forbidden
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"503":
description: Service Unavailable
schema:
$ref: '#/definitions/handlers.ErrorResponse'
summary: Get all memories
tags:
- memory
post:
consumes:
- application/json
description: Add memory into the bot-shared namespace
parameters:
- description: Bot ID
in: path
name: bot_id
required: true
type: string
- description: Memory add payload
in: body
name: payload
required: true
schema:
$ref: '#/definitions/handlers.memoryAddPayload'
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/memory.SearchResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"403":
description: Forbidden
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"503":
description: Service Unavailable
schema:
$ref: '#/definitions/handlers.ErrorResponse'
summary: Add memory
tags:
- memory
/bots/{bot_id}/memory/{id}:
delete:
description: Delete a single memory by its ID
parameters:
- description: Bot ID
in: path
name: bot_id
required: true
type: string
- description: Memory ID
in: path
name: id
required: true
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/memory.DeleteResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"403":
description: Forbidden
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"503":
description: Service Unavailable
schema:
$ref: '#/definitions/handlers.ErrorResponse'
summary: Delete a single memory
tags:
- memory
/bots/{bot_id}/memory/compact:
post:
consumes:
- application/json
description: |-
Consolidate memories by merging similar/redundant entries using LLM.
**ratio** (required, range (0,1]):
- 0.8 = light compression, mostly dedup, keep ~80% of entries
- 0.5 = moderate compression, merge similar facts, keep ~50%
- 0.3 = aggressive compression, heavily consolidate, keep ~30%
**decay_days** (optional): enable time decay — memories older than N days are treated as low priority and more likely to be merged/dropped.
parameters:
- description: Bot ID
in: path
name: bot_id
required: true
type: string
- description: ratio (0,1] required; decay_days optional
in: body
name: payload
required: true
schema:
$ref: '#/definitions/handlers.memoryCompactPayload'
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/memory.CompactResult'
"400":
description: Bad Request
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"403":
description: Forbidden
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"503":
description: Service Unavailable
schema:
$ref: '#/definitions/handlers.ErrorResponse'
summary: Compact memories
tags:
- memory
/bots/{bot_id}/memory/rebuild:
post:
description: Read memory files from the container filesystem (source of truth)
and restore missing entries to Qdrant
parameters:
- description: Bot ID
in: path
name: bot_id
required: true
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/memory.RebuildResult'
"400":
description: Bad Request
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"403":
description: Forbidden
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"503":
description: Service Unavailable
schema:
$ref: '#/definitions/handlers.ErrorResponse'
summary: Rebuild memories from filesystem
tags:
- memory
/bots/{bot_id}/memory/search:
post:
consumes:
- application/json
description: Search memory in the bot-shared namespace
parameters:
- description: Bot ID
in: path
name: bot_id
required: true
type: string
- description: Memory search payload
in: body
name: payload
required: true
schema:
$ref: '#/definitions/handlers.memorySearchPayload'
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/memory.SearchResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"403":
description: Forbidden
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"404":
description: Not Found
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"503":
description: Service Unavailable
schema:
$ref: '#/definitions/handlers.ErrorResponse'
summary: Search memory
tags:
- memory
/bots/{bot_id}/memory/usage:
get:
description: Query the estimated storage usage of current memories
parameters:
- description: Bot ID
in: path
name: bot_id
required: true
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/memory.UsageResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"403":
description: Forbidden
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/handlers.ErrorResponse'
"503":
description: Service Unavailable
schema:
$ref: '#/definitions/handlers.ErrorResponse'
summary: Get memory usage
tags:
- memory
/bots/{bot_id}/schedule:
get:
description: List schedules for current user
@@ -2628,6 +3244,45 @@ paths:
summary: List bot runtime checks
tags:
- bots
/bots/{id}/checks/keys:
get:
description: Returns all check keys available for a bot (builtin + MCP connections)
parameters:
- description: Bot ID
in: path
name: id
required: true
type: string
responses:
"200":
description: OK
schema:
$ref: '#/definitions/bots.ListCheckKeysResponse'
summary: List available check keys
tags:
- bots
/bots/{id}/checks/run/{key}:
get:
description: Evaluate one check key for a bot
parameters:
- description: Bot ID
in: path
name: id
required: true
type: string
- description: Check key
in: path
name: key
required: true
type: string
responses:
"200":
description: OK
schema:
$ref: '#/definitions/bots.BotCheck'
summary: Run a single bot check
tags:
- bots
/bots/{id}/members:
get:
description: List members for a bot