diff --git a/.air.toml b/.air.toml index 8df22911..8b8d40b3 100644 --- a/.air.toml +++ b/.air.toml @@ -2,7 +2,7 @@ root = "." tmp_dir = "tmp" [build] -cmd = "go build -o ./tmp/memoh-server ./cmd/agent/main.go" +cmd = "go build -o ./tmp/memoh-server ./cmd/agent/main.go && sh devenv/mcp-build.sh" bin = "./tmp/memoh-server" args_bin = ["serve"] include_ext = ["go", "toml"] diff --git a/AGENTS.md b/AGENTS.md index 79fbb38e..9687adf8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,18 +125,11 @@ Memoh/ ├── db/ # Database │ ├── migrations/ # SQL migration files │ └── queries/ # SQL query files (sqlc input) -├── conf/ # Configuration templates -│ ├── app.example.toml # Default configuration template -│ ├── app.dev.toml # Development configuration -│ ├── app.docker.toml # Docker deployment configuration -│ ├── app.apple.toml # macOS (Apple Virtualization) configuration -│ └── app.windows.toml # Windows configuration -├── devenv/ # Development environment (docker-compose for local infra) -├── docker/ # Docker build & runtime (Dockerfiles, entrypoints, nginx) +├── conf/ # Configuration templates (app.example.toml, app.docker.toml) +├── devenv/ # Dev environment (docker-compose, dev Dockerfiles, app.dev.toml, mcp-build.sh) +├── docker/ # Production Docker build & runtime (Dockerfiles, entrypoints, nginx.conf) ├── docs/ # Documentation site -├── scripts/ # Utility scripts -├── assets/ # Static assets (images, etc.) -├── data/ # Runtime data directory +├── scripts/ # Utility scripts (db, release, install) ├── docker-compose.yml # Docker Compose orchestration (production) ├── mise.toml # mise tasks and tool version definitions ├── sqlc.yaml # sqlc code generation config diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7eb9dd08..1e24776d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,9 +59,9 @@ mise run dev:restart -- server # Restart a specific service ## Project Layout ``` -conf/ — Configuration templates (app.example.toml, app.dev.toml, app.docker.toml) -devenv/ — Containerized development environment (docker-compose) -docker/ — Docker build & runtime (Dockerfiles, entrypoints) +conf/ — Configuration templates (app.example.toml, app.docker.toml) +devenv/ — Dev environment (docker-compose, dev Dockerfiles, app.dev.toml, mcp-build.sh) +docker/ — Production Docker build & runtime (Dockerfiles, entrypoints) cmd/ — Go application entry points internal/ — Go backend core code agent/ — Agent Gateway (Bun/Elysia) diff --git a/conf/app.example.toml b/conf/app.example.toml index f579024f..7c3df7ad 100644 --- a/conf/app.example.toml +++ b/conf/app.example.toml @@ -1,6 +1,6 @@ # Memoh configuration template # Copy to config.toml and adjust values for your environment. -# For local development, use: cp conf/app.dev.toml config.toml +# For local development, use: cp devenv/app.dev.toml config.toml [log] level = "info" diff --git a/docker/Dockerfile.server.dev b/devenv/Dockerfile.server similarity index 56% rename from docker/Dockerfile.server.dev rename to devenv/Dockerfile.server index 9c3a2185..63eac40b 100644 --- a/docker/Dockerfile.server.dev +++ b/devenv/Dockerfile.server @@ -1,5 +1,28 @@ # syntax=docker/dockerfile:1 +# ---- Stage 1: Assemble MCP image rootfs (runtime deps only, no Go binary) ---- +FROM alpine:latest AS mcp-rootfs + +RUN apk add --no-cache grep curl bash +RUN apk add --no-cache nodejs npm +RUN apk add --no-cache python3 && \ + curl -LsSf https://astral.sh/uv/install.sh | sh && \ + ln -sf /root/.local/bin/uv /usr/local/bin/uv && \ + ln -sf /root/.local/bin/uvx /usr/local/bin/uvx + +COPY cmd/mcp/template /opt/mcp-template + +RUN printf '#!/bin/sh\n\ +[ -e /app/mcp ] || { mkdir -p /app; [ -f /opt/mcp ] && cp -a /opt/mcp /app/mcp 2>/dev/null || true; }\n\ +if [ -x /app/mcp ]; then exec /app/mcp "$@"; fi\n\ +exec /opt/mcp "$@"\n' > /opt/entrypoint.sh && chmod +x /opt/entrypoint.sh + +RUN tar -cf /tmp/rootfs.tar \ + --exclude='./proc' --exclude='./sys' --exclude='./dev' \ + --exclude='./tmp' --exclude='./run' \ + -C / . + +# ---- Stage 2: Dev server image ---- FROM golang:1.25-alpine WORKDIR /workspace @@ -50,7 +73,10 @@ RUN printf '%s\n' \ ' ]' \ '}' > /etc/cni/net.d/10-memoh.conflist -COPY docker/server-dev-entrypoint.sh /entrypoint.sh +# Raw MCP rootfs for mcp-build.sh to package with compiled binary +COPY --from=mcp-rootfs /tmp/rootfs.tar /opt/images/memoh-mcp-rootfs.tar + +COPY devenv/server-entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh VOLUME ["/var/lib/containerd", "/opt/memoh/data"] diff --git a/docker/Dockerfile.web.dev b/devenv/Dockerfile.web similarity index 100% rename from docker/Dockerfile.web.dev rename to devenv/Dockerfile.web diff --git a/conf/app.dev.toml b/devenv/app.dev.toml similarity index 100% rename from conf/app.dev.toml rename to devenv/app.dev.toml diff --git a/devenv/docker-compose.yml b/devenv/docker-compose.yml index 2451406a..483a8910 100644 --- a/devenv/docker-compose.yml +++ b/devenv/docker-compose.yml @@ -37,7 +37,7 @@ services: deps: build: context: .. - dockerfile: docker/Dockerfile.web.dev + dockerfile: devenv/Dockerfile.web container_name: memoh-dev-deps working_dir: /workspace command: ["pnpm", "install"] @@ -50,13 +50,13 @@ services: migrate: build: context: .. - dockerfile: docker/Dockerfile.server.dev + dockerfile: devenv/Dockerfile.server container_name: memoh-dev-migrate working_dir: /workspace entrypoint: [] command: ["go", "run", "./cmd/agent/main.go", "migrate", "up"] environment: - CONFIG_PATH: /workspace/conf/app.dev.toml + CONFIG_PATH: /workspace/devenv/app.dev.toml volumes: - ..:/workspace - go_mod_cache:/go/pkg/mod @@ -71,14 +71,14 @@ services: server: build: context: .. - dockerfile: docker/Dockerfile.server.dev + dockerfile: devenv/Dockerfile.server container_name: memoh-dev-server working_dir: /workspace privileged: true pid: host command: ["air", "-c", ".air.toml"] environment: - CONFIG_PATH: /workspace/conf/app.dev.toml + CONFIG_PATH: /workspace/devenv/app.dev.toml volumes: - ..:/workspace - go_mod_cache:/go/pkg/mod @@ -121,7 +121,7 @@ services: web: build: context: .. - dockerfile: docker/Dockerfile.web.dev + dockerfile: devenv/Dockerfile.web container_name: memoh-dev-web working_dir: /workspace/packages/web command: ["pnpm", "dev"] diff --git a/devenv/mcp-build.sh b/devenv/mcp-build.sh new file mode 100755 index 00000000..2e4bbd37 --- /dev/null +++ b/devenv/mcp-build.sh @@ -0,0 +1,67 @@ +#!/bin/sh +# Build MCP binary, package as containerd image, and import. +# Called by air after server build — safe to skip outside dev container. +set -e + +MCP_IMAGE="${MCP_IMAGE:-docker.io/memohai/mcp:latest}" +MCP_BINARY="/opt/memoh/data/.dev/mcp" +BASE_ROOTFS="/opt/images/memoh-mcp-rootfs.tar" + +[ -f "$BASE_ROOTFS" ] || exit 0 +command -v ctr >/dev/null 2>&1 || exit 0 + +mkdir -p "$(dirname "$MCP_BINARY")" + +OLD_HASH=$(sha256sum "$MCP_BINARY" 2>/dev/null | cut -d' ' -f1) +go build -o "$MCP_BINARY" ./cmd/mcp || exit 0 +NEW_HASH=$(sha256sum "$MCP_BINARY" | cut -d' ' -f1) + +[ "$OLD_HASH" = "$NEW_HASH" ] && exit 0 + +echo "[mcp-dev] Binary changed, rebuilding MCP image..." + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +# Layer 1: base rootfs (symlink to avoid copying the large file) +LAYER1_SHA=$(sha256sum "$BASE_ROOTFS" | cut -d' ' -f1) +mkdir -p "$WORK/$LAYER1_SHA" +ln -s "$BASE_ROOTFS" "$WORK/$LAYER1_SHA/layer.tar" + +# Layer 2: compiled binary overlay +mkdir -p "$WORK/overlay/opt" +cp "$MCP_BINARY" "$WORK/overlay/opt/mcp" +chmod +x "$WORK/overlay/opt/mcp" +tar -cf "$WORK/layer2.tar" -C "$WORK/overlay" opt +LAYER2_SHA=$(sha256sum "$WORK/layer2.tar" | cut -d' ' -f1) +mkdir -p "$WORK/$LAYER2_SHA" +mv "$WORK/layer2.tar" "$WORK/$LAYER2_SHA/layer.tar" + +# OCI image config +ARCH=$(uname -m) +case "$ARCH" in aarch64|arm64) ARCH="arm64" ;; x86_64|amd64) ARCH="amd64" ;; esac + +printf '{"architecture":"%s","os":"linux","created":"1970-01-01T00:00:00Z","config":{"Entrypoint":["/opt/entrypoint.sh"],"WorkingDir":"/app","Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"]},"rootfs":{"type":"layers","diff_ids":["sha256:%s","sha256:%s"]},"history":[{"created":"1970-01-01T00:00:00Z","comment":"memoh-mcp rootfs"},{"created":"1970-01-01T00:00:00Z","comment":"memoh-mcp binary"}]}' \ + "$ARCH" "$LAYER1_SHA" "$LAYER2_SHA" > "$WORK/config.json" + +CONFIG_SHA=$(sha256sum "$WORK/config.json" | cut -d' ' -f1) +mv "$WORK/config.json" "$WORK/$CONFIG_SHA.json" + +printf '[{"Config":"%s.json","RepoTags":["%s"],"Layers":["%s/layer.tar","%s/layer.tar"]}]' \ + "$CONFIG_SHA" "$MCP_IMAGE" "$LAYER1_SHA" "$LAYER2_SHA" > "$WORK/manifest.json" + +# -h follows symlinks (layer 1 is symlinked to avoid copying) +tar -chf "$WORK/memoh-mcp.tar" -C "$WORK" manifest.json "$CONFIG_SHA.json" "$LAYER1_SHA/" "$LAYER2_SHA/" + +# Replace image in containerd +ctr -n default images rm "$MCP_IMAGE" 2>/dev/null || true +ctr -n default images import --all-platforms "$WORK/memoh-mcp.tar" 2>&1 || true + +# Clean old MCP containers so they recreate with new image +for c in $(ctr -n default containers ls -q 2>/dev/null | grep "^mcp-"); do + ctr -n default tasks kill "$c" 2>/dev/null || true + ctr -n default tasks delete "$c" 2>/dev/null || true + ctr -n default containers delete "$c" 2>/dev/null || true +done + +echo "[mcp-dev] Done. Containers will auto-recreate with new image." diff --git a/docker/server-dev-entrypoint.sh b/devenv/server-entrypoint.sh similarity index 81% rename from docker/server-dev-entrypoint.sh rename to devenv/server-entrypoint.sh index 14e7fa35..b072699c 100644 --- a/docker/server-dev-entrypoint.sh +++ b/devenv/server-entrypoint.sh @@ -28,8 +28,14 @@ if ! ctr version >/dev/null 2>&1; then echo "ERROR: containerd not responsive after 30s" exit 1 fi +echo "containerd is running (pid $CONTAINERD_PID)" -echo "containerd is ready, starting server command..." +# Build MCP binary and import as containerd image +echo "Building MCP image..." +(cd /workspace && sh devenv/mcp-build.sh) +echo "MCP image ready." + +echo "Starting server..." trap 'kill ${SERVER_PID:-0} 2>/dev/null || true; kill ${CONTAINERD_PID:-0} 2>/dev/null || true; wait' TERM INT diff --git a/internal/mcp/manager.go b/internal/mcp/manager.go index fb225480..dc6cae4a 100644 --- a/internal/mcp/manager.go +++ b/internal/mcp/manager.go @@ -91,6 +91,10 @@ func (m *Manager) lockContainer(containerID string) func() { func (m *Manager) Init(ctx context.Context) error { image := m.imageRef() + if _, err := m.service.GetImage(ctx, image); err == nil { + return nil + } + _, err := m.service.PullImage(ctx, image, &ctr.PullImageOptions{ Unpack: true, Snapshotter: m.cfg.Snapshotter, diff --git a/mise.toml b/mise.toml index a1d19ff6..67fc159a 100644 --- a/mise.toml +++ b/mise.toml @@ -53,7 +53,7 @@ description = "Start development environment" run = """ #!/bin/bash set -e -cp conf/app.dev.toml config.toml +cp devenv/app.dev.toml config.toml docker compose -f devenv/docker-compose.yml up --build """ @@ -69,6 +69,15 @@ run = "docker compose -f devenv/docker-compose.yml logs -f" description = "Restart a service (usage: mise run dev:restart -- server)" run = "docker compose -f devenv/docker-compose.yml restart $@" +[tasks."mcp:build"] +description = "Manually build MCP dev binary (normally auto-triggered by air)" +run = """ +#!/bin/bash +set -e +docker compose -f devenv/docker-compose.yml exec server \ + sh -c 'cd /workspace && sh devenv/mcp-build.sh' +""" + [tasks.db-up] description = "Initialize and Migrate Database" run = "scripts/db-up.sh" @@ -116,6 +125,6 @@ depends = [ run = """ #!/bin/bash set -e -cp conf/app.dev.toml config.toml +cp devenv/app.dev.toml config.toml echo '✓ Setup complete! Run: mise run dev' """