From 30fe3edc399cfd5750fd666a392a43c2e4ccf6c7 Mon Sep 17 00:00:00 2001 From: Ran <16112591+chen-ran@users.noreply.github.com> Date: Tue, 24 Feb 2026 22:29:15 +0800 Subject: [PATCH] fix(release): switch to agent-bin and gzip-only embedded web --- .github/workflows/release.yml | 8 +- internal/bun/runtime/manager.go | 36 +++-- internal/embedded/assets.go | 24 +-- internal/handlers/file_embed.go | 41 +++-- scripts/release.sh | 274 ++++++++++++++++++++------------ 5 files changed, 227 insertions(+), 156 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e735ae3f..242e2bcc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,8 +39,6 @@ jobs: goarch: amd64 - goos: linux goarch: arm64 - - goos: linux - goarch: riscv64 - goos: darwin goarch: amd64 - goos: darwin @@ -66,6 +64,9 @@ jobs: with: go-version-file: go.mod + - name: Install UPX + run: sudo apt-get update && sudo apt-get install -y upx-ucl + - name: Install JS dependencies run: pnpm install --frozen-lockfile @@ -76,6 +77,9 @@ jobs: VERSION: ${{ github.ref_name }} COMMIT_HASH: ${{ github.sha }} OUTPUT_DIR: dist + UPX_COMPRESS_AGENT_BIN: "true" + AUTO_INSTALL_UPX: "true" + UPX_ARGS: "-3" run: scripts/release.sh - name: Upload release assets diff --git a/internal/bun/runtime/manager.go b/internal/bun/runtime/manager.go index b6d1814e..3da8d517 100644 --- a/internal/bun/runtime/manager.go +++ b/internal/bun/runtime/manager.go @@ -36,6 +36,8 @@ const ( defaultGatewayHost = "127.0.0.1" defaultGatewayPort = 8081 agentConfigFileName = "config.toml" + agentBinName = "agent-bin" + agentUnavailableMarker = "UNAVAILABLE" healthCheckTimeout = 30 * time.Second healthCheckRetryBackoff = 400 * time.Millisecond processStopTimeout = 5 * time.Second @@ -69,37 +71,32 @@ func (m *Manager) Start(ctx context.Context) error { if err != nil { return err } - bunFS, bunBinName, err := embedded.BunFS("", "") - if err != nil { - return err - } agentDir := filepath.Join(workdir, "agent") - bunDir := filepath.Join(workdir, "bun") if err := extractFS(agentFS, agentDir); err != nil { return fmt.Errorf("extract agent assets: %w", err) } - if err := extractFS(bunFS, bunDir); err != nil { - return fmt.Errorf("extract bun runtime: %w", err) - } - bunPath := filepath.Join(bunDir, bunBinName) - if _, err := os.Stat(bunPath); err != nil { + agentBinPath := filepath.Join(agentDir, agentBinaryNameForRuntime()) + if _, err := os.Stat(agentBinPath); err != nil { if errors.Is(err, os.ErrNotExist) { - m.log.Warn("bundled bun runtime unavailable for current platform; falling back to configured agent gateway", slog.String("platform", runtimePlatform())) - return nil + markerPath := filepath.Join(agentDir, agentUnavailableMarker) + if _, markerErr := os.Stat(markerPath); markerErr == nil { + m.log.Warn("bundled agent binary unavailable for current platform; falling back to configured agent gateway", slog.String("platform", runtimePlatform())) + return nil + } } - return err + return fmt.Errorf("agent binary missing: %w", err) } - if err := os.Chmod(bunPath, 0o755); err != nil { - return fmt.Errorf("chmod bun binary: %w", err) + if err := os.Chmod(agentBinPath, 0o755); err != nil { + return fmt.Errorf("chmod agent binary: %w", err) } agentConfigPath := filepath.Join(agentDir, agentConfigFileName) if err := writeAgentConfig(agentConfigPath, m.cfg); err != nil { return err } - cmd := exec.Command(bunPath, "run", "dist/index.js") + cmd := exec.Command(agentBinPath) cmd.Dir = agentDir cmd.Env = append( os.Environ(), @@ -253,3 +250,10 @@ func trimTrailingNewline(s string) string { func runtimePlatform() string { return fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH) } + +func agentBinaryNameForRuntime() string { + if runtime.GOOS == "windows" { + return agentBinName + ".exe" + } + return agentBinName +} diff --git a/internal/embedded/assets.go b/internal/embedded/assets.go index f02f5885..60225f97 100644 --- a/internal/embedded/assets.go +++ b/internal/embedded/assets.go @@ -2,13 +2,10 @@ package embedded import ( "embed" - "fmt" "io/fs" - "path/filepath" - "runtime" ) -//go:embed all:web all:agent all:bun +//go:embed all:web all:agent var assetsFS embed.FS func AssetsFS() fs.FS { @@ -22,22 +19,3 @@ func WebFS() (fs.FS, error) { func AgentFS() (fs.FS, error) { return fs.Sub(assetsFS, "agent") } - -func BunFS(goos, goarch string) (fs.FS, string, error) { - if goos == "" { - goos = runtime.GOOS - } - if goarch == "" { - goarch = runtime.GOARCH - } - sub := filepath.ToSlash(filepath.Join("bun", goos+"-"+goarch)) - dirFS, err := fs.Sub(assetsFS, sub) - if err != nil { - return nil, "", fmt.Errorf("bun runtime not bundled for %s/%s: %w", goos, goarch, err) - } - bin := "bun" - if goos == "windows" { - bin = "bun.exe" - } - return dirFS, bin, nil -} diff --git a/internal/handlers/file_embed.go b/internal/handlers/file_embed.go index 852d6212..26349236 100644 --- a/internal/handlers/file_embed.go +++ b/internal/handlers/file_embed.go @@ -52,21 +52,12 @@ func (h *EmbeddedWebHandler) Register(e *echo.Echo) { } func (h *EmbeddedWebHandler) serveIndex(c echo.Context) error { - content, err := fs.ReadFile(h.webFS, "index.html") - if err != nil { - h.log.Error("read embedded index.html failed", slog.Any("error", err)) - return echo.ErrNotFound - } - return c.Blob(http.StatusOK, "text/html; charset=utf-8", content) + return h.serveKnownGzip(c, "index.html", "text/html; charset=utf-8") } func (h *EmbeddedWebHandler) serveStatic(targetPath, contentType string) echo.HandlerFunc { return func(c echo.Context) error { - content, err := fs.ReadFile(h.webFS, targetPath) - if err != nil { - return echo.ErrNotFound - } - return c.Blob(http.StatusOK, contentType, content) + return h.serveKnownGzip(c, targetPath, contentType) } } @@ -77,15 +68,33 @@ func (h *EmbeddedWebHandler) serveAsset(c echo.Context) error { } fullPath := path.Join("assets", assetPath) - content, err := fs.ReadFile(h.webFS, fullPath) - if err != nil { - return echo.ErrNotFound - } - contentType := mime.TypeByExtension(filepath.Ext(assetPath)) if contentType == "" { contentType = "application/octet-stream" } + gzipPath := fullPath + ".gz" + content, err := fs.ReadFile(h.webFS, gzipPath) + if err != nil { + return echo.ErrNotFound + } + header := c.Response().Header() + header.Set(echo.HeaderContentEncoding, "gzip") + header.Set(echo.HeaderVary, "Accept-Encoding") + return c.Blob(http.StatusOK, contentType, content) +} + +func (h *EmbeddedWebHandler) serveKnownGzip(c echo.Context, targetPath, contentType string) error { + gzipPath := targetPath + ".gz" + content, err := fs.ReadFile(h.webFS, gzipPath) + if err != nil { + if targetPath == "index.html" { + h.log.Error("read embedded index.html.gz failed", slog.Any("error", err)) + } + return echo.ErrNotFound + } + header := c.Response().Header() + header.Set(echo.HeaderContentEncoding, "gzip") + header.Set(echo.HeaderVary, "Accept-Encoding") return c.Blob(http.StatusOK, contentType, content) } diff --git a/scripts/release.sh b/scripts/release.sh index 07095a4c..d1b033fd 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -4,126 +4,201 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" TARGET_OS="${TARGET_OS:-$(go env GOOS)}" TARGET_ARCH="${TARGET_ARCH:-$(go env GOARCH)}" -BUN_VERSION="${BUN_VERSION:-latest}" VERSION="${VERSION:-dev}" COMMIT_HASH="${COMMIT_HASH:-unknown}" BUILD_TIME="${BUILD_TIME:-$(date -u +"%Y-%m-%dT%H:%M:%SZ")}" OUTPUT_DIR="${OUTPUT_DIR:-$ROOT_DIR/dist}" PREPARE_ASSETS_ONLY="false" +UPX_COMPRESS_AGENT_BIN="${UPX_COMPRESS_AGENT_BIN:-false}" +UPX_ARGS="${UPX_ARGS:--3}" +UPX_ALLOW_DARWIN="${UPX_ALLOW_DARWIN:-false}" +AUTO_INSTALL_UPX="${AUTO_INSTALL_UPX:-}" +if [[ -z "$AUTO_INSTALL_UPX" ]]; then + AUTO_INSTALL_UPX=$([[ "${GITHUB_ACTIONS:-}" == "true" ]] && echo "true" || echo "false") +fi -while [[ $# -gt 0 ]]; do - case "$1" in - --os) - TARGET_OS="$2" - shift 2 - ;; - --arch) - TARGET_ARCH="$2" - shift 2 - ;; - --bun-version) - BUN_VERSION="$2" - shift 2 - ;; - --version) - VERSION="$2" - shift 2 - ;; - --commit-hash) - COMMIT_HASH="$2" - shift 2 - ;; - --output-dir) - OUTPUT_DIR="$2" - shift 2 - ;; - --prepare-assets) - PREPARE_ASSETS_ONLY="true" - shift - ;; - *) - echo "Unknown arg: $1" >&2 - exit 1 - ;; +WEB_DIR="$ROOT_DIR/internal/embedded/web" +AGENT_DIR="$ROOT_DIR/internal/embedded/agent" +BUN_DIR="$ROOT_DIR/internal/embedded/bun" + +log() { + echo "[release] $*" +} + +usage() { + cat <<'EOF' +Usage: scripts/release.sh [options] + +Options: + --os Target OS (default: current GOOS) + --arch Target ARCH (default: current GOARCH) + --version Version string injected into memoh binary + --commit-hash Commit hash injected into memoh binary + --output-dir Output directory for release artifacts + --prepare-assets Only prepare embedded assets, do not build archive + +Compatibility options: + --bun-version Deprecated; ignored (kept for backward compatibility) +EOF +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --os) + TARGET_OS="$2" + shift 2 + ;; + --arch) + TARGET_ARCH="$2" + shift 2 + ;; + --bun-version) + # Bun runtime archives are no longer embedded; keep arg for compatibility. + shift 2 + ;; + --version) + VERSION="$2" + shift 2 + ;; + --commit-hash) + COMMIT_HASH="$2" + shift 2 + ;; + --output-dir) + OUTPUT_DIR="$2" + shift 2 + ;; + --prepare-assets) + PREPARE_ASSETS_ONLY="true" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown arg: $1" >&2 + usage >&2 + exit 1 + ;; + esac + done +} + +write_keep_gitignore() { + local dir="$1" + printf "*\n!.gitignore\n" > "$dir/.gitignore" +} + +resolve_agent_compile_target() { + case "${TARGET_OS}-${TARGET_ARCH}" in + linux-amd64) echo "bun-linux-x64|agent-bin" ;; + linux-arm64) echo "bun-linux-arm64|agent-bin" ;; + darwin-amd64) echo "bun-darwin-x64|agent-bin" ;; + darwin-arm64) echo "bun-darwin-arm64|agent-bin" ;; + windows-amd64) echo "bun-windows-x64|agent-bin.exe" ;; + *) echo "|" ;; esac -done +} + +prepare_embed_dirs() { + rm -rf "$WEB_DIR" "$AGENT_DIR" "$BUN_DIR" + mkdir -p "$WEB_DIR" "$AGENT_DIR" "$BUN_DIR" + write_keep_gitignore "$WEB_DIR" + write_keep_gitignore "$AGENT_DIR" + write_keep_gitignore "$BUN_DIR" +} prepare_assets() { - local web_dir="$ROOT_DIR/internal/embedded/web" - local agent_dir="$ROOT_DIR/internal/embedded/agent" - local bun_dir="$ROOT_DIR/internal/embedded/bun/${TARGET_OS}-${TARGET_ARCH}" + prepare_embed_dirs - rm -rf "$web_dir" "$agent_dir" "$bun_dir" - mkdir -p "$web_dir" "$agent_dir" "$bun_dir" - - echo "[release] building web assets" + log "building web assets" pnpm --dir "$ROOT_DIR" web:build - cp -R "$ROOT_DIR/packages/web/dist/." "$web_dir/" + cp -R "$ROOT_DIR/packages/web/dist/." "$WEB_DIR/" + gzip_embedded_web_assets "$WEB_DIR" - echo "[release] building agent bundle" - pnpm --dir "$ROOT_DIR" agent:build - mkdir -p "$agent_dir/dist" - cp "$ROOT_DIR/agent/dist/index.js" "$agent_dir/dist/index.js" - if [[ -f "$ROOT_DIR/agent/package.json" ]]; then - cp "$ROOT_DIR/agent/package.json" "$agent_dir/package.json" + local target_key="${TARGET_OS}-${TARGET_ARCH}" + local resolved bun_compile_target agent_bin_name + resolved="$(resolve_agent_compile_target)" + bun_compile_target="${resolved%%|*}" + agent_bin_name="${resolved##*|}" + if [[ -z "$bun_compile_target" || -z "$agent_bin_name" ]]; then + echo "agent-bin not available for ${target_key}" > "$AGENT_DIR/UNAVAILABLE" + log "skipped agent-bin compile for unsupported target ${target_key}" + return 0 fi - local bun_target="" - case "${TARGET_OS}-${TARGET_ARCH}" in - linux-amd64) bun_target="bun-linux-x64.zip" ;; - linux-arm64) bun_target="bun-linux-aarch64.zip" ;; - darwin-amd64) bun_target="bun-darwin-x64.zip" ;; - darwin-arm64) bun_target="bun-darwin-aarch64.zip" ;; - windows-amd64) bun_target="bun-windows-x64.zip" ;; - windows-arm64) bun_target="bun-windows-aarch64.zip" ;; - *) - echo "bun runtime not available for ${TARGET_OS}-${TARGET_ARCH}" > "$bun_dir/UNAVAILABLE" - echo "[release] skipped bun bundle for unsupported target ${TARGET_OS}-${TARGET_ARCH}" - return 0 - ;; - esac + log "building agent executable (${bun_compile_target})" + ( + cd "$ROOT_DIR/agent" + bun build src/index.ts --compile --target "$bun_compile_target" --outfile "$AGENT_DIR/$agent_bin_name" + ) + chmod +x "$AGENT_DIR/$agent_bin_name" || true + compress_agent_bin_if_enabled "$AGENT_DIR/$agent_bin_name" "$TARGET_OS" - local tmp_dir - tmp_dir="$(mktemp -d)" - trap 'rm -rf "$tmp_dir"' RETURN + log "embedded assets prepared (${target_key})" +} - local url - if [[ "$BUN_VERSION" == "latest" ]]; then - url="https://github.com/oven-sh/bun/releases/latest/download/${bun_target}" - else - url="https://github.com/oven-sh/bun/releases/download/bun-v${BUN_VERSION}/${bun_target}" +compress_agent_bin_if_enabled() { + local bin_path="$1" + local target_os="$2" + + if [[ "$UPX_COMPRESS_AGENT_BIN" != "true" ]]; then + return 0 + fi + ensure_upx_available + if [[ "$target_os" == "darwin" && "$UPX_ALLOW_DARWIN" != "true" ]]; then + log "skip upx on darwin (set UPX_ALLOW_DARWIN=true to force)" + return 0 fi - echo "[release] downloading ${url}" - curl -fsSL "$url" -o "$tmp_dir/bun.zip" - unzip -q -o "$tmp_dir/bun.zip" -d "$tmp_dir" + local before_bytes after_bytes + before_bytes="$(wc -c < "$bin_path" | tr -d ' ')" + read -r -a upx_flags <<< "$UPX_ARGS" + upx "${upx_flags[@]}" "$bin_path" + after_bytes="$(wc -c < "$bin_path" | tr -d ' ')" + log "upx compressed agent-bin: ${before_bytes} -> ${after_bytes} bytes" +} - local bun_bin_name="bun" - if [[ "$TARGET_OS" == "windows" ]]; then - bun_bin_name="bun.exe" +ensure_upx_available() { + if command -v upx >/dev/null 2>&1; then + return 0 fi - - local bun_source_path="" - if [[ -f "$tmp_dir/${bun_target%.zip}/${bun_bin_name}" ]]; then - bun_source_path="$tmp_dir/${bun_target%.zip}/${bun_bin_name}" - else - for candidate in "$tmp_dir"/bun-"${TARGET_OS}"-*/"${bun_bin_name}"; do - if [[ -f "$candidate" ]]; then - bun_source_path="$candidate" - break - fi - done - fi - - if [[ -z "$bun_source_path" ]]; then - echo "failed to locate bun binary in downloaded archive" >&2 + if [[ "$AUTO_INSTALL_UPX" != "true" ]]; then + echo "[release] UPX_COMPRESS_AGENT_BIN=true but upx not found in PATH" >&2 + echo "[release] install upx or set AUTO_INSTALL_UPX=true" >&2 exit 1 fi - cp "$bun_source_path" "$bun_dir/$bun_bin_name" - chmod +x "$bun_dir/$bun_bin_name" || true + log "upx not found; attempting auto-install" + if [[ "$OSTYPE" == linux* ]] && command -v apt-get >/dev/null 2>&1; then + if command -v sudo >/dev/null 2>&1; then + sudo apt-get update -y && sudo apt-get install -y upx-ucl + else + apt-get update -y && apt-get install -y upx-ucl + fi + elif [[ "$OSTYPE" == darwin* ]] && command -v brew >/dev/null 2>&1; then + brew install upx + fi - echo "[release] embedded assets prepared (${TARGET_OS}-${TARGET_ARCH})" + if ! command -v upx >/dev/null 2>&1; then + echo "[release] failed to auto-install upx" >&2 + exit 1 + fi +} + +gzip_embedded_web_assets() { + local web_dir="$1" + log "precompressing web assets (.gz)" + + while IFS= read -r -d '' file_path; do + if [[ "$(basename "$file_path")" == ".gitignore" ]]; then + continue + fi + gzip -9 -c "$file_path" > "${file_path}.gz" + rm -f "$file_path" + done < <(find "$web_dir" -type f -print0) } build_archive() { @@ -138,7 +213,7 @@ build_archive() { local target_dir="$OUTPUT_DIR/memoh_${VERSION}_${TARGET_OS}_${TARGET_ARCH}" mkdir -p "$target_dir" - echo "[release] building binary ${TARGET_OS}/${TARGET_ARCH}" + log "building binary ${TARGET_OS}/${TARGET_ARCH}" CGO_ENABLED=0 GOOS="$TARGET_OS" GOARCH="$TARGET_ARCH" \ go build \ -trimpath \ @@ -152,12 +227,13 @@ build_archive() { tar -C "$OUTPUT_DIR" -czf "$OUTPUT_DIR/memoh_${VERSION}_${TARGET_OS}_${TARGET_ARCH}.tar.gz" "memoh_${VERSION}_${TARGET_OS}_${TARGET_ARCH}" fi - echo "[release] archive created (${TARGET_OS}-${TARGET_ARCH})" + log "archive created (${TARGET_OS}-${TARGET_ARCH})" } +parse_args "$@" prepare_assets if [[ "$PREPARE_ASSETS_ONLY" == "true" ]]; then - echo "[release] prepare-assets only mode completed" + log "prepare-assets only mode completed" exit 0 fi