diff --git a/catalog/models/glm-4.7-flash.yaml b/catalog/models/glm-4.7-flash.yaml index 47038562..d13e576a 100644 --- a/catalog/models/glm-4.7-flash.yaml +++ b/catalog/models/glm-4.7-flash.yaml @@ -36,13 +36,16 @@ variants: default_config: quantization: int4 n_gpu_layers: 999 - ctx_size: 8192 + # Full trained context (202752). Deploy auto-clamps it down to fit hardware + # memory on smaller machines (weights ~18GB + KV cache), so this large default + # is safe everywhere. + ctx_size: 202752 expected_performance: startup_time_s: 20 cold_start_time_s: 40 tokens_per_second: [48, 60] latency_first_token_ms: [30, 200] - notes: "Verified 2026-06-08 on AMD Strix Halo Radeon 8060S iGPU, llama.cpp b9180 HIP, Q4_K_M (deepseek2 arch, 30B-A3B MoE) all 999 layers offloaded: 58.9 tok/s decode, 83 tok/s prompt." + notes: "Verified 2026-06-08 on AMD Strix Halo Radeon 8060S iGPU, llama.cpp b9180 HIP, Q4_K_M (deepseek2 arch, 30B-A3B MoE) all 999 layers offloaded: 58.9 tok/s decode, 83 tok/s prompt. Context verified 2026-06-16: llama-server loads & serves n_ctx=202752 (full trained context) on the iGPU." # --- GB10 (Blackwell, 128GB unified, K3S) --- # BF16 ~59GB, 128GB unified memory 可容纳 # Glm4MoeLiteForCausalLM 架构 (glm4_moe_lite model_type) diff --git a/catalog/models/qwen2.5-vl-3b-instruct.yaml b/catalog/models/qwen2.5-vl-3b-instruct.yaml index 62a605c4..0be16677 100644 --- a/catalog/models/qwen2.5-vl-3b-instruct.yaml +++ b/catalog/models/qwen2.5-vl-3b-instruct.yaml @@ -38,13 +38,22 @@ variants: hardware: gpu_arch: "*" vram_min_mib: 0 - ram_min_mib: 4096 + ram_min_mib: 8192 engine: llamacpp format: gguf default_config: quantization: int4 n_gpu_layers: 999 - ctx_size: 8192 + # Context window = the model's full trained context, 128000 (128K) — verified + # on Strix Halo: llama-server loads & serves n_ctx=128000 (KV cache ~36 KB/token + # → ~5 GB at 128K; model Q4 ~2 GB; so ~8 GB total, hence ram_min 8192). 8192 was + # too small for agent clients (OpenClaw injects all MCP tool schemas → base prompt + # ~9.3K tokens > 8192 → "context overflow", model never replies). A large n_ctx + # only reserves KV memory up front; it does NOT slow short conversations (llama + # processes only the actual tokens). Do NOT exceed 128000 (the trained limit) or + # llama-server warns of training-context overflow. Override per-deploy with + # `--config ctx_size=N` on memory-constrained hardware. + ctx_size: 128000 expected_performance: startup_time_s: 10 cold_start_time_s: 20 diff --git a/catalog/models/qwen3-embedding-4b.yaml b/catalog/models/qwen3-embedding-4b.yaml new file mode 100644 index 00000000..b182e454 --- /dev/null +++ b/catalog/models/qwen3-embedding-4b.yaml @@ -0,0 +1,51 @@ +kind: model_asset +metadata: + name: qwen3-embedding-4b + type: embedding + family: qwen + parameter_count: "4B" + released_at: "2025-06" + # Scan-name aliases: on-disk GGUF names so the local scanner matches this asset. + aliases: + - Qwen3-Embedding-4B + - Qwen3-Embedding-4B-Q4_K_M + - Qwen3-Embedding-4B-Q8_0 + - qwen3-embedding-4b-q4_k_m + - qwen3-embedding-4b-q8_0 +storage: + formats: [safetensors, gguf] + default_path_pattern: "{{.DataDir}}/models/{{.Name}}" + sources: + - type: huggingface + repo: Qwen/Qwen3-Embedding-4B + format: safetensors + - type: huggingface + repo: Qwen/Qwen3-Embedding-4B-GGUF + format: gguf + - type: modelscope + repo: Qwen/Qwen3-Embedding-4B + format: safetensors + - type: local_path + path: "" +variants: + # --- universal llamacpp GGUF (verified on AMD Strix Halo Radeon 8060S iGPU) --- + # Text-embedding model: deploy llama-server in --embedding mode (config embedding=true + # → --embedding) and serve OpenAI /v1/embeddings. 2560-dim output. Not a chat model, + # so it is NOT written into OpenClaw config by `aima openclaw sync` (embedding models + # have no chat provider mapping); use the /v1/embeddings endpoint directly. + - name: qwen3-embedding-4b-universal-llamacpp + hardware: + gpu_arch: "*" + vram_min_mib: 0 + ram_min_mib: 6144 + engine: llamacpp + format: gguf + default_config: + n_gpu_layers: 999 + embedding: true + # Native context (40960). Deploy auto-clamps to fit hardware memory if needed. + ctx_size: 40960 + expected_performance: + startup_time_s: 8 + cold_start_time_s: 15 + notes: "Verified 2026-06-16 on AMD Strix Halo Radeon 8060S iGPU, llama.cpp b9180 HIP, Q8_0 all layers offloaded: deploys in --embedding mode (config embedding=true), /v1/embeddings returns 2560-dim vectors. ctx_size 40960." diff --git a/catalog/models/qwen3.6-35b-a3b.yaml b/catalog/models/qwen3.6-35b-a3b.yaml index 9ec28cb6..a245a231 100644 --- a/catalog/models/qwen3.6-35b-a3b.yaml +++ b/catalog/models/qwen3.6-35b-a3b.yaml @@ -7,16 +7,42 @@ metadata: released_at: "2026-04" aliases: - Qwen3.6-35B-A3B + - Qwen3.6-35B-A3B-UD-Q4_K_M + - qwen3.6-35b-a3b-q4_k_m storage: - formats: [safetensors] + formats: [safetensors, gguf] default_path_pattern: "{{.DataDir}}/models/{{.Name}}" sources: - type: huggingface repo: Qwen/Qwen3.6-35B-A3B format: safetensors + - type: huggingface + repo: unsloth/Qwen3.6-35B-A3B-GGUF + format: gguf - type: local_path path: "" variants: + # --- universal llamacpp GGUF (verified on AMD Strix Halo Radeon 8060S iGPU) --- + # Qwen3.6-35B-A3B MoE, UD-Q4_K_M ~21GB, all layers offloaded. Native context 262144. + - name: qwen3.6-35b-a3b-universal-llamacpp-q4 + hardware: + gpu_arch: "*" + vram_min_mib: 0 + ram_min_mib: 24576 + engine: llamacpp + format: gguf + default_config: + quantization: int4 + n_gpu_layers: 999 + # Full trained context (262144 / 256K). Deploy auto-clamps it down to fit + # hardware memory on smaller machines, so this large default is safe. + ctx_size: 262144 + expected_performance: + startup_time_s: 30 + cold_start_time_s: 60 + tokens_per_second: [55, 70] + latency_first_token_ms: [30, 250] + notes: "Context verified 2026-06-16 on AMD Strix Halo Radeon 8060S iGPU, llama.cpp b9180 HIP, UD-Q4_K_M all 999 layers offloaded: llama-server loads & serves n_ctx=262144 (full trained context)." # --- GB10 (Blackwell, 128GB unified, Docker/K3S) --- # 实测稳定路径是 text-only vLLM: # - language_model_only=true diff --git a/cmd/aima/adapters.go b/cmd/aima/adapters.go index 6f1ef8bc..53f0874b 100644 --- a/cmd/aima/adapters.go +++ b/cmd/aima/adapters.go @@ -6,7 +6,9 @@ import ( "encoding/json" "fmt" "log/slog" + "os" "os/exec" + "path/filepath" "strings" "sync" "time" @@ -484,19 +486,35 @@ func (a *fleetMCPAdapter) ListToolDefs() json.RawMessage { // toEngineBinarySource converts a knowledge.EngineSource to engine.BinarySource. // Centralises the mapping so callers don't repeat the 4-field struct literal. func toEngineBinarySource(src *knowledge.EngineSource) *engine.BinarySource { + if src == nil { + return nil + } var probePaths []string - if src != nil && src.Probe != nil { + if src.Probe != nil { probePaths = append(probePaths, src.Probe.Paths...) } return &engine.BinarySource{ - Binary: src.Binary, - Platforms: src.Platforms, - Download: src.Download, - Mirror: src.Mirror, - SHA256: src.SHA256, - InstallType: src.InstallType, - ProbePaths: probePaths, + Binary: src.Binary, + Platforms: src.Platforms, + Download: src.Download, + Mirror: src.Mirror, + SHA256: src.SHA256, + InstallType: src.InstallType, + ProbePaths: probePaths, + LocalBundles: engineLocalBundlesFromEnv(), + } +} + +func engineLocalBundlesFromEnv() []string { + var bundles []string + for _, name := range []string{"AIMA_ENGINE_BUNDLE", "AIMA_ENGINE_ARCHIVE", "AIMA_ENGINE_OFFLINE_PACKAGE"} { + for _, path := range filepath.SplitList(os.Getenv(name)) { + if path = strings.TrimSpace(path); path != "" { + bundles = append(bundles, path) + } + } } + return bundles } // execRunner implements engine.CommandRunner using real exec. @@ -595,6 +613,7 @@ func (a openClawBackendAdapter) ListBackends() map[string]*openclaw.Backend { result[k] = &openclaw.Backend{ ModelName: b.ModelName, EngineType: b.EngineType, + ModelType: b.ModelType, Address: b.Address, Ready: b.Ready, Remote: b.Remote, @@ -628,7 +647,7 @@ type catalogAdapter struct{ cat *knowledge.Catalog } func (a catalogAdapter) ModelType(name string) string { for _, m := range a.cat.ModelAssets { - if strings.EqualFold(m.Metadata.Name, name) { + if catalogModelNameMatches(m, name) { return m.Metadata.Type } } @@ -641,7 +660,7 @@ func (a catalogAdapter) ModelContextWindow(name string) int { func (a catalogAdapter) ModelFamily(name string) string { for _, m := range a.cat.ModelAssets { - if strings.EqualFold(m.Metadata.Name, name) { + if catalogModelNameMatches(m, name) { return m.Metadata.Family } } @@ -650,7 +669,7 @@ func (a catalogAdapter) ModelFamily(name string) string { func (a catalogAdapter) ModelChatProvider(name string) bool { for _, m := range a.cat.ModelAssets { - if strings.EqualFold(m.Metadata.Name, name) { + if catalogModelNameMatches(m, name) { if m.OpenClaw != nil && m.OpenClaw.ChatProvider != nil { return *m.OpenClaw.ChatProvider } @@ -660,6 +679,18 @@ func (a catalogAdapter) ModelChatProvider(name string) bool { return true } +func catalogModelNameMatches(m knowledge.ModelAsset, name string) bool { + if strings.EqualFold(m.Metadata.Name, name) { + return true + } + for _, alias := range m.Metadata.Aliases { + if strings.EqualFold(alias, name) { + return true + } + } + return false +} + func (a catalogAdapter) Adapters(name string) []inferencehttp.Adapter { for _, m := range a.cat.ModelAssets { if !strings.EqualFold(m.Metadata.Name, name) || m.HTTP == nil { diff --git a/cmd/aima/compat.go b/cmd/aima/compat.go index ce5255ec..3fa856cc 100644 --- a/cmd/aima/compat.go +++ b/cmd/aima/compat.go @@ -103,7 +103,8 @@ func prepareContainerCompatibility( } probeSummary := summarizeCompatibilityProbeOutput(probeOutput, probeErr) - if allowAutoPull && len(resolved.EngineRegistries) > 0 && !strings.EqualFold(resolved.EngineDistribution, "local") { + engineRegistries := engineRegistriesWithEnv(resolved.EngineRegistries) + if allowAutoPull && len(engineRegistries) > 0 && !strings.EqualFold(resolved.EngineDistribution, "local") { if refreshErr := refreshDockerImageForCompatibilityProbe(ctx, runner, resolved); refreshErr == nil { plan.DockerImageChanged = true refreshedOutput, refreshedErr := timedContainerCompatibilityProbe(ctx, runner, resolved.CompatibilityProbe, resolved.EngineImage, modelPath, trustRemoteCode, nil) @@ -159,7 +160,7 @@ func ensureDockerImageForCompatibilityProbe(ctx context.Context, runner engine.C if engine.ImageExistsInDocker(ctx, resolved.EngineImage, runner) { return true, false, nil } - if !allowAutoPull || len(resolved.EngineRegistries) == 0 || strings.EqualFold(resolved.EngineDistribution, "local") { + if !allowAutoPull || len(engineRegistriesWithEnv(resolved.EngineRegistries)) == 0 || strings.EqualFold(resolved.EngineDistribution, "local") { return false, false, nil } if err := pullDockerImageForCompatibilityProbe(ctx, runner, resolved); err != nil { @@ -179,20 +180,21 @@ func pullDockerImageForCompatibilityProbe(ctx context.Context, runner engine.Com if resolved == nil { return fmt.Errorf("resolved config is nil") } - if len(resolved.EngineRegistries) == 0 { + engineRegistries := engineRegistriesWithEnv(resolved.EngineRegistries) + if len(engineRegistries) == 0 { return fmt.Errorf("no registries configured for %s", resolved.EngineImage) } imgName, imgTag := splitImageRef(resolved.EngineImage) if err := engine.Pull(ctx, engine.PullOptions{ Image: imgName, Tag: imgTag, - Registries: resolved.EngineRegistries, + Registries: engineRegistries, Runner: &dockerOnlyRunner{base: runner}, ExpectedDigest: resolved.EngineDigest, }); err != nil { return err } - return ensureDockerImageAlias(ctx, runner, resolved.EngineImage, resolved.EngineRegistries) + return ensureDockerImageAlias(ctx, runner, resolved.EngineImage, engineRegistries) } func ensureDockerImageAlias(ctx context.Context, runner engine.CommandRunner, image string, registries []string) error { diff --git a/cmd/aima/ctxfit_test.go b/cmd/aima/ctxfit_test.go new file mode 100644 index 00000000..63097ee8 --- /dev/null +++ b/cmd/aima/ctxfit_test.go @@ -0,0 +1,105 @@ +package main + +import ( + "testing" + + "github.com/jguan/aima/internal/knowledge" +) + +// Qwen2.5-VL-3B text backbone: 36 layers, 2 KV heads, head_dim 128 → 36864 B/token. +const qwen25vl3bKVPerTok = int64(2 * 36 * 2 * 128 * 2) + +func TestClampContextForMemory(t *testing.T) { + tests := []struct { + name string + reqCtx int + nCtxTrain int + kvPerTok int64 + usableMiB int + nonKVMiB int + want int + clamped bool // want < reqCtx + }{ + { + name: "huge memory, capped only by trained context", + reqCtx: 128000, nCtxTrain: 128000, kvPerTok: qwen25vl3bKVPerTok, + usableMiB: 112640, nonKVMiB: 4300, // ~110GB Strix Halo + want: 128000, clamped: false, + }, + { + name: "request above trained context is capped down", + reqCtx: 200000, nCtxTrain: 128000, kvPerTok: qwen25vl3bKVPerTok, + usableMiB: 112640, nonKVMiB: 4300, + want: 128000, clamped: true, + }, + { + name: "16GB unified clamps below 128k", + reqCtx: 128000, nCtxTrain: 128000, kvPerTok: qwen25vl3bKVPerTok, + usableMiB: 12288, nonKVMiB: 4300, // 16GB - 4GB reserve + // budget = 12288*0.9 - 4300 - 1024 = 5734 MiB → /36864 B = ~163k → but capped at train 128000? 163k>128000 so 128000 + want: 128000, clamped: false, + }, + { + name: "8GB discrete GPU clamps below 128k", + reqCtx: 128000, nCtxTrain: 128000, kvPerTok: qwen25vl3bKVPerTok, + usableMiB: 8192, nonKVMiB: 4300, + // budget = int(8192*0.9) - 4300 - 1024 = 7372 - 5324 = 2048 MiB + // maxCtx = 2048*1024*1024/36864 = 58254 → round down to 58112 + want: 58112, clamped: true, + }, + { + name: "tiny memory floors at minCtx", + reqCtx: 128000, nCtxTrain: 128000, kvPerTok: qwen25vl3bKVPerTok, + usableMiB: 6144, nonKVMiB: 4300, + // budget = 6144*0.9 - 4300 - 1024 = ~205 MiB → ~5832 → round 5632 + want: 5632, clamped: true, + }, + { + name: "unknown memory: only trained-context cap applies", + reqCtx: 65536, nCtxTrain: 128000, kvPerTok: qwen25vl3bKVPerTok, + usableMiB: 0, nonKVMiB: 0, + want: 65536, clamped: false, + }, + { + name: "unknown arch (kvPerTok 0): no memory clamp", + reqCtx: 65536, nCtxTrain: 0, kvPerTok: 0, + usableMiB: 8192, nonKVMiB: 4300, + want: 65536, clamped: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, _ := clampContextForMemory(tt.reqCtx, tt.nCtxTrain, tt.kvPerTok, tt.usableMiB, tt.nonKVMiB) + if got != tt.want { + t.Errorf("clampContextForMemory = %d, want %d", got, tt.want) + } + if (got < tt.reqCtx) != tt.clamped { + t.Errorf("clamped = %v, want %v (got=%d req=%d)", got < tt.reqCtx, tt.clamped, got, tt.reqCtx) + } + }) + } +} + +func TestUsableMemoryMiB(t *testing.T) { + tests := []struct { + name string + hw knowledge.HardwareInfo + want int + }{ + {"unified 128GB reserves 16GB cap", knowledge.HardwareInfo{UnifiedMemory: true, RAMTotalMiB: 131072}, 131072 - 16384}, + {"unified 16GB reserves 1/4", knowledge.HardwareInfo{UnifiedMemory: true, RAMTotalMiB: 16384}, 16384 - 4096}, + {"unified 8GB reserve floored at 2GB", knowledge.HardwareInfo{UnifiedMemory: true, RAMTotalMiB: 8192}, 8192 - 2048}, + {"unified APU prefers iGPU pool over under-detected OS RAM", knowledge.HardwareInfo{UnifiedMemory: true, RAMTotalMiB: 32768, GPUVRAMMiB: 110456}, 110456}, + {"discrete prefers free VRAM", knowledge.HardwareInfo{GPUVRAMMiB: 8192, GPUMemFreeMiB: 7000}, 7000}, + {"discrete falls back to total VRAM", knowledge.HardwareInfo{GPUVRAMMiB: 8192}, 8192}, + {"cpu-only uses system RAM", knowledge.HardwareInfo{RAMTotalMiB: 32768}, 32768 - 8192}, + {"unknown memory returns 0", knowledge.HardwareInfo{}, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := usableMemoryMiB(tt.hw); got != tt.want { + t.Errorf("usableMemoryMiB = %d, want %d", got, tt.want) + } + }) + } +} diff --git a/cmd/aima/infra.go b/cmd/aima/infra.go index 11d154a2..c891eea7 100644 --- a/cmd/aima/infra.go +++ b/cmd/aima/infra.go @@ -424,6 +424,46 @@ func splitImageRef(ref string) (name, tag string) { return ref[:absColon], ref[absColon+1:] } +func engineRegistriesWithEnv(registries []string) []string { + envRegistries := splitRegistryEnv(os.Getenv("AIMA_ENGINE_REGISTRIES")) + envRegistries = append(envRegistries, splitRegistryEnv(os.Getenv("AIMA_ENGINE_REGISTRY"))...) + if len(envRegistries) == 0 { + return registries + } + seen := make(map[string]struct{}, len(envRegistries)+len(registries)) + out := make([]string, 0, len(envRegistries)+len(registries)) + add := func(value string) { + value = strings.TrimSpace(value) + if value == "" { + return + } + if _, ok := seen[value]; ok { + return + } + seen[value] = struct{}{} + out = append(out, value) + } + for _, value := range envRegistries { + add(value) + } + for _, value := range registries { + add(value) + } + return out +} + +func splitRegistryEnv(raw string) []string { + var values []string + for _, part := range strings.FieldsFunc(raw, func(r rune) bool { + return r == ',' || r == '\n' + }) { + if trimmed := strings.TrimSpace(part); trimmed != "" { + values = append(values, trimmed) + } + } + return values +} + type deployOptions struct { allowAutoPull bool } diff --git a/cmd/aima/main.go b/cmd/aima/main.go index bbd1445c..f047dce0 100644 --- a/cmd/aima/main.go +++ b/cmd/aima/main.go @@ -319,12 +319,18 @@ func run() error { } inferenceHTTPRoutes := inferencehttp.RegisterRoutes(inferenceHTTPDeps) openclawDeps := &openclaw.Deps{ - Backends: openClawBackendAdapter{proxyServer}, - Catalog: catalogAdapter{cat}, - ConfigPath: openclaw.DefaultConfigPath(), + Backends: openClawBackendAdapter{proxyServer}, + Catalog: catalogAdapter{cat}, + // Config dir is overridable so a partner using a custom dir name (e.g. + // .byClaw) can target it: AIMA_OPENCLAW_CONFIG=/openclaw.json. Skills, + // extensions and managed-state all follow filepath.Dir(ConfigPath). + ConfigPath: firstNonEmpty(os.Getenv("AIMA_OPENCLAW_CONFIG"), openclaw.DefaultConfigPath()), ProxyAddr: fmt.Sprintf("http://127.0.0.1:%d/v1", proxy.DefaultPort), APIKey: proxyServer.APIKey, MCPCommand: mcpCommand, + // Whether sync sets the synced model as OpenClaw's primary. Partner-owned via + // AIMA_OPENCLAW_SET_DEFAULT (unset=set primary; false=leave user's primary). + SetDefaultModel: openclawSetDefaultFromEnv(), } proxyServer.SetRequestRewriter(inferencehttp.RequestBodyRewriter(inferenceHTTPDeps.Catalog)) refreshOpenClawBackends := func(ctx context.Context) { @@ -1356,6 +1362,17 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps { onPhase(phase, msg) } } + syncOpenClawAfterReady := func() { + if deps == nil || deps.OpenClawSync == nil { + return + } + if _, err := deps.OpenClawSync(ctx, false); err != nil { + slog.Warn("deploy: openclaw sync after ready failed", "model", model, "error", err) + notify("warning", "OpenClaw sync failed after deploy: "+err.Error()) + } else { + slog.Info("deploy: openclaw sync complete after ready", "model", model) + } + } waitForDeployment := func(deployName, runtimeName, resolvedEngine string, resolvedConfig map[string]any, warmup knowledge.WarmupConfig, deployTimeout time.Duration) (json.RawMessage, error) { notify("waiting", deployName) @@ -1408,6 +1425,7 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps { if status.Runtime != "" { runtimeName = status.Runtime } + syncOpenClawAfterReady() return json.Marshal(map[string]any{ "name": deployName, "model": model, "engine": resolvedEngine, "runtime": runtimeName, "address": status.Address, "status": "ready", @@ -1481,6 +1499,7 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps { if status.Runtime != "" { runtimeName = status.Runtime } + syncOpenClawAfterReady() return json.Marshal(map[string]any{ "name": deployName, "model": model, "engine": plan.Engine, "runtime": runtimeName, "address": status.Address, "status": "ready", diff --git a/cmd/aima/tooldeps_deploy.go b/cmd/aima/tooldeps_deploy.go index 3319fb67..b8f20dfa 100644 --- a/cmd/aima/tooldeps_deploy.go +++ b/cmd/aima/tooldeps_deploy.go @@ -15,6 +15,7 @@ import ( "github.com/jguan/aima/internal/engine" "github.com/jguan/aima/internal/knowledge" "github.com/jguan/aima/internal/mcp" + "github.com/jguan/aima/internal/model" "github.com/jguan/aima/internal/proxy" "github.com/jguan/aima/internal/runtime" @@ -43,6 +44,9 @@ func buildDeployDeps(ac *appContext, deps *mcp.ToolDeps, dataDir := ac.dataDir deps.DeployApply = func(ctx context.Context, engineType, modelName, slot string, configOverrides map[string]any, noPull bool) (json.RawMessage, error) { + // A1: keep the user's original input before it's canonicalized below, so the + // result can surface the original↔canonical mapping (requested_model). + requestedModel := modelName if noPull { ctx = withDeployAutoPull(ctx, false) } @@ -68,6 +72,7 @@ func buildDeployDeps(ac *appContext, deps *mcp.ToolDeps, modelName = rd.ModelName resolved := rd.Resolved upstreamModel := resolvedServedModelName(modelName, resolved.Config) + modelType := firstNonEmpty(resolved.ModelType, catalogModelType(cat, modelName)) modelPath, modelPathErr := resolveLocalModelPathNoPull(modelName, resolved, dataDir) if modelPathErr != nil { @@ -98,6 +103,15 @@ func buildDeployDeps(ac *appContext, deps *mcp.ToolDeps, slog.Info("auto-wired multimodal projector for vision", "model", modelName, "mmproj", mm) } } + + // Hardware-aware context sizing: a high catalog/user ctx_size can exceed + // what the detected memory holds — llama-server would OOM at load. Clamp + // ctx_size down to fit weights + projector + KV cache (and cap at the + // model's trained context), degrading gracefully instead of failing. + if clamped, oldCtx, newCtx, reason := fitContextToMemory(modelPath, resolved.Config, hwInfo); clamped { + slog.Warn("clamped context window to fit hardware memory", + "model", modelName, "ctx_size_requested", oldCtx, "ctx_size_applied", newCtx, "detail", reason) + } } req := &runtime.DeployRequest{ @@ -108,7 +122,7 @@ func buildDeployDeps(ac *appContext, deps *mcp.ToolDeps, PortSpecs: append([]knowledge.StartupPort(nil), resolved.PortSpecs...), InitCommands: resolved.InitCommands, ModelPath: modelPath, - ModelType: catalogModelType(cat, modelName), + ModelType: modelType, Config: resolved.Config, RuntimeClassName: resolved.RuntimeClassName, CPUArch: resolved.CPUArch, @@ -131,7 +145,7 @@ func buildDeployDeps(ac *appContext, deps *mcp.ToolDeps, if parameterCount := catalogModelParameterCount(cat, modelName); parameterCount != "" { req.Labels[proxy.LabelParameterCount] = parameterCount } - if modelType := catalogModelType(cat, modelName); modelType != "" { + if modelType != "" { req.Labels[proxy.LabelModelType] = modelType } if contextWindow := contextWindowFromResolvedConfig(resolved.Config); contextWindow > 0 { @@ -178,7 +192,7 @@ func buildDeployDeps(ac *appContext, deps *mcp.ToolDeps, ModelName: modelName, UpstreamModel: deploymentUpstreamModel(existing, upstreamModel), EngineType: resolved.Engine, - ModelType: catalogModelType(cat, modelName), + ModelType: modelType, Address: existing.Address, Ready: existing.Ready, ParameterCount: firstNonEmpty(existing.Labels[proxy.LabelParameterCount], catalogModelParameterCount(cat, modelName)), @@ -194,10 +208,11 @@ func buildDeployDeps(ac *appContext, deps *mcp.ToolDeps, } existingName := firstNonEmpty(existing.Name, deployName) result := map[string]any{ - "name": existingName, - "model": modelName, - "engine": resolved.Engine, - "slot": resolved.Slot, + "name": existingName, + "model": modelName, + "requested_model": requestedModel, + "engine": resolved.Engine, + "slot": resolved.Slot, "status": status, "phase": existing.Phase, "runtime": runtimeName, @@ -208,7 +223,7 @@ func buildDeployDeps(ac *appContext, deps *mcp.ToolDeps, if existing.Address != "" { result["address"] = existing.Address } - if err := setActiveLLMModelConfigForType(ctx, db, modelName, catalogModelType(cat, modelName)); err != nil { + if err := setActiveLLMModelConfigForType(ctx, db, modelName, modelType); err != nil { return nil, err } return json.Marshal(result) @@ -217,6 +232,7 @@ func buildDeployDeps(ac *appContext, deps *mcp.ToolDeps, // Auto-import from Docker or pre-pull from registries if needed. // Note: containerd operations require root; skip gracefully if not root. if activeRt.Name() == "k3s" && req.Image != "" { + engineRegistries := engineRegistriesWithEnv(resolved.EngineRegistries) inContainerd := engine.ImageExistsInContainerd(ctx, req.Image, &execRunner{}) if !inContainerd { inDocker := engine.ImageExistsInDocker(ctx, req.Image, &execRunner{}) @@ -232,16 +248,16 @@ func buildDeployDeps(ac *appContext, deps *mcp.ToolDeps, slog.Warn("auto-import failed, K3S will try registries.yaml", "image", req.Image, "error", importErr) } } - } else if activeRt.Name() == "k3s" && len(resolved.EngineRegistries) > 0 { + } else if activeRt.Name() == "k3s" && len(engineRegistries) > 0 { if !allowAutoPull { return nil, fmt.Errorf("engine image %s not found in K3S containerd and auto-pull is disabled", req.Image) } - slog.Info("pre-pulling engine image", "image", req.Image, "registries", len(resolved.EngineRegistries)) + slog.Info("pre-pulling engine image", "image", req.Image, "registries", len(engineRegistries)) imgName, imgTag := splitImageRef(req.Image) if pullErr := engine.Pull(ctx, engine.PullOptions{ Image: imgName, Tag: imgTag, - Registries: resolved.EngineRegistries, + Registries: engineRegistries, Runner: &execRunner{}, ExpectedDigest: resolved.EngineDigest, }); pullErr != nil { @@ -257,7 +273,8 @@ func buildDeployDeps(ac *appContext, deps *mcp.ToolDeps, fullRef += ":latest" } if !engine.ImageExistsInDocker(ctx, fullRef, &execRunner{}) { - if len(resolved.EngineRegistries) > 0 { + engineRegistries := engineRegistriesWithEnv(resolved.EngineRegistries) + if len(engineRegistries) > 0 { if !allowAutoPull { return nil, fmt.Errorf("engine image %s not found in Docker and auto-pull is disabled", req.Image) } @@ -266,13 +283,13 @@ func buildDeployDeps(ac *appContext, deps *mcp.ToolDeps, if pullErr := engine.Pull(ctx, engine.PullOptions{ Image: imgName, Tag: imgTag, - Registries: resolved.EngineRegistries, + Registries: engineRegistries, Runner: &execRunner{}, ExpectedDigest: resolved.EngineDigest, }); pullErr != nil { return nil, fmt.Errorf("auto-pull engine image %s: %w", req.Image, pullErr) } - if aliasErr := ensureDockerImageAlias(ctx, &execRunner{}, req.Image, resolved.EngineRegistries); aliasErr != nil { + if aliasErr := ensureDockerImageAlias(ctx, &execRunner{}, req.Image, engineRegistries); aliasErr != nil { return nil, fmt.Errorf("normalize pulled docker image %s: %w", req.Image, aliasErr) } } else { @@ -317,12 +334,12 @@ func buildDeployDeps(ac *appContext, deps *mcp.ToolDeps, ModelName: modelName, UpstreamModel: upstreamModel, EngineType: resolved.Engine, - ModelType: catalogModelType(cat, modelName), + ModelType: modelType, Ready: false, ParameterCount: catalogModelParameterCount(cat, modelName), ContextWindowTokens: contextWindowFromResolvedConfig(resolved.Config), }) - if err := setActiveLLMModelConfigForType(ctx, db, modelName, catalogModelType(cat, modelName)); err != nil { + if err := setActiveLLMModelConfigForType(ctx, db, modelName, modelType); err != nil { return nil, err } result := map[string]any{ @@ -333,7 +350,7 @@ func buildDeployDeps(ac *appContext, deps *mcp.ToolDeps, // sanitized deployName never matched → status was never found → the deploy // looked stuck "not ready" even though the engine was serving fine. "name": req.Name, - "model": modelName, "engine": resolved.Engine, + "model": modelName, "requested_model": requestedModel, "engine": resolved.Engine, "slot": resolved.Slot, "status": "deploying", "runtime": activeRt.Name(), "config": resolved.Config, @@ -373,6 +390,7 @@ func buildDeployDeps(ac *appContext, deps *mcp.ToolDeps, result := map[string]any{ "model": rd.ModelName, + "requested_model": modelName, "engine": resolved.Engine, "engine_image": resolved.EngineImage, "slot": resolved.Slot, @@ -503,6 +521,18 @@ func buildDeployDeps(ac *appContext, deps *mcp.ToolDeps, matches = findMatchingDeployments(ctx, modelStatus.Name, nil, rt, nativeRt, dockerRt) } } + if len(matches) == 0 { + // A1: deploy canonicalizes the model name (e.g. the alias + // "Qwen2.5-VL-3B-Instruct-Q4_K_M" deploys as "qwen2.5-vl-3b-instruct"), + // but undeploy with the original alias would not match. Canonicalize the + // query and retry so alias-deploy → alias-undeploy works. + if canonical := canonicalModelAlt(cat, name); canonical != "" { + matches = findExactDeploymentNameMatches(ctx, canonical, nil, rt, nativeRt, dockerRt) + if len(matches) == 0 { + matches = findMatchingDeployments(ctx, canonical, nil, rt, nativeRt, dockerRt) + } + } + } if len(matches) == 0 { return fmt.Errorf("deployment %q not found", name) } @@ -567,6 +597,14 @@ func buildDeployDeps(ac *appContext, deps *mcp.ToolDeps, deps.DeployStatus = func(ctx context.Context, name string) (json.RawMessage, error) { suppressRecentlyDeleted := loadDeletedDeploymentSuppressor(ctx, db) s, err := findDeploymentStatus(ctx, name, suppressRecentlyDeleted, rt, nativeRt, dockerRt) + if err != nil { + // A1: retry with the canonical name so `status ` works too. + if canonical := canonicalModelAlt(cat, name); canonical != "" { + if s2, err2 := findDeploymentStatus(ctx, canonical, suppressRecentlyDeleted, rt, nativeRt, dockerRt); err2 == nil { + s, err = s2, nil + } + } + } if err != nil { return nil, err } @@ -614,9 +652,11 @@ func buildDeployDeps(ac *appContext, deps *mcp.ToolDeps, } if err != nil { // Exact pod name failed -- search by model label across all runtimes. + // A1: also match the canonical name so `logs ` works. + canonical := canonicalModelAlt(cat, name) allDeps := listAllRuntimes(ctx, rt, nativeRt, dockerRt) for _, d := range allDeps { - if deploymentMatchesQuery(d, name) { + if deploymentMatchesQuery(d, name) || (canonical != "" && deploymentMatchesQuery(d, canonical)) { // Try each runtime for logs by actual deployment name. for _, tryRt := range []runtime.Runtime{rt, nativeRt, dockerRt} { if tryRt == nil { @@ -662,7 +702,7 @@ func catalogModelParameterCount(cat *knowledge.Catalog, name string) string { return "" } for _, model := range cat.ModelAssets { - if strings.EqualFold(model.Metadata.Name, name) { + if modelAssetNameMatches(model, name) { return strings.TrimSpace(model.Metadata.ParameterCount) } } @@ -674,7 +714,7 @@ func catalogModelType(cat *knowledge.Catalog, name string) string { return "" } for i := range cat.ModelAssets { - if strings.EqualFold(cat.ModelAssets[i].Metadata.Name, name) { + if modelAssetNameMatches(cat.ModelAssets[i], name) { return strings.TrimSpace(cat.ModelAssets[i].Metadata.Type) } } @@ -684,6 +724,18 @@ func catalogModelType(cat *knowledge.Catalog, name string) string { return "" } +func modelAssetNameMatches(model knowledge.ModelAsset, name string) bool { + if strings.EqualFold(model.Metadata.Name, name) { + return true + } + for _, alias := range model.Metadata.Aliases { + if strings.EqualFold(alias, name) { + return true + } + } + return false +} + func firstNonEmpty(values ...string) string { for _, value := range values { if trimmed := strings.TrimSpace(value); trimmed != "" { @@ -693,6 +745,37 @@ func firstNonEmpty(values ...string) string { return "" } +// canonicalModelAlt returns the canonical catalog model name for a query when it +// differs from the input (e.g. an alias carrying a quant suffix / different case), +// or "" otherwise. Deployments are stored under the canonical name, so name-taking +// commands (undeploy/status/logs) use this to also accept the original deploy-time +// alias the user typed. +func canonicalModelAlt(cat *knowledge.Catalog, name string) string { + if cat == nil { + return "" + } + c := strings.TrimSpace(cat.ResolveCatalogModelName(name)) + if c != "" && !strings.EqualFold(c, name) { + return c + } + return "" +} + +// openclawSetDefaultFromEnv reads AIMA_OPENCLAW_SET_DEFAULT as a tri-state: +// unset/unparseable → nil (default: AIMA sets the primary chat model); otherwise +// the parsed bool (false = leave the user's primary model untouched). +func openclawSetDefaultFromEnv() *bool { + v := strings.TrimSpace(os.Getenv("AIMA_OPENCLAW_SET_DEFAULT")) + if v == "" { + return nil + } + b, err := strconv.ParseBool(v) + if err != nil { + return nil + } + return &b +} + func populateDeploymentOverviewFields(status *runtime.DeploymentStatus) { if status == nil { return @@ -776,6 +859,116 @@ func deploymentOverviewFromStatus(status *runtime.DeploymentStatus, cat *knowled } } +// fitContextToMemory shrinks config["ctx_size"] so the llama.cpp KV cache plus +// model weights and the multimodal projector fit the detected memory, and caps it +// at the model's trained context. It only ever lowers ctx_size — never raises it. +// Returns (clamped, requestedCtx, appliedCtx, reason). It is a no-op (clamped=false) +// when ctx_size is unset, the GGUF architecture can't be read, or memory is unknown, +// so unsupported models/hardware degrade gracefully instead of erroring. +func fitContextToMemory(modelPath string, config map[string]any, hw knowledge.HardwareInfo) (bool, int, int, string) { + reqCtx := contextWindowFromResolvedConfig(config) + if reqCtx <= 0 || modelPath == "" || config == nil { + return false, 0, 0, "" + } + arch, ok := model.ReadKVArch(modelPath) + if !ok { + return false, 0, 0, "" // can't estimate KV → leave ctx_size untouched + } + + nonKVMiB := fileSizeMiB(modelPath) + if mm, _ := config["mmproj"].(string); mm != "" { + nonKVMiB += fileSizeMiB(mm) + } + + target, reasons := clampContextForMemory(reqCtx, arch.NCtxTrain, arch.KVBytesPerToken(), usableMemoryMiB(hw), nonKVMiB) + if target >= reqCtx { + return false, reqCtx, reqCtx, "" + } + config["ctx_size"] = target + return true, reqCtx, target, strings.Join(reasons, "; ") +} + +// clampContextForMemory computes the largest context window ≤ reqCtx that fits: +// (a) the model's trained context (nCtxTrain, 0 = unknown/skip) and (b) the KV +// budget left after weights+projector in usableMiB (0 = unknown/skip). kvPerTok +// is the f16 KV bytes per token. It floors at a minimally useful context. Pure +// (no I/O) for testability. +func clampContextForMemory(reqCtx, nCtxTrain int, kvPerTok int64, usableMiB, nonKVMiB int) (int, []string) { + const ( + computeReserveMiB = 1024 // llama.cpp compute buffers + minCtx = 2048 // floor — keep a usable context even on tiny memory + ) + target := reqCtx + var reasons []string + + if nCtxTrain > 0 && target > nCtxTrain { + target = nCtxTrain + reasons = append(reasons, fmt.Sprintf("capped at trained context %d", nCtxTrain)) + } + + if usableMiB > 0 && kvPerTok > 0 { + kvBudgetMiB := int(float64(usableMiB)*0.90) - nonKVMiB - computeReserveMiB + if kvBudgetMiB < 0 { + kvBudgetMiB = 0 + } + maxCtx := int(int64(kvBudgetMiB) * 1024 * 1024 / kvPerTok) + maxCtx -= maxCtx % 256 // clean multiple + if maxCtx < target { + target = maxCtx + reasons = append(reasons, fmt.Sprintf("%d MiB usable, weights+projector %d MiB, KV %d B/token", + usableMiB, nonKVMiB, kvPerTok)) + } + } + + if target < minCtx { + target = minCtx + } + return target, reasons +} + +// usableMemoryMiB returns the memory budget an all-layers-offloaded llama.cpp +// model can use: GPU VRAM for discrete GPUs, or system RAM minus an OS reserve +// for unified-memory / CPU hosts. Returns 0 when memory is unknown. +func usableMemoryMiB(hw knowledge.HardwareInfo) int { + ramReserve := func(total int) int { + reserve := total / 4 + if reserve < 2048 { + reserve = 2048 + } + if reserve > 16384 { + reserve = 16384 + } + return reserve + } + // An all-layers-offloaded llama.cpp model is bounded by GPU memory. For a + // unified-memory APU this is the carved iGPU pool (read via ROCm), which is the + // correct budget — NOT the OS-visible system RAM, which Win32 under-reports on + // such APUs (e.g. Strix Halo shows ~32 GB OS RAM but ~110 GB iGPU VRAM). Prefer + // GPU memory whenever it's known; fall back to system RAM only for CPU-only hosts. + if hw.GPUMemFreeMiB > 0 { + return hw.GPUMemFreeMiB + } + if hw.GPUVRAMMiB > 0 { + return hw.GPUVRAMMiB + } + if hw.RAMTotalMiB > 0 { + return hw.RAMTotalMiB - ramReserve(hw.RAMTotalMiB) + } + return 0 +} + +// fileSizeMiB returns the file's size in MiB, or 0 if it can't be stat'd. +func fileSizeMiB(path string) int { + if path == "" { + return 0 + } + fi, err := os.Stat(path) + if err != nil { + return 0 + } + return int(fi.Size() / (1024 * 1024)) +} + func contextWindowFromResolvedConfig(config map[string]any) int { if len(config) == 0 { return 0 diff --git a/cmd/aima/tooldeps_engine.go b/cmd/aima/tooldeps_engine.go index e3430931..8180dfd2 100644 --- a/cmd/aima/tooldeps_engine.go +++ b/cmd/aima/tooldeps_engine.go @@ -215,7 +215,7 @@ func buildEngineDeps(ac *appContext, deps *mcp.ToolDeps, if err := engine.Pull(ctx, engine.PullOptions{ Image: ea.Image.Name, Tag: ea.Image.Tag, - Registries: ea.Image.Registries, + Registries: engineRegistriesWithEnv(ea.Image.Registries), SizeHintMB: ea.Image.SizeApproxMB, OnProgress: reportProgress, Runner: &execRunner{}, @@ -256,8 +256,28 @@ func buildEngineDeps(ac *appContext, deps *mcp.ToolDeps, if err != nil { return fmt.Errorf("resolve path %s: %w", path, err) } + importNative := func() error { + distPlatform := goruntime.GOOS + "-" + goruntime.GOARCH + distDir := filepath.Join(dataDir, "dist", distPlatform) + mgr := engine.NewBinaryManager(distDir) + if err := mgr.ImportBundle(ctx, absPath, "", nil); err != nil { + return err + } + _, _ = scanEnginesCore(ctx, "native", false) + return nil + } + if looksLikeNativeEngineBundle(absPath) { + if err := importNative(); err != nil { + return fmt.Errorf("import native engine bundle from %s: %w", path, err) + } + return nil + } if err := engine.Import(ctx, absPath, &execRunner{}); err != nil { - return fmt.Errorf("import engine from %s: %w", path, err) + if nativeErr := importNative(); nativeErr == nil { + return nil + } else { + return fmt.Errorf("import engine from %s as container image failed: %w; native bundle import also failed: %v", path, err, nativeErr) + } } // Refresh DB: imported image only visible via runtime scan _, _ = scanEnginesCore(ctx, "auto", false) @@ -309,5 +329,23 @@ func buildEngineDeps(ac *appContext, deps *mcp.ToolDeps, } +func looksLikeNativeEngineBundle(path string) bool { + info, err := os.Stat(path) + if err == nil && info.IsDir() { + return true + } + lower := strings.ToLower(path) + switch { + case strings.HasSuffix(lower, ".zip"), + strings.HasSuffix(lower, ".tgz"), + strings.HasSuffix(lower, ".tar.gz"), + strings.HasSuffix(lower, ".exe"), + strings.HasSuffix(lower, ".appimage"): + return true + default: + return false + } +} + // suppress "imported and not used" for packages only used in type literals var _ = goruntime.GOOS diff --git a/internal/cli/deploy.go b/internal/cli/deploy.go index cda2a25b..6ed071c5 100644 --- a/internal/cli/deploy.go +++ b/internal/cli/deploy.go @@ -14,6 +14,7 @@ func newDeployCmd(app *App) *cobra.Command { engineType string slot string dryRun bool + noPull bool configOverrides []string maxColdStartS int ) @@ -46,7 +47,7 @@ func newDeployCmd(app *App) *cobra.Command { return nil } - data, err := app.ToolDeps.DeployApply(ctx, engineType, modelName, slot, configMap, false) + data, err := app.ToolDeps.DeployRun(ctx, modelName, engineType, slot, configMap, noPull, nil, nil, nil) if err != nil { return fmt.Errorf("deploy %s: %w", modelName, err) } @@ -59,6 +60,7 @@ func newDeployCmd(app *App) *cobra.Command { cmd.Flags().StringVar(&engineType, "engine", "", "Engine type (e.g., vllm, llamacpp)") cmd.Flags().StringVar(&slot, "slot", "", "Partition slot name") cmd.Flags().BoolVar(&dryRun, "dry-run", false, "Preview deployment without executing") + cmd.Flags().BoolVar(&noPull, "no-pull", false, "Skip auto-downloading missing engine/model") cmd.Flags().StringSliceVar(&configOverrides, "config", nil, "Config overrides (key=value, can repeat)") cmd.Flags().IntVar(&maxColdStartS, "max-cold-start", 0, "Max acceptable cold start seconds (0=no constraint)") cmd.AddCommand(newDeployListCmd(app)) @@ -142,6 +144,9 @@ func newUndeployCmd(app *App) *cobra.Command { } fmt.Fprintf(cmd.OutOrStdout(), "Deployment %s removed\n", name) + // A6: undeploy only stops the service; model files are kept on purpose + // (re-downloading is expensive). Point users at the cleanup command. + fmt.Fprintf(cmd.OutOrStdout(), "Note: model files are kept. To free disk: aima model remove --delete-files %s\n", name) return nil }, } diff --git a/internal/cli/engine.go b/internal/cli/engine.go index dc3f769d..31181cc1 100644 --- a/internal/cli/engine.go +++ b/internal/cli/engine.go @@ -142,8 +142,8 @@ type pullProgressRenderer struct { mu sync.Mutex w interface{ Write([]byte) (int, error) } isTTY bool - lastReport int // last reported percentage (for non-TTY deduplication) - started bool // whether we've printed any progress line + lastReport int // last reported percentage (for non-TTY deduplication) + started bool // whether we've printed any progress line lastUpdate time.Time } @@ -254,7 +254,7 @@ func formatDuration(seconds float64) string { func newEngineImportCmd(app *App) *cobra.Command { return &cobra.Command{ Use: "import ", - Short: "Import an engine image from a tar file", + Short: "Import an engine image or native runtime package", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -264,7 +264,7 @@ func newEngineImportCmd(app *App) *cobra.Command { return fmt.Errorf("import engine from %s: %w", tarPath, err) } - fmt.Fprintf(cmd.OutOrStdout(), "Engine image imported from %s\n", tarPath) + fmt.Fprintf(cmd.OutOrStdout(), "Engine imported from %s\n", tarPath) return nil }, } diff --git a/internal/cli/http_exec_test.go b/internal/cli/http_exec_test.go index 62fff817..229d2fa5 100644 --- a/internal/cli/http_exec_test.go +++ b/internal/cli/http_exec_test.go @@ -29,16 +29,18 @@ func TestExecuteLineDeployUsesRealCLIFlags(t *testing.T) { gotModel string gotSlot string gotConfig map[string]any + gotNoPull bool ) - app.ToolDeps.DeployApply = func(ctx context.Context, engine, model, slot string, config map[string]any, noPull bool) (json.RawMessage, error) { - gotEngine = engine + app.ToolDeps.DeployRun = func(ctx context.Context, model, engineType, slot string, config map[string]any, noPull bool, onPhase func(string, string), onProgress func(engine.ProgressEvent), onModelProgress func(int64, int64)) (json.RawMessage, error) { + gotEngine = engineType gotModel = model gotSlot = slot gotConfig = config - return json.RawMessage(`{"status":"ok"}`), nil + gotNoPull = noPull + return json.RawMessage(`{"status":"ready","name":"qwen3-8b-llamacpp","address":"127.0.0.1:8080","runtime":"native"}`), nil } - result := ExecuteLine(context.Background(), app, `deploy qwen3-8b --engine llamacpp --slot slot-1 --config gpu_memory_utilization=0.9 --config max_model_len=4096 --max-cold-start 12`, nil) + result := ExecuteLine(context.Background(), app, `deploy qwen3-8b --engine llamacpp --slot slot-1 --config gpu_memory_utilization=0.9 --config max_model_len=4096 --max-cold-start 12 --no-pull`, nil) if result.ExitCode != 0 { t.Fatalf("ExecuteLine exit_code=%d error=%q output=%q", result.ExitCode, result.Error, result.Output) } @@ -52,6 +54,9 @@ func TestExecuteLineDeployUsesRealCLIFlags(t *testing.T) { if gotSlot != "slot-1" { t.Fatalf("slot = %q, want %q", gotSlot, "slot-1") } + if !gotNoPull { + t.Fatal("expected no-pull=true") + } if gotConfig["gpu_memory_utilization"] != 0.9 { t.Fatalf("gpu_memory_utilization = %#v, want 0.9", gotConfig["gpu_memory_utilization"]) } diff --git a/internal/cli/serve.go b/internal/cli/serve.go index 1dcd8fb2..9d539d22 100644 --- a/internal/cli/serve.go +++ b/internal/cli/serve.go @@ -37,6 +37,7 @@ func newServeCmd(app *App) *cobra.Command { discoverEnabled bool allowInsecure bool staticBackends []string + noOpenClawSync bool ) cmd := &cobra.Command{ @@ -104,8 +105,10 @@ func newServeCmd(app *App) *cobra.Command { } go proxy.StartSyncLoop(ctx, app.Proxy, listFn, 5*time.Second) } - if app.OpenClaw != nil { + if app.OpenClaw != nil && openClawAutoSyncEnabled(noOpenClawSync) { go openclaw.StartSyncLoop(ctx, app.OpenClaw, 10*time.Second) + } else if app.OpenClaw != nil { + slog.Info("openclaw auto-sync loop disabled; sync only via `aima openclaw sync`") } // Auto-reconcile local assets on startup so Explorer, onboarding, and @@ -228,6 +231,7 @@ func newServeCmd(app *App) *cobra.Command { cmd.Flags().BoolVar(&discoverEnabled, "discover", false, "Discover remote inference services via mDNS") cmd.Flags().BoolVar(&allowInsecure, "allow-insecure-no-auth", false, "Allow non-loopback listen addresses without API key (NOT recommended)") cmd.Flags().StringArrayVar(&staticBackends, "backend", nil, "Static backend registration: model=http://host:port[/base],engine=vllm,upstream=served,param=35B,context=32768") + cmd.Flags().BoolVar(&noOpenClawSync, "no-openclaw-sync", false, "Disable the automatic OpenClaw config sync loop; sync only on explicit `aima openclaw sync` (or set AIMA_OPENCLAW_SYNC=manual)") return cmd } @@ -350,6 +354,21 @@ func parseMCPProfile(profile string) (mcp.Profile, error) { return p, nil } +// openClawAutoSyncEnabled reports whether `aima serve` should run the background +// OpenClaw config sync loop. A5: a partner can disable it (and drive sync only via +// explicit `aima openclaw sync`) with the --no-openclaw-sync flag or, at the +// product level, AIMA_OPENCLAW_SYNC=manual|off|false|0|no. +func openClawAutoSyncEnabled(noFlag bool) bool { + if noFlag { + return false + } + switch strings.ToLower(strings.TrimSpace(os.Getenv("AIMA_OPENCLAW_SYNC"))) { + case "manual", "off", "false", "0", "no": + return false + } + return true +} + func validateServeSecurity(addr, mcpAddr string, mcpEnabled bool, apiKey string, allowInsecure bool) error { if apiKey != "" || allowInsecure { return nil diff --git a/internal/cli/serve_a5_test.go b/internal/cli/serve_a5_test.go new file mode 100644 index 00000000..71b2c62d --- /dev/null +++ b/internal/cli/serve_a5_test.go @@ -0,0 +1,32 @@ +package cli + +import "testing" + +func TestOpenClawAutoSyncEnabled(t *testing.T) { + tests := []struct { + name string + flag bool + env string + envSet bool + want bool + }{ + {"default on", false, "", false, true}, + {"flag disables", true, "", false, false}, + {"env manual disables", false, "manual", true, false}, + {"env off disables", false, "off", true, false}, + {"env false disables", false, "false", true, false}, + {"env 0 disables", false, "0", true, false}, + {"env auto stays on", false, "auto", true, true}, + {"env empty stays on", false, "", true, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.envSet { + t.Setenv("AIMA_OPENCLAW_SYNC", tt.env) + } + if got := openClawAutoSyncEnabled(tt.flag); got != tt.want { + t.Errorf("openClawAutoSyncEnabled(flag=%v, env=%q) = %v, want %v", tt.flag, tt.env, got, tt.want) + } + }) + } +} diff --git a/internal/engine/binary.go b/internal/engine/binary.go index 2ec7270a..bfcbacec 100644 --- a/internal/engine/binary.go +++ b/internal/engine/binary.go @@ -10,7 +10,9 @@ import ( "io" "log/slog" "net/http" + "net/url" "os" + "path" "path/filepath" goruntime "runtime" "strings" @@ -29,13 +31,14 @@ func NewBinaryManager(distDir string) *BinaryManager { // BinarySource describes where to download a native binary. type BinarySource struct { - Binary string // e.g. "llama-server" - Platforms []string // e.g. ["linux/amd64", "darwin/arm64"] - Download map[string]string // platform -> URL - Mirror map[string][]string // platform -> mirror URLs (tried in order) - SHA256 map[string]string // platform -> expected hex digest (optional) - InstallType string // e.g. "preinstalled" - ProbePaths []string // explicit binary paths for pre-installed engines + Binary string // e.g. "llama-server" + Platforms []string // e.g. ["linux/amd64", "darwin/arm64"] + Download map[string]string // platform -> URL + Mirror map[string][]string // platform -> mirror URLs (tried in order) + SHA256 map[string]string // platform -> expected hex digest (optional) + InstallType string // e.g. "preinstalled" + ProbePaths []string // explicit binary paths for pre-installed engines + LocalBundles []string // local archive/binary/dir paths used before network download } // Supports reports whether this source supports the given platform string (e.g. "linux/amd64"). @@ -83,6 +86,15 @@ func (m *BinaryManager) Ensure(ctx context.Context, source *BinarySource, onProg return "", false, fmt.Errorf("preinstalled engine binary not found (probe paths: %v)", source.ProbePaths) } + if installed, err := m.installFromLocalBundles(ctx, source, onProgress); installed { + if path, ok := m.findExisting(source); ok { + return path, true, nil + } + return "", true, fmt.Errorf("binary %s not found in %s after installing local bundle", name, m.distDir) + } else if err != nil { + slog.Warn("local engine bundle install failed, falling back to download", "binary", name, "error", err) + } + url := source.Download[platform] mirrorURLs := source.Mirror[platform] if url == "" && len(mirrorURLs) == 0 { @@ -117,6 +129,12 @@ func (m *BinaryManager) Download(ctx context.Context, source *BinarySource, onPr return fmt.Errorf("engine is preinstalled on this host; no downloadable artifact is configured") } + if installed, err := m.installFromLocalBundles(ctx, source, onProgress); installed { + return nil + } else if err != nil { + slog.Warn("local engine bundle install failed, falling back to download", "binary", source.Binary, "error", err) + } + url := source.Download[platform] mirrorURLs := source.Mirror[platform] if url == "" && len(mirrorURLs) == 0 { @@ -156,6 +174,167 @@ func (m *BinaryManager) findExisting(source *BinarySource) (string, bool) { return "", false } +// ImportBundle installs a native engine archive, directory, or single binary into +// the manager's dist directory. It is used by air-gapped deployments and by +// `aima engine import` for native runtimes. +func (m *BinaryManager) ImportBundle(ctx context.Context, bundlePath, binaryName string, onProgress func(ProgressEvent)) error { + if strings.TrimSpace(bundlePath) == "" { + return fmt.Errorf("bundle path is required") + } + if err := os.MkdirAll(m.distDir, 0o755); err != nil { + return fmt.Errorf("create dist dir: %w", err) + } + if err := ctx.Err(); err != nil { + return err + } + if onProgress != nil { + onProgress(ProgressEvent{Phase: "importing", Message: "installing local engine bundle"}) + } + if err := installLocalBundle(bundlePath, m.distDir, binaryName, onProgress); err != nil { + return err + } + finalizeNativeDist(m.distDir, binaryName) + if onProgress != nil { + onProgress(ProgressEvent{Phase: "complete", Message: "engine binary ready"}) + } + return nil +} + +func (m *BinaryManager) installFromLocalBundles(ctx context.Context, source *BinarySource, onProgress func(ProgressEvent)) (bool, error) { + if source == nil || len(source.LocalBundles) == 0 { + return false, nil + } + var lastErr error + for _, candidate := range source.LocalBundles { + candidate = strings.TrimSpace(candidate) + if candidate == "" { + continue + } + if err := ctx.Err(); err != nil { + return false, err + } + slog.Info("installing engine binary from local bundle", "path", candidate, "dest", m.distDir) + if err := m.ImportBundle(ctx, candidate, source.Binary, onProgress); err != nil { + lastErr = err + slog.Warn("local engine bundle failed", "path", candidate, "error", err) + continue + } + return true, nil + } + if lastErr != nil { + return false, fmt.Errorf("all local engine bundles failed: %w", lastErr) + } + return false, nil +} + +func installLocalBundle(bundlePath, destDir, binaryName string, onProgress func(ProgressEvent)) error { + info, err := os.Stat(bundlePath) + if err != nil { + return fmt.Errorf("stat local engine bundle %s: %w", bundlePath, err) + } + if info.IsDir() { + return copyDirContents(bundlePath, destDir) + } + lower := strings.ToLower(bundlePath) + switch { + case strings.HasSuffix(lower, ".tar.gz") || strings.HasSuffix(lower, ".tgz"): + if onProgress != nil { + onProgress(ProgressEvent{Phase: "extracting", Message: "extracting local engine archive"}) + } + return extractTarGz(bundlePath, destDir) + case strings.HasSuffix(lower, ".zip"): + if onProgress != nil { + onProgress(ProgressEvent{Phase: "extracting", Message: "extracting local engine archive"}) + } + return extractZip(bundlePath, destDir) + default: + destName := filepath.Base(bundlePath) + if binaryName != "" && !isBinaryCandidate(destName, binaryName) { + destName = binaryName + if goruntime.GOOS == "windows" && !strings.HasSuffix(strings.ToLower(destName), ".exe") { + destName += ".exe" + } + } + return copyFile(bundlePath, filepath.Join(destDir, destName), info.Mode()) + } +} + +func finalizeNativeDist(destDir, binaryName string) { + if goruntime.GOOS != "windows" { + for _, c := range binaryCandidates(binaryName) { + if c == "" { + continue + } + p := filepath.Join(destDir, c) + if _, err := os.Stat(p); err == nil { + _ = os.Chmod(p, 0o755) + break + } + } + createSoSymlinks(destDir) + } +} + +func isBinaryCandidate(name, binaryName string) bool { + if binaryName == "" { + return true + } + for _, candidate := range binaryCandidates(binaryName) { + if strings.EqualFold(name, candidate) { + return true + } + } + return false +} + +func copyFile(src, dst string, perm os.FileMode) error { + in, err := os.Open(src) + if err != nil { + return fmt.Errorf("open %s: %w", src, err) + } + defer in.Close() + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return fmt.Errorf("create directory %s: %w", filepath.Dir(dst), err) + } + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, perm) + if err != nil { + return fmt.Errorf("create %s: %w", dst, err) + } + if _, err := io.Copy(out, in); err != nil { + _ = out.Close() + return fmt.Errorf("copy %s to %s: %w", src, dst, err) + } + if err := out.Close(); err != nil { + return fmt.Errorf("close %s: %w", dst, err) + } + return nil +} + +func copyDirContents(srcDir, dstDir string) error { + srcClean := filepath.Clean(srcDir) + return filepath.WalkDir(srcClean, func(srcPath string, d os.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(srcClean, srcPath) + if err != nil { + return err + } + if rel == "." { + return nil + } + dstPath := filepath.Join(dstDir, rel) + if d.IsDir() { + return os.MkdirAll(dstPath, 0o755) + } + info, err := d.Info() + if err != nil { + return err + } + return copyFile(srcPath, dstPath, info.Mode()) + }) +} + // download tries mirror URLs first, then primary. Extracts zip/tar.gz archives. func (m *BinaryManager) download(ctx context.Context, url string, mirrorURLs []string, destDir, binaryName, expectedSHA256 string, onProgress func(ProgressEvent)) error { if err := os.MkdirAll(destDir, 0o755); err != nil { @@ -163,10 +342,9 @@ func (m *BinaryManager) download(ctx context.Context, url string, mirrorURLs []s } // Mirrors first (typically faster for CN users), primary last - urls := make([]string, 0, len(mirrorURLs)+1) - urls = append(urls, mirrorURLs...) - if url != "" { - urls = append(urls, url) + urls := buildDownloadSourceList(url, mirrorURLs) + if len(urls) == 0 { + return fmt.Errorf("no download URL configured") } var lastErr error @@ -187,21 +365,7 @@ func (m *BinaryManager) download(ctx context.Context, url string, mirrorURLs []s } } - // Make binary executable on non-Windows - if goruntime.GOOS != "windows" { - for _, c := range binaryCandidates(binaryName) { - p := filepath.Join(destDir, c) - if _, err := os.Stat(p); err == nil { - os.Chmod(p, 0o755) - break - } - } - } - - // Create missing .so.X → .so.X.Y.Z symlinks so dlopen finds versioned libraries. - if goruntime.GOOS != "windows" { - createSoSymlinks(destDir) - } + finalizeNativeDist(destDir, binaryName) slog.Info("engine binary ready", "dir", destDir, "binary", binaryName) if onProgress != nil { @@ -213,6 +377,102 @@ func (m *BinaryManager) download(ctx context.Context, url string, mirrorURLs []s return fmt.Errorf("all download sources failed: %w", lastErr) } +func buildDownloadSourceList(primary string, mirrors []string) []string { + urls := make([]string, 0, len(mirrors)+1+4) + urls = append(urls, envMirrorURLs(primary)...) + urls = append(urls, mirrors...) + if primary != "" { + urls = append(urls, primary) + } + return uniqueNonEmpty(urls) +} + +func envMirrorURLs(primary string) []string { + if primary == "" { + return nil + } + filename := downloadFileName(primary) + var out []string + for _, base := range splitCommaEnv("AIMA_ENGINE_MIRROR_BASE", "AIMA_ENGINE_MIRROR") { + if filename == "" { + continue + } + out = append(out, joinURLPath(base, filename)) + } + for _, tmpl := range splitCommaEnv("AIMA_ENGINE_MIRROR_TEMPLATE") { + repl := strings.NewReplacer( + "{url}", primary, + "{escaped_url}", url.QueryEscape(primary), + "{filename}", filename, + ) + out = append(out, repl.Replace(tmpl)) + } + for _, rule := range splitCommaEnv("AIMA_ENGINE_URL_REWRITE") { + from, to, ok := strings.Cut(rule, "=>") + if !ok { + from, to, ok = strings.Cut(rule, "=") + } + from = strings.TrimSpace(from) + to = strings.TrimSpace(to) + if ok && from != "" && strings.HasPrefix(primary, from) { + out = append(out, to+strings.TrimPrefix(primary, from)) + } + } + return uniqueNonEmpty(out) +} + +func splitCommaEnv(names ...string) []string { + var values []string + for _, name := range names { + raw := os.Getenv(name) + for _, part := range strings.FieldsFunc(raw, func(r rune) bool { + return r == ',' || r == '\n' + }) { + if trimmed := strings.TrimSpace(part); trimmed != "" { + values = append(values, trimmed) + } + } + } + return values +} + +func downloadFileName(rawURL string) string { + parsed, err := url.Parse(rawURL) + if err == nil && parsed.Path != "" { + return path.Base(parsed.Path) + } + return path.Base(rawURL) +} + +func joinURLPath(base, elem string) string { + base = strings.TrimRight(strings.TrimSpace(base), "/") + elem = strings.TrimLeft(strings.TrimSpace(elem), "/") + if base == "" { + return elem + } + if elem == "" { + return base + } + return base + "/" + elem +} + +func uniqueNonEmpty(in []string) []string { + seen := make(map[string]struct{}, len(in)) + out := make([]string, 0, len(in)) + for _, value := range in { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + return out +} + // downloadAndExtract downloads url to a temp file then extracts or renames it. // Returns the SHA256 hex digest of the downloaded content. func downloadAndExtract(ctx context.Context, url, destDir, binaryName string, onProgress func(ProgressEvent)) (string, error) { diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index 399eac65..998c7129 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -1,6 +1,7 @@ package engine import ( + "archive/zip" "context" "encoding/json" "fmt" @@ -299,6 +300,86 @@ func TestBinaryManagerEnsureUsesProbePathsForPreinstalledEngine(t *testing.T) { } } +func TestBinaryManagerEnsureInstallsLocalZipBundle(t *testing.T) { + t.Parallel() + + distDir := t.TempDir() + bundleDir := t.TempDir() + binaryName := "llama-server" + binaryFile := binaryName + if goruntime.GOOS == "windows" { + binaryFile += ".exe" + } + archivePath := filepath.Join(bundleDir, "llama-runtime.zip") + zipFile, err := os.Create(archivePath) + if err != nil { + t.Fatalf("create zip: %v", err) + } + zw := zip.NewWriter(zipFile) + header := &zip.FileHeader{Name: "llama-runtime/" + binaryFile, Method: zip.Deflate} + header.SetMode(0o755) + w, err := zw.CreateHeader(header) + if err != nil { + t.Fatalf("create zip entry: %v", err) + } + if _, err := w.Write([]byte("bin")); err != nil { + t.Fatalf("write zip entry: %v", err) + } + if err := zw.Close(); err != nil { + t.Fatalf("close zip writer: %v", err) + } + if err := zipFile.Close(); err != nil { + t.Fatalf("close zip file: %v", err) + } + + mgr := NewBinaryManager(distDir) + source := &BinarySource{ + Binary: binaryName, + Platforms: []string{goruntime.GOOS + "/" + goruntime.GOARCH}, + LocalBundles: []string{archivePath}, + } + + path, downloaded, err := mgr.Ensure(context.Background(), source, nil) + if err != nil { + t.Fatalf("Ensure: %v", err) + } + if !downloaded { + t.Fatal("Ensure should report local bundle installation") + } + want := filepath.Join(distDir, binaryFile) + if path != want { + t.Fatalf("Ensure path = %q, want %q", path, want) + } + if data, err := os.ReadFile(want); err != nil || string(data) != "bin" { + t.Fatalf("installed binary = %q, %v; want bin", data, err) + } +} + +func TestBuildDownloadSourceListUsesEnterpriseMirrors(t *testing.T) { + t.Setenv("AIMA_ENGINE_MIRROR_BASE", "https://repo.local/aima") + t.Setenv("AIMA_ENGINE_MIRROR_TEMPLATE", "https://proxy.local/{filename},https://encoded.local/{escaped_url}") + t.Setenv("AIMA_ENGINE_URL_REWRITE", "https://github.com/=>https://gitcache.local/") + + primary := "https://github.com/ggml-org/llama.cpp/releases/download/b9330/llama-b9330-bin-win-hip-radeon-x64.zip" + got := buildDownloadSourceList(primary, []string{"https://catalog.local/llama.zip"}) + wantPrefix := []string{ + "https://repo.local/aima/llama-b9330-bin-win-hip-radeon-x64.zip", + "https://proxy.local/llama-b9330-bin-win-hip-radeon-x64.zip", + "https://encoded.local/https%3A%2F%2Fgithub.com%2Fggml-org%2Fllama.cpp%2Freleases%2Fdownload%2Fb9330%2Fllama-b9330-bin-win-hip-radeon-x64.zip", + "https://gitcache.local/ggml-org/llama.cpp/releases/download/b9330/llama-b9330-bin-win-hip-radeon-x64.zip", + "https://catalog.local/llama.zip", + primary, + } + if len(got) != len(wantPrefix) { + t.Fatalf("sources = %#v, want %#v", got, wantPrefix) + } + for i := range wantPrefix { + if got[i] != wantPrefix[i] { + t.Fatalf("sources[%d] = %q, want %q\nall=%#v", i, got[i], wantPrefix[i], got) + } + } +} + func TestPatternMatchExactAnchors(t *testing.T) { // ^pattern$ should match exactly patterns := []patternEntry{ diff --git a/internal/mcp/tools_engine.go b/internal/mcp/tools_engine.go index ac4e2945..97b5d577 100644 --- a/internal/mcp/tools_engine.go +++ b/internal/mcp/tools_engine.go @@ -108,8 +108,8 @@ func registerEngineTools(s *Server, deps *ToolDeps) { // engine.import s.RegisterTool(&Tool{ Name: "engine.import", - Description: "Import an engine container image from a local OCI tar file and register it (airgap use case).", - InputSchema: schema(`"path":{"type":"string","description":"Absolute path to the OCI tar file, e.g. '/data/images/vllm-cuda.tar'"}`, "path"), + Description: "Import an engine container image or native runtime package from a local file and register it (airgap use case). Supports OCI tar files, native zip/tar.gz bundles, directories, and single binaries.", + InputSchema: schema(`"path":{"type":"string","description":"Absolute path to the engine package, e.g. '/data/images/vllm-cuda.tar' or '/data/runtime/llama-b9330-win-hip-radeon-x64.zip'"}`, "path"), Handler: func(ctx context.Context, params json.RawMessage) (*ToolResult, error) { if deps.ImportEngine == nil { return ErrorResult("engine.import not implemented"), nil @@ -126,7 +126,7 @@ func registerEngineTools(s *Server, deps *ToolDeps) { if err := deps.ImportEngine(ctx, p.Path); err != nil { return nil, fmt.Errorf("import engine from %s: %w", p.Path, err) } - return TextResult(fmt.Sprintf("engine image imported from %s", p.Path)), nil + return TextResult(fmt.Sprintf("engine imported from %s", p.Path)), nil }, }) diff --git a/internal/model/gguf.go b/internal/model/gguf.go index be4169bf..9d8e3208 100644 --- a/internal/model/gguf.go +++ b/internal/model/gguf.go @@ -207,10 +207,12 @@ func parseGGUFMeta(path string) map[string]any { keyMap := map[string]string{ ".block_count": "num_hidden_layers", + ".context_length": "max_position_embeddings", ".embedding_length": "hidden_size", ".feed_forward_length": "intermediate_size", ".attention.head_count": "num_attention_heads", ".attention.head_count_kv": "num_key_value_heads", + ".attention.key_length": "head_dim", ".vocab_size": "vocab_size", ".expert_count": "num_experts", ".expert_used_count": "num_experts_per_tok", @@ -233,6 +235,49 @@ func parseGGUFMeta(path string) map[string]any { return config } +// KVArch holds the GGUF architecture fields needed to size the KV cache. +type KVArch struct { + NLayer int // transformer blocks (block_count) + NHeadKV int // KV heads (head_count_kv; equals head_count for MHA) + HeadDim int // per-head dimension + NCtxTrain int // trained context length (max_position_embeddings); 0 if unknown +} + +// KVBytesPerToken returns the f16 KV-cache size for one token across all layers: +// 2 (K+V) * n_layer * n_head_kv * head_dim * 2 bytes. +func (a KVArch) KVBytesPerToken() int64 { + return int64(2) * int64(a.NLayer) * int64(a.NHeadKV) * int64(a.HeadDim) * 2 +} + +// ReadKVArch parses a GGUF file's header and returns the architecture needed to +// estimate KV-cache memory. ok is false when the file can't be parsed or lacks +// the required fields (caller should then skip memory-based context sizing). +func ReadKVArch(path string) (KVArch, bool) { + meta := parseGGUFMeta(path) + if meta == nil { + return KVArch{}, false + } + nLayer := jsonInt(meta, "num_hidden_layers") + nHeadKV := jsonInt(meta, "num_key_value_heads") + nHead := jsonInt(meta, "num_attention_heads") + if nHeadKV == 0 { + nHeadKV = nHead // MHA: KV heads == attention heads + } + headDim := jsonInt(meta, "head_dim") + if headDim == 0 && nHead > 0 { + headDim = jsonInt(meta, "hidden_size") / nHead + } + if nLayer == 0 || nHeadKV == 0 || headDim == 0 { + return KVArch{}, false + } + return KVArch{ + NLayer: nLayer, + NHeadKV: nHeadKV, + HeadDim: headDim, + NCtxTrain: jsonInt(meta, "max_position_embeddings"), + }, true +} + func ggufReadString(r io.Reader) (string, error) { var length uint64 if err := binary.Read(r, binary.LittleEndian, &length); err != nil { diff --git a/internal/openclaw/config.go b/internal/openclaw/config.go index 9f04ae9a..e3ac324a 100644 --- a/internal/openclaw/config.go +++ b/internal/openclaw/config.go @@ -138,6 +138,16 @@ func mergeLLMProvider(cfg map[string]any, managed, next *ManagedState, result *S } func mergeChatModelDefault(cfg map[string]any, managed, next *ManagedState, result *SyncResult) { + if result != nil && result.SkipDefaultModel { + // Partner opted out of AIMA managing the primary model: leave whatever the + // user has set as their default untouched, but carry forward any prior AIMA + // ownership record so toggling this back on still works cleanly. + if managed != nil { + next.ChatModelProvider = managed.ChatModelProvider + next.ChatModelModels = managed.ChatModelModels + } + return + } desired := uniqueSorted(modelIDs(result.LLMModels)) if len(desired) == 0 { if managedOwnsChatModel(managed) { @@ -147,9 +157,6 @@ func mergeChatModelDefault(cfg map[string]any, managed, next *ManagedState, resu } return } - if !canManageChatModelDefault(cfg, managed) { - return - } setAgentDefaultModel(cfg, "model", aimaLLMProviderID, desired) next.ChatModelProvider = aimaLLMProviderID next.ChatModelModels = desired diff --git a/internal/openclaw/openclaw.go b/internal/openclaw/openclaw.go index 891eb206..e7a920e9 100644 --- a/internal/openclaw/openclaw.go +++ b/internal/openclaw/openclaw.go @@ -15,6 +15,11 @@ type Deps struct { ProxyAddr string // e.g. "http://127.0.0.1:6188/v1" APIKey func() string // AIMA proxy API key getter (may return empty) MCPCommand string // Absolute path or command name for spawning `aima mcp` + // SetDefaultModel controls whether sync sets the synced LLM as OpenClaw's + // primary/default chat model. nil = default (set it); *false = register the + // provider+models but leave the user's current primary untouched. Lets a + // partner own that choice at the product level (env AIMA_OPENCLAW_SET_DEFAULT). + SetDefaultModel *bool } // BackendLister provides read-only access to the proxy's backend table. @@ -26,6 +31,7 @@ type BackendLister interface { type Backend struct { ModelName string EngineType string + ModelType string Address string Ready bool Remote bool diff --git a/internal/openclaw/setdefault_test.go b/internal/openclaw/setdefault_test.go new file mode 100644 index 00000000..4306f6a7 --- /dev/null +++ b/internal/openclaw/setdefault_test.go @@ -0,0 +1,67 @@ +package openclaw + +import "testing" + +// Item 4: SkipDefaultModel registers the provider+models but leaves OpenClaw's +// primary chat model untouched. +func TestMergeSkipDefaultModel(t *testing.T) { + build := func(skip bool) map[string]any { + result := &SyncResult{ + LLMModels: []ModelEntry{{ + ID: "qwen3-8b", Name: "Qwen3 8B", Input: []string{"text"}, + ContextWindow: 32768, MaxTokens: 16384, + }}, + ProxyAddr: "http://127.0.0.1:6188/v1", + SkipDefaultModel: skip, + } + cfg, _ := MergeAIMAConfigWithState(map[string]any{}, nil, result) + return cfg + } + + // Default: provider registered AND primary set to the AIMA model. + def := build(false) + if lookupMap(def, "models", "providers", aimaLLMProviderID) == nil { + t.Fatal("default: aima provider should be registered") + } + if !hasAgentDefaultModel(def, "model") { + t.Error("default: expected agents.defaults.model to be set") + } + + // Skip: provider STILL registered, but the primary is NOT set. + sk := build(true) + if lookupMap(sk, "models", "providers", aimaLLMProviderID) == nil { + t.Error("skip: aima provider should still be registered") + } + if hasAgentDefaultModel(sk, "model") { + t.Error("skip: agents.defaults.model must be left untouched") + } +} + +func TestMergeDefaultModelOverridesExistingWhenNotSkipped(t *testing.T) { + result := &SyncResult{ + LLMModels: []ModelEntry{{ + ID: "qwen3-8b", Name: "Qwen3 8B", Input: []string{"text"}, + ContextWindow: 32768, MaxTokens: 16384, + }}, + ProxyAddr: "http://127.0.0.1:6188/v1", + } + existing := map[string]any{ + "agents": map[string]any{ + "defaults": map[string]any{ + "model": map[string]any{"primary": "minimax/MiniMax-M2.1"}, + }, + }, + } + + cfg, managed := MergeAIMAConfigWithState(existing, nil, result) + defaultModel := lookupMap(cfg, "agents", "defaults", "model") + if defaultModel == nil { + t.Fatal("agents.defaults.model missing") + } + if got := defaultModel["primary"]; got != "aima/qwen3-8b" { + t.Fatalf("primary = %v, want aima/qwen3-8b", got) + } + if managed.ChatModelProvider != "aima" { + t.Fatalf("managed chat provider = %q, want aima", managed.ChatModelProvider) + } +} diff --git a/internal/openclaw/sync.go b/internal/openclaw/sync.go index 01aff1c1..a07bef39 100644 --- a/internal/openclaw/sync.go +++ b/internal/openclaw/sync.go @@ -31,19 +31,20 @@ var deployedPluginRoots = []string{ // SyncResult holds the categorized models ready for OpenClaw config generation. type SyncResult struct { - LLMModels []ModelEntry `json:"llmModels,omitempty"` - VLMModels []ModelEntry `json:"vlmModels,omitempty"` - ASRModels []AudioEntry `json:"asrModels,omitempty"` - TTSModel *TTSEntry `json:"ttsModel,omitempty"` - ImageGenModels []ImageGenEntry `json:"imageGenModels,omitempty"` - MCPServer *MCPServerEntry `json:"mcpServer,omitempty"` - ProxyAddr string `json:"proxyAddr"` - APIKey string `json:"apiKey,omitempty"` - ProxyReachable bool `json:"proxyReachable"` - ProxyWarning string `json:"proxyWarning,omitempty"` - ConfigPath string `json:"configPath"` - ConfigExists bool `json:"configExists"` - Written bool `json:"written"` + LLMModels []ModelEntry `json:"llmModels,omitempty"` + VLMModels []ModelEntry `json:"vlmModels,omitempty"` + ASRModels []AudioEntry `json:"asrModels,omitempty"` + TTSModel *TTSEntry `json:"ttsModel,omitempty"` + ImageGenModels []ImageGenEntry `json:"imageGenModels,omitempty"` + MCPServer *MCPServerEntry `json:"mcpServer,omitempty"` + ProxyAddr string `json:"proxyAddr"` + APIKey string `json:"apiKey,omitempty"` + ProxyReachable bool `json:"proxyReachable"` + ProxyWarning string `json:"proxyWarning,omitempty"` + SkipDefaultModel bool `json:"skipDefaultModel,omitempty"` + ConfigPath string `json:"configPath"` + ConfigExists bool `json:"configExists"` + Written bool `json:"written"` } // MCPServerEntry describes the stdio MCP server entry AIMA wants OpenClaw to use. @@ -91,6 +92,8 @@ func Sync(ctx context.Context, deps *Deps, dryRun bool) (*SyncResult, error) { APIKey: deps.proxyAPIKey(), ConfigPath: deps.ConfigPath, MCPServer: desiredMCPServer(deps), + // Skip touching OpenClaw's primary chat model only when explicitly disabled. + SkipDefaultModel: deps.SetDefaultModel != nil && !*deps.SetDefaultModel, } // Preflight: the provider we write points OpenClaw's chat data plane at @@ -112,7 +115,10 @@ func Sync(ctx context.Context, deps *Deps, dryRun bool) (*SyncResult, error) { continue } - modelType := deps.Catalog.ModelType(b.ModelName) + modelType := strings.TrimSpace(deps.Catalog.ModelType(b.ModelName)) + if modelType == "" { + modelType = strings.TrimSpace(b.ModelType) + } switch modelType { case "llm", "vlm": ctxWindow := b.ContextWindowTokens // prefer actual deployment config diff --git a/internal/openclaw/sync_test.go b/internal/openclaw/sync_test.go index 6420ef05..fc6dc6ca 100644 --- a/internal/openclaw/sync_test.go +++ b/internal/openclaw/sync_test.go @@ -211,6 +211,50 @@ func TestSyncWritesConfig(t *testing.T) { } } +func TestSyncUsesBackendModelTypeWhenCatalogMisses(t *testing.T) { + tmpDir := t.TempDir() + configPath := filepath.Join(tmpDir, "openclaw.json") + deps := &Deps{ + Backends: &mockBackends{backends: map[string]*Backend{ + "Qwen3.5-2B-Q4_K_M": { + ModelName: "Qwen3.5-2B-Q4_K_M", + EngineType: "llamacpp", + ModelType: "llm", + Address: "http://127.0.0.1:8080", + Ready: true, + }, + }}, + Catalog: &mockCatalog{}, + ConfigPath: configPath, + ProxyAddr: "http://127.0.0.1:6188/v1", + MCPCommand: "/usr/local/bin/aima", + } + + if _, err := Sync(context.Background(), deps, false); err != nil { + t.Fatalf("Sync failed: %v", err) + } + cfg, err := ReadConfig(configPath) + if err != nil { + t.Fatalf("ReadConfig failed: %v", err) + } + provider := lookupMap(cfg, "models", "providers", "aima") + if provider == nil { + t.Fatal("aima provider missing") + } + models, ok := provider["models"].([]any) + if !ok || len(models) != 1 { + t.Fatalf("provider models = %#v, want one model", provider["models"]) + } + model, ok := models[0].(map[string]any) + if !ok || model["id"] != "Qwen3.5-2B-Q4_K_M" { + t.Fatalf("provider model = %#v, want Qwen3.5-2B-Q4_K_M", models[0]) + } + defaultModel := lookupMap(cfg, "agents", "defaults", "model") + if defaultModel == nil || defaultModel["primary"] != "aima/Qwen3.5-2B-Q4_K_M" { + t.Fatalf("agents.defaults.model = %#v, want aima/Qwen3.5-2B-Q4_K_M", defaultModel) + } +} + func TestMergeAIMAConfigReplacesStaleAIMAMediaChatDefault(t *testing.T) { proxyAddr := "http://127.0.0.1:6188/v1" existing := map[string]any{