perf(bluemagpie): drop per-eval weight dequant + fuse QKV/gate-up + native rope

Phase-4 profiling (new CODEC_OP_PROFILE per-node profiler in
graph_exec.cpp, ggml_backend_sched_set_eval_callback based, zero-cost
when unset) showed the CFM step spent 31% of compute in CPY nodes:
codec_graph_weight bakes an F16->F32 dequant of every weight into the
graph, re-run every eval, while the matmuls themselves were 1%.

- tensor_utils: new codec_graph_weight_mat — F16/BF16 matmul weights
  pass through to ggml_mul_mat natively (no dequant CPY); quantized
  types still cast to F32 for parity (native quantized mul_mat drops
  Q8 audio corr to 0.9981; CODEC_MAT_NATIVE_QUANT=1 opts into the
  lossy fast path).  codec_graph_weight itself unchanged, other
  models unaffected; only BlueMagpie call sites switched.
- Converter writes fused .attn_qkv.w / .gate_up.w for the three
  MiniCPM stacks (7 -> 5 matmuls per block); runtime splits with
  views and keeps unfused-name fallback.  _emit_lm now honors
  --quantization (F16 default restores a genuine F16 lm gguf).
- RoPE via native ggml_rope_ext (NEOX, freq_factors=rope_short_factor,
  attn_factor baked at convert time); ~1540 fewer nodes; baked
  cos/sin fallback kept.  Scale folded into soft_max_ext.
- flash_attn_ext and in-graph set_rows KV write evaluated and
  deliberately skipped: attention is <8% of compute and the KV host
  round-trip <3 ms; measurements in the phase report.

At official defaults (cfg=2.8, ts=9): CFM-only CPU-8 F16 305 -> 154
ms/step, Vulkan F16 116 ms/step (new best; beats Q8 on both speed and
precision on UMA), full step_generate 173 ms/step warm (~realtime for
160 ms audio/step).  Parity: F16 patch corr 0.999997 / audio 0.999977;
Q8 unchanged; decode + moss cross-model smokes PASS; 72-step runs
bit-identical.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: hans00

scripts/converters/bluemagpie.py

@@ -290,19 +290,22 @@ def lm_t(name: str) -> np.ndarray:
                 raise KeyError(f"missing LM tensor: {name}")
             return np.asarray(arr)
 
-        # LM weight-streaming acceleration: 2D matmul weights under lm.* are
-        # quantized to Q8_0 (halves the FP16 bytes streamed per CFM step).  The
-        # gguf Q8_0 helper quantizes along ne[0] (the innermost dim = the matmul
-        # contraction dim), so rows must be a multiple of the 32-elem block.
-        # Anything else (norms, biases, 1D vectors, non-.w tensors, or rows not
-        # /32) falls back to F16 — matching the rest of the LM section's default.
-        # This is independent of the global --quantization flag so the AudioVAE
-        # section keeps its existing F16 behaviour untouched.
+        # LM matmul weight dtype follows the --quantization flag:
+        #   - Q8_0  → 2D lm.* matmul weights are Q8_0 (halves the bytes streamed
+        #             per CFM step; the gguf Q8_0 helper quantizes along ne[0] =
+        #             the matmul contraction dim, so rows must be a 32-multiple).
+        #   - else  → F16 (the genuine "F16 GGUF"), which the runtime feeds to
+        #             ggml_mul_mat natively (no per-eval dequant CPY, see
+        #             codec_graph_weight_mat).  On CPU the F16 path is markedly
+        #             faster AND higher-parity than cast-from-Q8, so the two
+        #             GGUFs are meant to be genuinely different here.
+        # Anything not eligible (norms, biases, 1D vectors, rows not /32) is F16.
         _Q8_BLK = 32
+        _lm_use_q8 = (self.quantization == "Q8_0")
 
         def _add_lm_weight(name: str, arr: np.ndarray) -> None:
             arr = np.asarray(arr)
-            if arr.ndim == 2 and (int(arr.shape[-1]) % _Q8_BLK) == 0:
+            if _lm_use_q8 and arr.ndim == 2 and (int(arr.shape[-1]) % _Q8_BLK) == 0:
                 self._add_tensor(writer, name, arr, "Q8_0")
             else:
                 self._add_tensor(writer, name, arr, "F16")
@@ -318,12 +321,19 @@ def add_norm(prefix: str, out: str) -> None:
         def add_minicpm_stack(src: str, out: str, n_layers: int) -> None:
             for i in range(n_layers):
                 s, o = f"{src}.layers.{i}", f"{out}.layers.{i}"
-                add_lin(f"{s}.self_attn.q_proj", o + ".attn_q")
-                add_lin(f"{s}.self_attn.k_proj", o + ".attn_k")
-                add_lin(f"{s}.self_attn.v_proj", o + ".attn_v")
+                # Fuse Q|K|V and gate|up into single matmul weights (concat along
+                # the output dim = PyTorch axis 0 of the (out,in) weight).  The
+                # contraction dim ne[0]=in is unchanged, so Q8_0 block alignment
+                # (in % 32) is preserved.  The runtime splits the outputs with
+                # views (see codec_bm_minicpm_block_htb).
+                q = lm_t(f"{s}.self_attn.q_proj.weight")
+                k = lm_t(f"{s}.self_attn.k_proj.weight")
+                v = lm_t(f"{s}.self_attn.v_proj.weight")
+                _add_lm_weight(o + ".attn_qkv.w", np.concatenate([q, k, v], axis=0))
                 add_lin(f"{s}.self_attn.o_proj", o + ".attn_o")
-                add_lin(f"{s}.mlp.gate_proj", o + ".gate")
-                add_lin(f"{s}.mlp.up_proj", o + ".up")
+                g  = lm_t(f"{s}.mlp.gate_proj.weight")
+                up = lm_t(f"{s}.mlp.up_proj.weight")
+                _add_lm_weight(o + ".gate_up.w", np.concatenate([g, up], axis=0))
                 add_lin(f"{s}.mlp.down_proj", o + ".down")
                 add_norm(f"{s}.input_layernorm", o + ".ln1")
                 add_norm(f"{s}.post_attention_layernorm", o + ".ln2")
@@ -384,6 +394,16 @@ def add_minicpm_stack(src: str, out: str, n_layers: int) -> None:
         self._add_tensor(writer, "lm.rope.cos", (np.cos(emb) * scaling).astype(np.float32), "F32")
         self._add_tensor(writer, "lm.rope.sin", (np.sin(emb) * scaling).astype(np.float32), "F32")
 
+        # Native-rope path: ggml_rope_ext(mode=NEOX, freq_base=theta,
+        # freq_factors=short, attn_factor=scaling, ext_factor=0, freq_scale=1)
+        # reproduces the LongRoPE short_factor branch above in a single node per
+        # q/k (vs the ~8-node baked cos/sin rotate_half).  short_factor is stored
+        # F32 len head_dim/2; theta + attn_factor go to KV.  The cos/sin table is
+        # retained so older runtimes / a fallback path still work.
+        self._add_tensor(writer, "lm.rope.short_factor", short.astype(np.float32), "F32")
+        writer.add_float32("codec.lm.rope_theta", float(cfg["rope_theta"]))
+        writer.add_float32("codec.lm.rope_attn_factor", float(scaling))
+
         # LM metadata
         writer.add_bool("codec.lm.has_adaptor", True)
         writer.add_string("codec.lm.kind", "continuous_latent_cfm")

src/lm/bluemagpie_cfm.cpp

@@ -66,6 +66,10 @@ ggml_tensor * lin(ggml_context * c, ggml_tensor * w, ggml_tensor * x, ggml_tenso
     return b ? ggml_add(c, y, codec_graph_cast_f32(c, b)) : y;
 }
 
+// Matmul-weight accessor: no dequant CPY for F16/BF16 (see
+// codec_graph_weight_mat).  Use for every tensor that lands as the LHS of
+// ggml_mul_mat in this per-step graph.
+
 // One RALM layer, incremental KV (1 new token).  Causal, no rope, no qk_norm.
 // Attends over a fixed bucket cache[0..B) plus the 1 new token, where B is a
 // 64-multiple >= kv_pos+1 so the graph shape is kv_pos-independent within a
@@ -79,13 +83,24 @@ ggml_tensor * bm_ralm_kv_step(ggml_context * ctx, ggml_tensor * x_ht, const std:
     ggml_tensor * k_cache_l, ggml_tensor * v_cache_l, int32_t bucket, ggml_tensor * mask,
     ggml_tensor ** kk_out, ggml_tensor ** vv_out) {
 
-    auto W = [&](const char * s) { return codec_graph_weight(ctx, model, prefix + s); };
+    auto W  = [&](const char * s) { return codec_graph_weight(ctx, model, prefix + s); };
+    auto WM = [&](const char * s) { return codec_graph_weight_mat(ctx, model, prefix + s); };
     const int32_t q_dim = n_heads * head_dim;
 
+    const int32_t kv_dim = n_kv * head_dim;
     ggml_tensor * h = codec_op_rms_norm_ct(ctx, x_ht, eps, W(".ln1.w"));
-    ggml_tensor * q  = ggml_reshape_3d(ctx, ggml_mul_mat(ctx, W(".attn_q.w"), h), head_dim, n_heads, 1);
-    ggml_tensor * kk = ggml_reshape_3d(ctx, ggml_mul_mat(ctx, W(".attn_k.w"), h), head_dim, n_kv, 1);
-    ggml_tensor * vv = ggml_reshape_3d(ctx, ggml_mul_mat(ctx, W(".attn_v.w"), h), head_dim, n_kv, 1);
+    ggml_tensor * q, * kk, * vv;
+    ggml_tensor * w_qkv = codec_graph_weight_mat(ctx, model, prefix + ".attn_qkv.w");
+    if (w_qkv != nullptr) {
+        ggml_tensor * qkv = ggml_mul_mat(ctx, w_qkv, h);   // (q_dim+2*kv_dim, 1)
+        q  = ggml_reshape_3d(ctx, ggml_cont(ctx, ggml_view_1d(ctx, qkv, q_dim,  0)), head_dim, n_heads, 1);
+        kk = ggml_reshape_3d(ctx, ggml_cont(ctx, ggml_view_1d(ctx, qkv, kv_dim, (size_t) q_dim * qkv->nb[0])), head_dim, n_kv, 1);
+        vv = ggml_reshape_3d(ctx, ggml_cont(ctx, ggml_view_1d(ctx, qkv, kv_dim, (size_t) (q_dim + kv_dim) * qkv->nb[0])), head_dim, n_kv, 1);
+    } else {
+        q  = ggml_reshape_3d(ctx, ggml_mul_mat(ctx, WM(".attn_q.w"), h), head_dim, n_heads, 1);
+        kk = ggml_reshape_3d(ctx, ggml_mul_mat(ctx, WM(".attn_k.w"), h), head_dim, n_kv, 1);
+        vv = ggml_reshape_3d(ctx, ggml_mul_mat(ctx, WM(".attn_v.w"), h), head_dim, n_kv, 1);
+    }
     *kk_out = kk;
     *vv_out = vv;
 
@@ -107,12 +122,22 @@ ggml_tensor * bm_ralm_kv_step(ggml_context * ctx, ggml_tensor * x_ht, const std:
     ggml_tensor * v_p = ggml_cont(ctx, ggml_permute(ctx, v_all, 1, 2, 0, 3));
     ggml_tensor * attn = ggml_cont(ctx, ggml_permute(ctx, ggml_mul_mat(ctx, v_p, scores), 0, 2, 1, 3));
     attn = ggml_reshape_2d(ctx, attn, q_dim, 1);
-    x_ht = ggml_add(ctx, x_ht, ggml_mul_mat(ctx, W(".attn_o.w"), attn));
+    x_ht = ggml_add(ctx, x_ht, ggml_mul_mat(ctx, WM(".attn_o.w"), attn));
 
     h = codec_op_rms_norm_ct(ctx, x_ht, eps, W(".ln2.w"));
-    ggml_tensor * g = ggml_silu(ctx, ggml_mul_mat(ctx, W(".gate.w"), h));
-    ggml_tensor * u = ggml_mul_mat(ctx, W(".up.w"), h);
-    return ggml_add(ctx, x_ht, ggml_mul_mat(ctx, W(".down.w"), ggml_mul(ctx, g, u)));
+    ggml_tensor * g, * u;
+    ggml_tensor * w_gu = codec_graph_weight_mat(ctx, model, prefix + ".gate_up.w");
+    if (w_gu != nullptr) {
+        ggml_tensor * gu = ggml_mul_mat(ctx, w_gu, h);   // (2*ffn, 1)
+        const int64_t ffn = gu->ne[0] / 2;
+        g = ggml_view_1d(ctx, gu, ffn, 0);
+        u = ggml_view_1d(ctx, gu, ffn, (size_t) ffn * gu->nb[0]);
+    } else {
+        g = ggml_mul_mat(ctx, WM(".gate.w"), h);
+        u = ggml_mul_mat(ctx, WM(".up.w"), h);
+    }
+    g = ggml_silu(ctx, g);
+    return ggml_add(ctx, x_ht, ggml_mul_mat(ctx, WM(".down.w"), ggml_mul(ctx, g, u)));
 }
 
 // Build the whole per-step graph.
@@ -138,8 +163,13 @@ bool build_step(ggml_context * ctx, void * ud, ggml_tensor ** out) {
     cfm_build * p = static_cast<cfm_build *>(ud);
     const cfm_impl & I = p->imp;
     const codec_model * model = p->model;
-    auto W = [&](const char * s) { return codec_graph_weight(ctx, model, s); };
+    auto W  = [&](const char * s) { return codec_graph_weight(ctx, model, s); };
+    auto WM = [&](const char * s) { return codec_graph_weight_mat(ctx, model, s); };
     const int32_t P = I.patch_size, D = I.latent_dim;
+    // lin() wrapper that fetches the weight via the no-dequant matmul accessor.
+    auto linW = [&](const char * wn, ggml_tensor * x, const char * bn) {
+        return lin(ctx, WM(wn), x, bn ? W(bn) : nullptr);
+    };
 
     ggml_tensor * h_in   = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, I.h_barbet, 1);  ggml_set_name(h_in, "bm.cfm.h_in");
     ggml_tensor * pfb_lm = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, I.h_vox, 1);      ggml_set_name(pfb_lm, "bm.cfm.pfb_lm");
@@ -150,21 +180,21 @@ bool build_step(ggml_context * ctx, void * ud, ggml_tensor ** out) {
 
     // ---- tslm_adapter(h_in) → h_vox ----
     ggml_tensor * a = codec_op_rms_norm_ct(ctx, h_in, I.eps, W("lm.tslm_adapter.norm.w"));
-    a = lin(ctx, W("lm.tslm_adapter.proj.w"), a, W("lm.tslm_adapter.proj.b"));   // (h_vox,1)
+    a = linW("lm.tslm_adapter.proj.w", a, "lm.tslm_adapter.proj.b");   // (h_vox,1)
     {   // residual SwiGLU block
         ggml_tensor * bn = codec_op_rms_norm_ct(ctx, a, I.eps, W("lm.tslm_adapter.blk0.ln.w"));
-        ggml_tensor * g = ggml_silu(ctx, lin(ctx, W("lm.tslm_adapter.blk0.gate.w"), bn, nullptr));
-        ggml_tensor * u = lin(ctx, W("lm.tslm_adapter.blk0.up.w"), bn, nullptr);
-        a = ggml_add(ctx, a, lin(ctx, W("lm.tslm_adapter.blk0.down.w"), ggml_mul(ctx, g, u), nullptr));
+        ggml_tensor * g = ggml_silu(ctx, linW("lm.tslm_adapter.blk0.gate.w", bn, nullptr));
+        ggml_tensor * u = linW("lm.tslm_adapter.blk0.up.w", bn, nullptr);
+        a = ggml_add(ctx, a, linW("lm.tslm_adapter.blk0.down.w", ggml_mul(ctx, g, u), nullptr));
     }
     // ---- FSQ: round(tanh(in_proj(a))*s)/s then out_proj ----
-    ggml_tensor * q = ggml_tanh(ctx, lin(ctx, W("lm.fsq.in_proj.w"), a, W("lm.fsq.in_proj.b")));
+    ggml_tensor * q = ggml_tanh(ctx, linW("lm.fsq.in_proj.w", a, "lm.fsq.in_proj.b"));
     q = ggml_scale(ctx, ggml_round(ctx, ggml_scale(ctx, q, (float) I.fsq_scale)), 1.0f / (float) I.fsq_scale);
-    ggml_tensor * lm_hidden = lin(ctx, W("lm.fsq.out_proj.w"), q, W("lm.fsq.out_proj.b"));   // (h_vox,1)
+    ggml_tensor * lm_hidden = linW("lm.fsq.out_proj.w", q, "lm.fsq.out_proj.b");   // (h_vox,1)
 
     // ---- RALM input = fusion_concat_proj([lm_hidden ; prev_feedback_lm]) ----
     ggml_tensor * fus = ggml_concat(ctx, lm_hidden, pfb_lm, 0);                                // (2*h_vox,1)
-    ggml_tensor * ralm_new = lin(ctx, W("lm.proj.fusion_concat.w"), fus, W("lm.proj.fusion_concat.b")); // (h_vox,1)
+    ggml_tensor * ralm_new = linW("lm.proj.fusion_concat.w", fus, "lm.proj.fusion_concat.b"); // (h_vox,1)
 
     // ---- RALM: one incremental step over the persistent KV cache ----
     // Shared additive attention mask for all RALM layers.  scores has ne0 =
@@ -190,16 +220,16 @@ bool build_step(ggml_context * ctx, void * ud, ggml_tensor ** out) {
     ggml_tensor * residual_hidden = codec_op_rms_norm_ct(ctx, rh, I.eps, W("lm.ralm.norm.w"));  // (h_vox,1)
 
     // ---- mu = [lm_to_dit(lm_hidden) ; res_to_dit(residual_hidden)] → (h_dit, n_mu) ----
-    ggml_tensor * mu1 = lin(ctx, W("lm.proj.lm_to_dit.w"), lm_hidden, W("lm.proj.lm_to_dit.b"));
-    ggml_tensor * mu2 = lin(ctx, W("lm.proj.res_to_dit.w"), residual_hidden, W("lm.proj.res_to_dit.b"));
+    ggml_tensor * mu1 = linW("lm.proj.lm_to_dit.w", lm_hidden, "lm.proj.lm_to_dit.b");
+    ggml_tensor * mu2 = linW("lm.proj.res_to_dit.w", residual_hidden, "lm.proj.res_to_dit.b");
     ggml_tensor * mu = ggml_concat(ctx, mu1, mu2, 1);   // (h_dit, 2)
 
     // ---- CFM Euler solver (unrolled, cfg_zero_star) ----
-    ggml_tensor * cond_h = lin(ctx, W("lm.locdit.cond_proj.w"), cond, W("lm.locdit.cond_proj.b"));  // (h_dit,P)
+    ggml_tensor * cond_h = linW("lm.locdit.cond_proj.w", cond, "lm.locdit.cond_proj.b");  // (h_dit,P)
     ggml_tensor * mu_zero = ggml_scale(ctx, mu, 0.0f);
     auto time_mlp = [&](const char * pfx, ggml_tensor * s) {
-        ggml_tensor * h = ggml_silu(ctx, lin(ctx, W((std::string(pfx) + ".l1.w").c_str()), s, W((std::string(pfx) + ".l1.b").c_str())));
-        return lin(ctx, W((std::string(pfx) + ".l2.w").c_str()), h, W((std::string(pfx) + ".l2.b").c_str()));
+        ggml_tensor * h = ggml_silu(ctx, lin(ctx, WM((std::string(pfx) + ".l1.w").c_str()), s, W((std::string(pfx) + ".l1.b").c_str())));
+        return lin(ctx, WM((std::string(pfx) + ".l2.w").c_str()), h, W((std::string(pfx) + ".l2.b").c_str()));
     };
     ggml_tensor * dt_emb = time_mlp("lm.locdit.dtime_mlp", dtsin);
     const int64_t T = (int64_t) I.n_mu + 1 + 2 * P;
@@ -209,7 +239,7 @@ bool build_step(ggml_context * ctx, void * ud, ggml_tensor ** out) {
     const bool cfg_one = (p->cfg_value == 1.0f);
     ggml_tensor * x = z;
     for (int32_t s = 0; s < p->n_real; ++s) {
-        ggml_tensor * x_h = lin(ctx, W("lm.locdit.in_proj.w"), x, W("lm.locdit.in_proj.b"));
+        ggml_tensor * x_h = linW("lm.locdit.in_proj.w", x, "lm.locdit.in_proj.b");
         ggml_tensor * tsin_s = ggml_cont(ctx, ggml_view_2d(ctx, tsin, I.h_dit, 1, tsin->nb[1], (size_t) s * tsin->nb[1]));
         ggml_tensor * t_h = ggml_add(ctx, time_mlp("lm.locdit.time_mlp", tsin_s), dt_emb);
         ggml_tensor * dphi;
@@ -234,13 +264,13 @@ bool build_step(ggml_context * ctx, void * ud, ggml_tensor ** out) {
     ggml_set_output(x);
 
     // ---- stop head ----
-    ggml_tensor * sp = ggml_silu(ctx, lin(ctx, W("lm.stop.proj.w"), lm_hidden, W("lm.stop.proj.b")));
-    ggml_tensor * stop_logit = lin(ctx, W("lm.stop.head.w"), sp, nullptr);   // (2,1)
+    ggml_tensor * sp = ggml_silu(ctx, linW("lm.stop.proj.w", lm_hidden, "lm.stop.proj.b"));
+    ggml_tensor * stop_logit = linW("lm.stop.head.w", sp, nullptr);   // (2,1)
     ggml_set_name(stop_logit, "bm.cfm.stop");
     ggml_set_output(stop_logit);
 
     // ---- LocEnc(patch) → feedback ----
-    ggml_tensor * le = lin(ctx, W("lm.locenc.in_proj.w"), x, W("lm.locenc.in_proj.b"));   // (h_enc, P)
+    ggml_tensor * le = linW("lm.locenc.in_proj.w", x, "lm.locenc.in_proj.b");   // (h_enc, P)
     ggml_tensor * sptok = ggml_reshape_2d(ctx, codec_graph_cast_f32(ctx, W("lm.locenc.special_token")), I.h_enc, 1);
     le = ggml_concat(ctx, sptok, le, 1);                                                   // (h_enc, P+1)
     const int64_t Te = le->ne[1];
@@ -252,10 +282,10 @@ bool build_step(ggml_context * ctx, void * ud, ggml_tensor ** out) {
     }
     le = codec_op_rms_norm_ct(ctx, le, I.eps, W("lm.locenc.norm.w"));
     ggml_tensor * cls = ggml_cont(ctx, ggml_view_1d(ctx, le, I.h_enc, 0));                 // (h_enc,)
-    ggml_tensor * fb_tslm = lin(ctx, W("lm.proj.enc_to_tslm.w"), cls, W("lm.proj.enc_to_tslm.b"));  // (h_barbet,)
+    ggml_tensor * fb_tslm = linW("lm.proj.enc_to_tslm.w", cls, "lm.proj.enc_to_tslm.b");  // (h_barbet,)
     ggml_set_name(fb_tslm, "bm.cfm.fb_tslm");
     ggml_set_output(fb_tslm);
-    ggml_tensor * fb_lm = lin(ctx, W("lm.proj.enc_to_lm.w"), cls, W("lm.proj.enc_to_lm.b"));        // (h_vox,)
+    ggml_tensor * fb_lm = linW("lm.proj.enc_to_lm.w", cls, "lm.proj.enc_to_lm.b");        // (h_vox,)
     ggml_set_name(fb_lm, "bm.cfm.fb_lm");
     ggml_set_output(fb_lm);