Skip to content

Commit c4ef3b2

Browse files
committed
Record checked sampling and prefetch waiter experiments
1 parent 2d94967 commit c4ef3b2

3 files changed

Lines changed: 14984 additions & 0 deletions

File tree

‎evidence/minicpm-vllm-seedtts/README.md‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,30 @@ verification SIM fell by about 0.224 and 0.336 points at C16/C32. These
196196
descriptive scores do not establish non-inferiority. No service PASS follows
197197
from the component parity or development latency change.
198198

199+
## Checked sampling and idle-prefetch follow-up
200+
201+
`results/service-phase13-development.json` retains 18 further tuning cells.
202+
The sampler now computes finite/nonnegative/positive-mass validation alongside
203+
argmax and retains a device assertion. Sparse and dense rows at widths 25,
204+
626 and 6562 preserve valid-row tokens and generator state in eager and graph
205+
checks. Invalid-row checks inspect rejection flags without deliberately
206+
poisoning a CUDA context. Existing regressions: 244 passed, 27 skipped.
207+
The experimental patch applies after the asynchronous-history candidate.
208+
209+
| Mean request RTF, two tuning repetitions | C1 | C16 | C32 |
210+
| --- | ---: | ---: | ---: |
211+
| Checked sampler | 0.06066 | 0.23625 | 0.31924 |
212+
| Same sampler, idle-prefetch waiter limit 32 | 0.06051 | 0.24031 | 0.32161 |
213+
| Sequential paired engine | 0.06226 | 0.21039 | 0.26523 |
214+
215+
The larger waiter limit was delivered: both Code2Wav replicas logged a
216+
prefetch phase with multiple waiting references. It did not improve the
217+
objective and is not retained for subsequent candidates. The checked sampler
218+
still misses C16/C32, and its C16 variation prevents a stable benefit claim.
219+
All cells completed 64 requests without failures. Saved-audio scores are
220+
retained as descriptive tuning evidence, not speech-quality qualification.
221+
The overall verdict remains **INCONCLUSIVE**.
222+
199223
## Lessons supported by this investigation
200224

201225
- Match component precision to the running service. The HiFT diagnostic in
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
--- /dev/null
2+
+++ b/vllm_omni/model_executor/models/minicpmo_4_5/talker_cuda_sample.py
3+
@@ -0,0 +1,33 @@
4+
+# SPDX-License-Identifier: Apache-2.0
5+
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
6+
+"""Experimental CUDA single-row sampler retaining invalid-probability checks."""
7+
+import torch
8+
+import triton
9+
+import triton.language as tl
10+
+
11+
+
12+
+@triton.jit
13+
+def _validate_argmax(probabilities, ratios, output, valid, N: tl.constexpr, BLOCK: tl.constexpr):
14+
+ offsets = tl.arange(0, BLOCK)
15+
+ p = tl.load(probabilities + offsets, offsets < N, other=0.0)
16+
+ r = tl.load(ratios + offsets, offsets < N, other=-float('inf'))
17+
+ finite_nonnegative = (p >= 0.0) & (p < float('inf'))
18+
+ legal = (tl.sum(finite_nonnegative.to(tl.int32), 0) == BLOCK) & (tl.sum(p, 0) > 0.0)
19+
+ token = tl.argmax(r, 0, tie_break_left=True)
20+
+ tl.store(output, token.to(tl.int64))
21+
+ tl.store(valid, legal)
22+
+
23+
+
24+
+def sample(probabilities, generator):
25+
+ if (not probabilities.is_cuda or probabilities.dtype != torch.float32
26+
+ or probabilities.ndim != 2 or probabilities.shape[0] != 1
27+
+ or not probabilities.is_contiguous() or probabilities.shape[1] > 8192):
28+
+ return torch.multinomial(probabilities, 1, generator=generator).reshape(1)
29+
+ noise = torch.empty_like(probabilities).exponential_(1, generator=generator)
30+
+ ratios = probabilities / noise
31+
+ output = torch.empty(1, dtype=torch.long, device=probabilities.device)
32+
+ valid = torch.empty(1, dtype=torch.bool, device=probabilities.device)
33+
+ width = probabilities.shape[1]
34+
+ _validate_argmax[(1,)](probabilities, ratios, output, valid, width, triton.next_power_of_2(width))
35+
+ torch._assert_async(valid, 'probability tensor contains inf, nan or a negative value, or has zero mass')
36+
+ return output
37+
--- a/vllm_omni/model_executor/models/minicpmo_4_5/talker_codec_sample.py
38+
+++ b/vllm_omni/model_executor/models/minicpmo_4_5/talker_codec_sample.py
39+
@@ -454,11 +454,14 @@
40+
for row, (config, generator) in enumerate(zip(sampling, generators)):
41+
temperature = config[0]
42+
scores = filtered[row]
43+
- token = (
44+
- torch.multinomial(scores.softmax(-1), 1, generator=generator).reshape(1)
45+
- if temperature
46+
- else scores.argmax(dim=-1)
47+
- )
48+
+ if temperature and scores.is_cuda and _HAS_TRITON:
49+
+ from .talker_cuda_sample import sample
50+
+
51+
+ token = sample(scores.softmax(-1), generator)
52+
+ elif temperature:
53+
+ token = torch.multinomial(scores.softmax(-1), 1, generator=generator).reshape(1)
54+
+ else:
55+
+ token = scores.argmax(dim=-1)
56+
sampled.append(token)
57+
return self.advance(torch.cat(sampled), frame, eos_token_id)
58+
59+
--- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py
60+
+++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py
61+
@@ -1690,7 +1690,12 @@
62+
),
63+
_stage_matmul_policy(self._tf32_mode),
64+
):
65+
- return self._run_prefetch_phase(state_id, record)
66+
+ queued_before = len(self._prefetch_queue)
67+
+ ran = self._run_prefetch_phase(state_id, record)
68+
+ if ran and queued_before > 1 and not getattr(self, "_rsi_multiqueue_prefetch_logged", False):
69+
+ logger.info("[RSI prefetch treatment] executed with %d waiting references", queued_before)
70+
+ self._rsi_multiqueue_prefetch_logged = True
71+
+ return ran
72+
73+
def _run_prefetch_phase(self, state_id: str, record: _RefPrefetch) -> bool:
74+
"""Run the next phase of ``record``; see ``run_idle_prefetch``."""

0 commit comments

Comments
 (0)