Skip to content

Commit cbc2ded

Browse files
committed
Record background reference preparation regression
1 parent c4ef3b2 commit cbc2ded

4 files changed

Lines changed: 16116 additions & 0 deletions

File tree

‎evidence/minicpm-vllm-seedtts/README.md‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,39 @@ All cells completed 64 requests without failures. Saved-audio scores are
220220
retained as descriptive tuning evidence, not speech-quality qualification.
221221
The overall verdict remains **INCONCLUSIVE**.
222222

223+
## Background CPU reference preparation
224+
225+
`results/service-phase14-development.json` records 18 tuning cells from the
226+
same source snapshot with zero or four background CPU feature workers, followed
227+
by a sequential engine reference on the same reserved GPU. Both replicas logged
228+
submission and consumption of background features. The queue is bounded at 32;
229+
reference files stay pinned until their readers finish. GPU feature completion
230+
remains on the service thread. This is an experimental patch layered on the
231+
checked-sampler snapshot, not a production change.
232+
233+
| Mean request RTF, two tuning repetitions | C1 | C16 | C32 |
234+
| --- | ---: | ---: | ---: |
235+
| CPU prefetch disabled | 0.06070 | 0.23153 | 0.33311 |
236+
| Four background CPU workers | 0.06166 | 0.39736 | 0.33687 |
237+
| Sequential paired engine | 0.06424 | 0.21633 | 0.26031 |
238+
239+
All cells completed 64 requests without failures. CPU features matched exactly
240+
on 16 real references, and complete real-weight prompt features matched on four.
241+
Regressions and lifecycle checks: 247 passed, 27 skipped, including cancellation,
242+
pinning and bounded queues. An isolated CPU diagnostic found roughly 0.048 s for
243+
16 references with four workers versus 0.17 s with one, with one PyTorch thread
244+
per worker in that diagnostic. This local speedup did not translate to the
245+
service. C16 audio chunk gaps increased; graph logs show no cache flushes in
246+
either variant. These observations do not establish the cause of the regression.
247+
The four-worker configuration is not retained.
248+
249+
C1 saved waveforms had identical lengths, with 54/64 PCM-exact pairs and a
250+
maximum sample difference of 0.000214. Two fresh control servers themselves
251+
had 55/64 exact pairs and a maximum difference of 0.000519. Component feature
252+
identity therefore is not presented as complete generated-waveform identity.
253+
Saved-audio quality scores remain descriptive; the service objective remains
254+
**INCONCLUSIVE**.
255+
223256
## Lessons supported by this investigation
224257

225258
- Match component precision to the running service. The HiFT diagnostic in
Lines changed: 231 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,231 @@
1+
--- a/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py
2+
+++ b/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py
3+
@@ -9,6 +9,7 @@
4+
import time
5+
from collections import OrderedDict
6+
from collections.abc import Mapping
7+
+from concurrent.futures import ThreadPoolExecutor
8+
from contextlib import nullcontext
9+
from dataclasses import dataclass
10+
from typing import Any, cast
11+
@@ -721,6 +722,31 @@
12+
next_mel = torch.cat((old_mel[..., -(keep - chunk_mel.shape[2]) :], chunk_mel), dim=2)
13+
return speech, source, next_mel.detach()
14+
15+
+ def prefetch_prompt_cpu(self, prompt_cache_id: str, prompt_wav: str, workers: int) -> bool:
16+
+ """Queue bounded CPU extraction; accelerator work stays on the caller."""
17+
+ if workers <= 0 or torch.get_default_dtype() != torch.float32:
18+
+ return False
19+
+ key = (prompt_cache_id, prompt_wav)
20+
+ if key in self._prompt_features:
21+
+ return False
22+
+ if not hasattr(self, "_cpu_prompt_futures"):
23+
+ self._cpu_prompt_futures = {}
24+
+ self._cpu_prompt_pool = ThreadPoolExecutor(max_workers=min(workers, 32))
25+
+ if key in self._cpu_prompt_futures or len(self._cpu_prompt_futures) >= 32:
26+
+ return False
27+
+
28+
+ def extract():
29+
+ with torch.inference_mode():
30+
+ if torch.get_default_dtype() != torch.float32:
31+
+ return None
32+
+ return self._token2wav._prepare_prompt_cpu(prompt_wav)
33+
+
34+
+ self._cpu_prompt_futures[key] = self._cpu_prompt_pool.submit(extract)
35+
+ if not getattr(self, "_cpu_prefetch_submitted_logged", False):
36+
+ logger.info("[RSI CPU prefetch] queued reference features with %d workers", min(workers, 32))
37+
+ self._cpu_prefetch_submitted_logged = True
38+
+ return True
39+
+
40+
def prepare_prompt(self, prompt_cache_id: str, prompt_wav: str) -> PromptFeatures:
41+
cache_key = (prompt_cache_id, prompt_wav)
42+
cached = self._prompt_features.get(cache_key)
43+
@@ -732,7 +758,15 @@
44+
try:
45+
torch.set_default_dtype(torch.float32)
46+
with _autocast_disabled(self.speech_window.device):
47+
- values = self._token2wav._prepare_prompt(prompt_wav)
48+
+ pending = getattr(self, "_cpu_prompt_futures", {}).pop(cache_key, None)
49+
+ cpu_features = pending.result() if pending is not None else None
50+
+ if cpu_features is None:
51+
+ values = self._token2wav._prepare_prompt(prompt_wav)
52+
+ else:
53+
+ values = self._token2wav._prepare_prompt(prompt_wav, cpu_features=cpu_features)
54+
+ if not getattr(self, "_cpu_prefetch_consumed_logged", False):
55+
+ logger.info("[RSI CPU prefetch] consumed background reference features")
56+
+ self._cpu_prefetch_consumed_logged = True
57+
finally:
58+
torch.set_default_dtype(previous_dtype)
59+
cached = PromptFeatures(
60+
@@ -747,6 +781,13 @@
61+
def evict_prompt(self, prompt_cache_id: str, prompt_wav: str) -> None:
62+
"""Release all cached artifacts associated with one prompt."""
63+
prompt_key = (prompt_cache_id, prompt_wav)
64+
+ pending = getattr(self, "_cpu_prompt_futures", {}).pop(prompt_key, None)
65+
+ if pending is not None and not pending.cancel():
66+
+ # The caller may unlink the WAV immediately after eviction.
67+
+ try:
68+
+ pending.result()
69+
+ except Exception:
70+
+ pass # An abandoned request has no consumer for extraction errors.
71+
self._prompt_features.pop(prompt_key, None)
72+
for setup_key in list(self._setup_cache):
73+
if setup_key[0] == prompt_key:
74+
--- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py
75+
+++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py
76+
@@ -424,6 +424,9 @@
77+
# competes with the thinker's decode on a shared GPU.
78+
self._ref_prefetch_enabled = read_bool_flag(extra, "token2wav_ref_prefetch", True)
79+
self._ref_prefetch_setup = read_bool_flag(extra, "token2wav_ref_prefetch_setup", False)
80+
+ self._ref_cpu_prefetch_workers = int(extra.get("token2wav_ref_cpu_prefetch_workers", 0))
81+
+ if self._ref_cpu_prefetch_workers < 0:
82+
+ raise ValueError("token2wav_ref_cpu_prefetch_workers must be >= 0")
83+
self._ref_prefetch_max_waiters = int(extra.get("token2wav_ref_prefetch_max_waiters", 1))
84+
if self._ref_prefetch_max_waiters < 1:
85+
raise ValueError("token2wav_ref_prefetch_max_waiters must be >= 1")
86+
@@ -1645,6 +1648,18 @@
87+
)
88+
continue
89+
self._prefetch_queue[state_id] = _RefPrefetch(ref_audio=ref_audio, sample_rate=sample_rate)
90+
+ workers = getattr(self, "_ref_cpu_prefetch_workers", 0)
91+
+ if workers and ref_audio.device.type == "cpu":
92+
+ try:
93+
+ cache_key, entry = self._materialize_runtime_prompt(ref_audio, sample_rate)
94+
+ entry.pins.add(state_id)
95+
+ self._prefetch_pins[state_id] = cache_key
96+
+ self.backend.prefetch_prompt_cpu(entry.cache_id, entry.path, workers)
97+
+ self._trim_runtime_prompts()
98+
+ except Exception:
99+
+ logger.exception("MiniCPM-o CPU reference prefetch failed for %s", state_id)
100+
+ self._drop_prefetch(state_id)
101+
+
102+
103+
@torch.inference_mode()
104+
def run_idle_prefetch(self) -> bool:
105+
--- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_token2wav.py
106+
+++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_token2wav.py
107+
@@ -119,9 +119,12 @@
108+
def speech_window(self) -> torch.Tensor:
109+
return self._core.speech_window
110+
111+
- def _prepare_prompt(self, prompt_wav: str):
112+
+ def _prepare_prompt_cpu(self, prompt_wav: str):
113+
+ return self._core._prepare_prompt_cpu(prompt_wav)
114+
+
115+
+ def _prepare_prompt(self, prompt_wav: str, *, cpu_features=None):
116+
"""Delegate prompt feature extraction to the wrapped core."""
117+
- return self._core._prepare_prompt(prompt_wav)
118+
+ return self._core._prepare_prompt(prompt_wav, cpu_features=cpu_features)
119+
120+
def enable_trt_spk_embedding(self, *, cpu_reference: bool = False) -> None:
121+
"""Run the campplus speaker-embedding model on TensorRT."""
122+
--- a/vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py
123+
+++ b/vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py
124+
@@ -514,8 +514,9 @@
125+
)
126+
return code, code_len
127+
128+
- def _prepare_prompt(self, prompt_wav: str):
129+
- """Prepare prompt audio for conditioning"""
130+
+ @staticmethod
131+
+ def _prepare_prompt_cpu(prompt_wav: str):
132+
+ """Extract reference features without accessing the accelerator."""
133+
audio_np_raw, sample_rate = load_audio(prompt_wav, sr=None, mono=True)
134+
sample_rate = int(sample_rate)
135+
136+
@@ -524,29 +525,8 @@
137+
audio = torch.from_numpy(audio_16k)
138+
mels = s3tokenizer.log_mel_spectrogram(audio)
139+
mels, mels_lens = s3tokenizer.padding([mels])
140+
- mels_d = _h2d_async(mels, self.device)
141+
- mels_lens_d = _h2d_async(mels_lens, self.device)
142+
- if mels.shape[-1] > _S3_MAX_FRAMES:
143+
- # Long audio (>30 s) keeps the library call: it routes to the
144+
- # sliding-window mixed-batch path.
145+
- prompt_speech_tokens, prompt_speech_tokens_lens = self.audio_tokenizer.quantize(mels_d, mels_lens_d)
146+
- else:
147+
- # Short audio: NPU graph replay of the library's else-branch
148+
- # (encoder + quantizer.encode). The device-side
149+
- # `mel_len > max_frames` check is skipped — its `.any()` forces an
150+
- # implicit D2H sync; the frame count is static on host
151+
- # (mels.shape[-1]), so the branch decision needs no sync.
152+
- prompt_speech_tokens, prompt_speech_tokens_lens = self._quantize_s3_graph(mels_d, mels_lens_d)
153+
-
154+
spk_feat = kaldi.fbank(audio.unsqueeze(0), num_mel_bins=80, dither=0, sample_frequency=16000)
155+
spk_feat = spk_feat - spk_feat.mean(dim=0, keepdim=True)
156+
- spk_emb = self._forward_spk_embedding(spk_feat)
157+
- if self._spk_event is not None:
158+
- # Device-side join for the private campplus stream (never blocks the
159+
- # host); it lands here so the host-side 24 kHz mel work overlaps.
160+
- torch.npu.current_stream(self.device).wait_event(self._spk_event)
161+
- self._spk_event = None
162+
-
163+
# 24 kHz branch: mel spectrogram for flow model conditioning
164+
# Must resample from the ORIGINAL audio, not the 16 kHz version.
165+
audio_24k = AudioResampler(target_sr=24000).resample(audio_np_raw.astype(np.float32), orig_sr=sample_rate)
166+
@@ -566,6 +546,65 @@
167+
.transpose(1, 2)
168+
.squeeze(0)
169+
) # [T, num_mels]
170+
+ return mels, mels_lens, spk_feat, prompt_mel
171+
+
172+
+ def _prepare_prompt(self, prompt_wav: str, *, cpu_features=None):
173+
+ """Prepare prompt audio for conditioning"""
174+
+ if cpu_features is None:
175+
+ audio_np_raw, sample_rate = load_audio(prompt_wav, sr=None, mono=True)
176+
+ sample_rate = int(sample_rate)
177+
+
178+
+ # 16 kHz branch: s3tokenizer + speaker embedding
179+
+ audio_16k = AudioResampler(target_sr=16000).resample(audio_np_raw.astype(np.float32), orig_sr=sample_rate)
180+
+ audio = torch.from_numpy(audio_16k)
181+
+ mels = s3tokenizer.log_mel_spectrogram(audio)
182+
+ mels, mels_lens = s3tokenizer.padding([mels])
183+
+ else:
184+
+ mels, mels_lens, spk_feat, prompt_mel = cpu_features
185+
+ mels_d = _h2d_async(mels, self.device)
186+
+ mels_lens_d = _h2d_async(mels_lens, self.device)
187+
+ if mels.shape[-1] > _S3_MAX_FRAMES:
188+
+ # Long audio (>30 s) keeps the library call: it routes to the
189+
+ # sliding-window mixed-batch path.
190+
+ prompt_speech_tokens, prompt_speech_tokens_lens = self.audio_tokenizer.quantize(mels_d, mels_lens_d)
191+
+ else:
192+
+ # Short audio: NPU graph replay of the library's else-branch
193+
+ # (encoder + quantizer.encode). The device-side
194+
+ # `mel_len > max_frames` check is skipped — its `.any()` forces an
195+
+ # implicit D2H sync; the frame count is static on host
196+
+ # (mels.shape[-1]), so the branch decision needs no sync.
197+
+ prompt_speech_tokens, prompt_speech_tokens_lens = self._quantize_s3_graph(mels_d, mels_lens_d)
198+
+
199+
+ if cpu_features is None:
200+
+ spk_feat = kaldi.fbank(audio.unsqueeze(0), num_mel_bins=80, dither=0, sample_frequency=16000)
201+
+ spk_feat = spk_feat - spk_feat.mean(dim=0, keepdim=True)
202+
+ spk_emb = self._forward_spk_embedding(spk_feat)
203+
+ if self._spk_event is not None:
204+
+ # Device-side join for the private campplus stream (never blocks the
205+
+ # host); it lands here so the host-side 24 kHz mel work overlaps.
206+
+ torch.npu.current_stream(self.device).wait_event(self._spk_event)
207+
+ self._spk_event = None
208+
+
209+
+ if cpu_features is None:
210+
+ # 24 kHz branch: mel spectrogram for flow model conditioning
211+
+ # Must resample from the ORIGINAL audio, not the 16 kHz version.
212+
+ audio_24k = AudioResampler(target_sr=24000).resample(audio_np_raw.astype(np.float32), orig_sr=sample_rate)
213+
+ audio_24k_t = torch.from_numpy(audio_24k).unsqueeze(0) # [1, T]
214+
+ prompt_mel = (
215+
+ mel_spectrogram(
216+
+ audio_24k_t,
217+
+ n_fft=1920,
218+
+ num_mels=80,
219+
+ sampling_rate=24000,
220+
+ hop_size=480,
221+
+ win_size=1920,
222+
+ fmin=0,
223+
+ fmax=8000,
224+
+ center=False,
225+
+ )
226+
+ .transpose(1, 2)
227+
+ .squeeze(0)
228+
+ ) # [T, num_mels]
229+
prompt_mels = _h2d_async(prompt_mel.unsqueeze(0), self.device)
230+
# Build on host then async-copy: torch.tensor(..., device=...) is itself
231+
# a synchronous H2D that blocks behind queued decode work.

0 commit comments

Comments
 (0)