|
| 1 | +--- a/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py |
| 2 | ++++ b/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py |
| 3 | +@@ -9,6 +9,7 @@ |
| 4 | + import time |
| 5 | + from collections import OrderedDict |
| 6 | + from collections.abc import Mapping |
| 7 | ++from concurrent.futures import ThreadPoolExecutor |
| 8 | + from contextlib import nullcontext |
| 9 | + from dataclasses import dataclass |
| 10 | + from typing import Any, cast |
| 11 | +@@ -721,6 +722,31 @@ |
| 12 | + next_mel = torch.cat((old_mel[..., -(keep - chunk_mel.shape[2]) :], chunk_mel), dim=2) |
| 13 | + return speech, source, next_mel.detach() |
| 14 | + |
| 15 | ++ def prefetch_prompt_cpu(self, prompt_cache_id: str, prompt_wav: str, workers: int) -> bool: |
| 16 | ++ """Queue bounded CPU extraction; accelerator work stays on the caller.""" |
| 17 | ++ if workers <= 0 or torch.get_default_dtype() != torch.float32: |
| 18 | ++ return False |
| 19 | ++ key = (prompt_cache_id, prompt_wav) |
| 20 | ++ if key in self._prompt_features: |
| 21 | ++ return False |
| 22 | ++ if not hasattr(self, "_cpu_prompt_futures"): |
| 23 | ++ self._cpu_prompt_futures = {} |
| 24 | ++ self._cpu_prompt_pool = ThreadPoolExecutor(max_workers=min(workers, 32)) |
| 25 | ++ if key in self._cpu_prompt_futures or len(self._cpu_prompt_futures) >= 32: |
| 26 | ++ return False |
| 27 | ++ |
| 28 | ++ def extract(): |
| 29 | ++ with torch.inference_mode(): |
| 30 | ++ if torch.get_default_dtype() != torch.float32: |
| 31 | ++ return None |
| 32 | ++ return self._token2wav._prepare_prompt_cpu(prompt_wav) |
| 33 | ++ |
| 34 | ++ self._cpu_prompt_futures[key] = self._cpu_prompt_pool.submit(extract) |
| 35 | ++ if not getattr(self, "_cpu_prefetch_submitted_logged", False): |
| 36 | ++ logger.info("[RSI CPU prefetch] queued reference features with %d workers", min(workers, 32)) |
| 37 | ++ self._cpu_prefetch_submitted_logged = True |
| 38 | ++ return True |
| 39 | ++ |
| 40 | + def prepare_prompt(self, prompt_cache_id: str, prompt_wav: str) -> PromptFeatures: |
| 41 | + cache_key = (prompt_cache_id, prompt_wav) |
| 42 | + cached = self._prompt_features.get(cache_key) |
| 43 | +@@ -732,7 +758,15 @@ |
| 44 | + try: |
| 45 | + torch.set_default_dtype(torch.float32) |
| 46 | + with _autocast_disabled(self.speech_window.device): |
| 47 | +- values = self._token2wav._prepare_prompt(prompt_wav) |
| 48 | ++ pending = getattr(self, "_cpu_prompt_futures", {}).pop(cache_key, None) |
| 49 | ++ cpu_features = pending.result() if pending is not None else None |
| 50 | ++ if cpu_features is None: |
| 51 | ++ values = self._token2wav._prepare_prompt(prompt_wav) |
| 52 | ++ else: |
| 53 | ++ values = self._token2wav._prepare_prompt(prompt_wav, cpu_features=cpu_features) |
| 54 | ++ if not getattr(self, "_cpu_prefetch_consumed_logged", False): |
| 55 | ++ logger.info("[RSI CPU prefetch] consumed background reference features") |
| 56 | ++ self._cpu_prefetch_consumed_logged = True |
| 57 | + finally: |
| 58 | + torch.set_default_dtype(previous_dtype) |
| 59 | + cached = PromptFeatures( |
| 60 | +@@ -747,6 +781,13 @@ |
| 61 | + def evict_prompt(self, prompt_cache_id: str, prompt_wav: str) -> None: |
| 62 | + """Release all cached artifacts associated with one prompt.""" |
| 63 | + prompt_key = (prompt_cache_id, prompt_wav) |
| 64 | ++ pending = getattr(self, "_cpu_prompt_futures", {}).pop(prompt_key, None) |
| 65 | ++ if pending is not None and not pending.cancel(): |
| 66 | ++ # The caller may unlink the WAV immediately after eviction. |
| 67 | ++ try: |
| 68 | ++ pending.result() |
| 69 | ++ except Exception: |
| 70 | ++ pass # An abandoned request has no consumer for extraction errors. |
| 71 | + self._prompt_features.pop(prompt_key, None) |
| 72 | + for setup_key in list(self._setup_cache): |
| 73 | + if setup_key[0] == prompt_key: |
| 74 | +--- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py |
| 75 | ++++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py |
| 76 | +@@ -424,6 +424,9 @@ |
| 77 | + # competes with the thinker's decode on a shared GPU. |
| 78 | + self._ref_prefetch_enabled = read_bool_flag(extra, "token2wav_ref_prefetch", True) |
| 79 | + self._ref_prefetch_setup = read_bool_flag(extra, "token2wav_ref_prefetch_setup", False) |
| 80 | ++ self._ref_cpu_prefetch_workers = int(extra.get("token2wav_ref_cpu_prefetch_workers", 0)) |
| 81 | ++ if self._ref_cpu_prefetch_workers < 0: |
| 82 | ++ raise ValueError("token2wav_ref_cpu_prefetch_workers must be >= 0") |
| 83 | + self._ref_prefetch_max_waiters = int(extra.get("token2wav_ref_prefetch_max_waiters", 1)) |
| 84 | + if self._ref_prefetch_max_waiters < 1: |
| 85 | + raise ValueError("token2wav_ref_prefetch_max_waiters must be >= 1") |
| 86 | +@@ -1645,6 +1648,18 @@ |
| 87 | + ) |
| 88 | + continue |
| 89 | + self._prefetch_queue[state_id] = _RefPrefetch(ref_audio=ref_audio, sample_rate=sample_rate) |
| 90 | ++ workers = getattr(self, "_ref_cpu_prefetch_workers", 0) |
| 91 | ++ if workers and ref_audio.device.type == "cpu": |
| 92 | ++ try: |
| 93 | ++ cache_key, entry = self._materialize_runtime_prompt(ref_audio, sample_rate) |
| 94 | ++ entry.pins.add(state_id) |
| 95 | ++ self._prefetch_pins[state_id] = cache_key |
| 96 | ++ self.backend.prefetch_prompt_cpu(entry.cache_id, entry.path, workers) |
| 97 | ++ self._trim_runtime_prompts() |
| 98 | ++ except Exception: |
| 99 | ++ logger.exception("MiniCPM-o CPU reference prefetch failed for %s", state_id) |
| 100 | ++ self._drop_prefetch(state_id) |
| 101 | ++ |
| 102 | + |
| 103 | + @torch.inference_mode() |
| 104 | + def run_idle_prefetch(self) -> bool: |
| 105 | +--- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_token2wav.py |
| 106 | ++++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_token2wav.py |
| 107 | +@@ -119,9 +119,12 @@ |
| 108 | + def speech_window(self) -> torch.Tensor: |
| 109 | + return self._core.speech_window |
| 110 | + |
| 111 | +- def _prepare_prompt(self, prompt_wav: str): |
| 112 | ++ def _prepare_prompt_cpu(self, prompt_wav: str): |
| 113 | ++ return self._core._prepare_prompt_cpu(prompt_wav) |
| 114 | ++ |
| 115 | ++ def _prepare_prompt(self, prompt_wav: str, *, cpu_features=None): |
| 116 | + """Delegate prompt feature extraction to the wrapped core.""" |
| 117 | +- return self._core._prepare_prompt(prompt_wav) |
| 118 | ++ return self._core._prepare_prompt(prompt_wav, cpu_features=cpu_features) |
| 119 | + |
| 120 | + def enable_trt_spk_embedding(self, *, cpu_reference: bool = False) -> None: |
| 121 | + """Run the campplus speaker-embedding model on TensorRT.""" |
| 122 | +--- a/vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py |
| 123 | ++++ b/vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py |
| 124 | +@@ -514,8 +514,9 @@ |
| 125 | + ) |
| 126 | + return code, code_len |
| 127 | + |
| 128 | +- def _prepare_prompt(self, prompt_wav: str): |
| 129 | +- """Prepare prompt audio for conditioning""" |
| 130 | ++ @staticmethod |
| 131 | ++ def _prepare_prompt_cpu(prompt_wav: str): |
| 132 | ++ """Extract reference features without accessing the accelerator.""" |
| 133 | + audio_np_raw, sample_rate = load_audio(prompt_wav, sr=None, mono=True) |
| 134 | + sample_rate = int(sample_rate) |
| 135 | + |
| 136 | +@@ -524,29 +525,8 @@ |
| 137 | + audio = torch.from_numpy(audio_16k) |
| 138 | + mels = s3tokenizer.log_mel_spectrogram(audio) |
| 139 | + mels, mels_lens = s3tokenizer.padding([mels]) |
| 140 | +- mels_d = _h2d_async(mels, self.device) |
| 141 | +- mels_lens_d = _h2d_async(mels_lens, self.device) |
| 142 | +- if mels.shape[-1] > _S3_MAX_FRAMES: |
| 143 | +- # Long audio (>30 s) keeps the library call: it routes to the |
| 144 | +- # sliding-window mixed-batch path. |
| 145 | +- prompt_speech_tokens, prompt_speech_tokens_lens = self.audio_tokenizer.quantize(mels_d, mels_lens_d) |
| 146 | +- else: |
| 147 | +- # Short audio: NPU graph replay of the library's else-branch |
| 148 | +- # (encoder + quantizer.encode). The device-side |
| 149 | +- # `mel_len > max_frames` check is skipped — its `.any()` forces an |
| 150 | +- # implicit D2H sync; the frame count is static on host |
| 151 | +- # (mels.shape[-1]), so the branch decision needs no sync. |
| 152 | +- prompt_speech_tokens, prompt_speech_tokens_lens = self._quantize_s3_graph(mels_d, mels_lens_d) |
| 153 | +- |
| 154 | + spk_feat = kaldi.fbank(audio.unsqueeze(0), num_mel_bins=80, dither=0, sample_frequency=16000) |
| 155 | + spk_feat = spk_feat - spk_feat.mean(dim=0, keepdim=True) |
| 156 | +- spk_emb = self._forward_spk_embedding(spk_feat) |
| 157 | +- if self._spk_event is not None: |
| 158 | +- # Device-side join for the private campplus stream (never blocks the |
| 159 | +- # host); it lands here so the host-side 24 kHz mel work overlaps. |
| 160 | +- torch.npu.current_stream(self.device).wait_event(self._spk_event) |
| 161 | +- self._spk_event = None |
| 162 | +- |
| 163 | + # 24 kHz branch: mel spectrogram for flow model conditioning |
| 164 | + # Must resample from the ORIGINAL audio, not the 16 kHz version. |
| 165 | + audio_24k = AudioResampler(target_sr=24000).resample(audio_np_raw.astype(np.float32), orig_sr=sample_rate) |
| 166 | +@@ -566,6 +546,65 @@ |
| 167 | + .transpose(1, 2) |
| 168 | + .squeeze(0) |
| 169 | + ) # [T, num_mels] |
| 170 | ++ return mels, mels_lens, spk_feat, prompt_mel |
| 171 | ++ |
| 172 | ++ def _prepare_prompt(self, prompt_wav: str, *, cpu_features=None): |
| 173 | ++ """Prepare prompt audio for conditioning""" |
| 174 | ++ if cpu_features is None: |
| 175 | ++ audio_np_raw, sample_rate = load_audio(prompt_wav, sr=None, mono=True) |
| 176 | ++ sample_rate = int(sample_rate) |
| 177 | ++ |
| 178 | ++ # 16 kHz branch: s3tokenizer + speaker embedding |
| 179 | ++ audio_16k = AudioResampler(target_sr=16000).resample(audio_np_raw.astype(np.float32), orig_sr=sample_rate) |
| 180 | ++ audio = torch.from_numpy(audio_16k) |
| 181 | ++ mels = s3tokenizer.log_mel_spectrogram(audio) |
| 182 | ++ mels, mels_lens = s3tokenizer.padding([mels]) |
| 183 | ++ else: |
| 184 | ++ mels, mels_lens, spk_feat, prompt_mel = cpu_features |
| 185 | ++ mels_d = _h2d_async(mels, self.device) |
| 186 | ++ mels_lens_d = _h2d_async(mels_lens, self.device) |
| 187 | ++ if mels.shape[-1] > _S3_MAX_FRAMES: |
| 188 | ++ # Long audio (>30 s) keeps the library call: it routes to the |
| 189 | ++ # sliding-window mixed-batch path. |
| 190 | ++ prompt_speech_tokens, prompt_speech_tokens_lens = self.audio_tokenizer.quantize(mels_d, mels_lens_d) |
| 191 | ++ else: |
| 192 | ++ # Short audio: NPU graph replay of the library's else-branch |
| 193 | ++ # (encoder + quantizer.encode). The device-side |
| 194 | ++ # `mel_len > max_frames` check is skipped — its `.any()` forces an |
| 195 | ++ # implicit D2H sync; the frame count is static on host |
| 196 | ++ # (mels.shape[-1]), so the branch decision needs no sync. |
| 197 | ++ prompt_speech_tokens, prompt_speech_tokens_lens = self._quantize_s3_graph(mels_d, mels_lens_d) |
| 198 | ++ |
| 199 | ++ if cpu_features is None: |
| 200 | ++ spk_feat = kaldi.fbank(audio.unsqueeze(0), num_mel_bins=80, dither=0, sample_frequency=16000) |
| 201 | ++ spk_feat = spk_feat - spk_feat.mean(dim=0, keepdim=True) |
| 202 | ++ spk_emb = self._forward_spk_embedding(spk_feat) |
| 203 | ++ if self._spk_event is not None: |
| 204 | ++ # Device-side join for the private campplus stream (never blocks the |
| 205 | ++ # host); it lands here so the host-side 24 kHz mel work overlaps. |
| 206 | ++ torch.npu.current_stream(self.device).wait_event(self._spk_event) |
| 207 | ++ self._spk_event = None |
| 208 | ++ |
| 209 | ++ if cpu_features is None: |
| 210 | ++ # 24 kHz branch: mel spectrogram for flow model conditioning |
| 211 | ++ # Must resample from the ORIGINAL audio, not the 16 kHz version. |
| 212 | ++ audio_24k = AudioResampler(target_sr=24000).resample(audio_np_raw.astype(np.float32), orig_sr=sample_rate) |
| 213 | ++ audio_24k_t = torch.from_numpy(audio_24k).unsqueeze(0) # [1, T] |
| 214 | ++ prompt_mel = ( |
| 215 | ++ mel_spectrogram( |
| 216 | ++ audio_24k_t, |
| 217 | ++ n_fft=1920, |
| 218 | ++ num_mels=80, |
| 219 | ++ sampling_rate=24000, |
| 220 | ++ hop_size=480, |
| 221 | ++ win_size=1920, |
| 222 | ++ fmin=0, |
| 223 | ++ fmax=8000, |
| 224 | ++ center=False, |
| 225 | ++ ) |
| 226 | ++ .transpose(1, 2) |
| 227 | ++ .squeeze(0) |
| 228 | ++ ) # [T, num_mels] |
| 229 | + prompt_mels = _h2d_async(prompt_mel.unsqueeze(0), self.device) |
| 230 | + # Build on host then async-copy: torch.tensor(..., device=...) is itself |
| 231 | + # a synchronous H2D that blocks behind queued decode work. |
0 commit comments