|
| 1 | +--- /dev/null |
| 2 | ++++ b/vllm_omni/model_executor/models/minicpmo_4_5/talker_cuda_sample.py |
| 3 | +@@ -0,0 +1,33 @@ |
| 4 | ++# SPDX-License-Identifier: Apache-2.0 |
| 5 | ++# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project |
| 6 | ++"""Experimental CUDA single-row sampler retaining invalid-probability checks.""" |
| 7 | ++import torch |
| 8 | ++import triton |
| 9 | ++import triton.language as tl |
| 10 | ++ |
| 11 | ++ |
| 12 | ++@triton.jit |
| 13 | ++def _validate_argmax(probabilities, ratios, output, valid, N: tl.constexpr, BLOCK: tl.constexpr): |
| 14 | ++ offsets = tl.arange(0, BLOCK) |
| 15 | ++ p = tl.load(probabilities + offsets, offsets < N, other=0.0) |
| 16 | ++ r = tl.load(ratios + offsets, offsets < N, other=-float('inf')) |
| 17 | ++ finite_nonnegative = (p >= 0.0) & (p < float('inf')) |
| 18 | ++ legal = (tl.sum(finite_nonnegative.to(tl.int32), 0) == BLOCK) & (tl.sum(p, 0) > 0.0) |
| 19 | ++ token = tl.argmax(r, 0, tie_break_left=True) |
| 20 | ++ tl.store(output, token.to(tl.int64)) |
| 21 | ++ tl.store(valid, legal) |
| 22 | ++ |
| 23 | ++ |
| 24 | ++def sample(probabilities, generator): |
| 25 | ++ if (not probabilities.is_cuda or probabilities.dtype != torch.float32 |
| 26 | ++ or probabilities.ndim != 2 or probabilities.shape[0] != 1 |
| 27 | ++ or not probabilities.is_contiguous() or probabilities.shape[1] > 8192): |
| 28 | ++ return torch.multinomial(probabilities, 1, generator=generator).reshape(1) |
| 29 | ++ noise = torch.empty_like(probabilities).exponential_(1, generator=generator) |
| 30 | ++ ratios = probabilities / noise |
| 31 | ++ output = torch.empty(1, dtype=torch.long, device=probabilities.device) |
| 32 | ++ valid = torch.empty(1, dtype=torch.bool, device=probabilities.device) |
| 33 | ++ width = probabilities.shape[1] |
| 34 | ++ _validate_argmax[(1,)](probabilities, ratios, output, valid, width, triton.next_power_of_2(width)) |
| 35 | ++ torch._assert_async(valid, 'probability tensor contains inf, nan or a negative value, or has zero mass') |
| 36 | ++ return output |
| 37 | +--- a/vllm_omni/model_executor/models/minicpmo_4_5/talker_codec_sample.py |
| 38 | ++++ b/vllm_omni/model_executor/models/minicpmo_4_5/talker_codec_sample.py |
| 39 | +@@ -454,11 +454,14 @@ |
| 40 | + for row, (config, generator) in enumerate(zip(sampling, generators)): |
| 41 | + temperature = config[0] |
| 42 | + scores = filtered[row] |
| 43 | +- token = ( |
| 44 | +- torch.multinomial(scores.softmax(-1), 1, generator=generator).reshape(1) |
| 45 | +- if temperature |
| 46 | +- else scores.argmax(dim=-1) |
| 47 | +- ) |
| 48 | ++ if temperature and scores.is_cuda and _HAS_TRITON: |
| 49 | ++ from .talker_cuda_sample import sample |
| 50 | ++ |
| 51 | ++ token = sample(scores.softmax(-1), generator) |
| 52 | ++ elif temperature: |
| 53 | ++ token = torch.multinomial(scores.softmax(-1), 1, generator=generator).reshape(1) |
| 54 | ++ else: |
| 55 | ++ token = scores.argmax(dim=-1) |
| 56 | + sampled.append(token) |
| 57 | + return self.advance(torch.cat(sampled), frame, eos_token_id) |
| 58 | + |
| 59 | +--- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py |
| 60 | ++++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py |
| 61 | +@@ -1690,7 +1690,12 @@ |
| 62 | + ), |
| 63 | + _stage_matmul_policy(self._tf32_mode), |
| 64 | + ): |
| 65 | +- return self._run_prefetch_phase(state_id, record) |
| 66 | ++ queued_before = len(self._prefetch_queue) |
| 67 | ++ ran = self._run_prefetch_phase(state_id, record) |
| 68 | ++ if ran and queued_before > 1 and not getattr(self, "_rsi_multiqueue_prefetch_logged", False): |
| 69 | ++ logger.info("[RSI prefetch treatment] executed with %d waiting references", queued_before) |
| 70 | ++ self._rsi_multiqueue_prefetch_logged = True |
| 71 | ++ return ran |
| 72 | + |
| 73 | + def _run_prefetch_phase(self, state_id: str, record: _RefPrefetch) -> bool: |
| 74 | + """Run the next phase of ``record``; see ``run_idle_prefetch``.""" |
0 commit comments