diff --git a/src/llm/language_model/continuous_batching/servable.cpp b/src/llm/language_model/continuous_batching/servable.cpp index 90a2342fcf..9734adf4d8 100644 --- a/src/llm/language_model/continuous_batching/servable.cpp +++ b/src/llm/language_model/continuous_batching/servable.cpp @@ -17,7 +17,6 @@ #include #include #include -#include #include #include "../../../logging.hpp" @@ -57,20 +56,6 @@ void ContinuousBatchingServable::logPerfMetrics(ov::genai::PerfMetrics& perfMetr prefillSpeedTps); } -// CB stepping thread writes metrics in _free_non_running_requests() slightly after -// pushing the final output. Yield briefly to close the race window. -// TODO: remove once GenAI's get_perf_metrics() blocks instead of asserting (fix in generation_stream.hpp) -static std::optional tryGetPerfMetrics(const ov::genai::GenerationHandle& handle) { - for (int i = 0; i < 1000; ++i) { - try { - return handle->get_perf_metrics(); - } catch (const ov::Exception&) { - std::this_thread::yield(); - } - } - return std::nullopt; -} - void ContinuousBatchingServable::notifyExecutorThread() { SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Notifying executor thread"); if (properties->llmExecutorWrapper == nullptr) { @@ -177,9 +162,8 @@ absl::Status ContinuousBatchingServable::prepareCompleteResponse(std::shared_ptr auto status = GenAiServable::prepareCompleteResponse(executionContext); if (status.ok() && llm_calculator_logger->should_log(spdlog::level::debug)) { auto cbExecutionContext = std::static_pointer_cast(executionContext); - auto perfMetrics = tryGetPerfMetrics(cbExecutionContext->generationHandle); - if (perfMetrics) - logPerfMetrics(*perfMetrics); + auto perfMetrics = cbExecutionContext->generationHandle->get_perf_metrics(); + logPerfMetrics(perfMetrics); } return status; } @@ -190,9 +174,8 @@ absl::Status ContinuousBatchingServable::preparePartialResponse(std::shared_ptr< !executionContext->sendLoopbackSignal && llm_calculator_logger->should_log(spdlog::level::debug)) { auto cbExecutionContext = std::static_pointer_cast(executionContext); - auto perfMetrics = tryGetPerfMetrics(cbExecutionContext->generationHandle); - if (perfMetrics) - logPerfMetrics(*perfMetrics); + auto perfMetrics = cbExecutionContext->generationHandle->get_perf_metrics(); + logPerfMetrics(perfMetrics); } return status; } diff --git a/src/llm/visual_language_model/continuous_batching/servable.cpp b/src/llm/visual_language_model/continuous_batching/servable.cpp index d459d2dba9..e3237f125b 100644 --- a/src/llm/visual_language_model/continuous_batching/servable.cpp +++ b/src/llm/visual_language_model/continuous_batching/servable.cpp @@ -19,7 +19,6 @@ #include #include #include -#include #include #include "src/port/rapidjson_document.hpp" @@ -36,21 +35,6 @@ namespace ovms { -// CB stepping thread writes base perf metrics in _free_non_running_requests() slightly -// after pushing the final output; get_vlm_perf_metrics() calls get_perf_metrics() internally. -// Yield briefly to close the race window. -// TODO: remove once GenAI's get_perf_metrics() blocks instead of asserting (fix in generation_stream.hpp) -static std::optional tryGetVlmPerfMetrics(const ov::genai::GenerationHandle& handle) { - for (int i = 0; i < 1000; ++i) { - try { - return handle->get_vlm_perf_metrics(); - } catch (const ov::Exception&) { - std::this_thread::yield(); - } - } - return std::nullopt; -} - void VisualLanguageModelServable::logPerfMetrics(ov::genai::VLMPerfMetrics& perfMetrics) { const size_t inputTokenCount = perfMetrics.get_num_input_tokens(); const size_t outputTokenCount = perfMetrics.get_num_generated_tokens(); @@ -100,9 +84,8 @@ absl::Status VisualLanguageModelServable::prepareCompleteResponse(std::shared_pt auto status = GenAiServable::prepareCompleteResponse(executionContext); if (status.ok() && llm_calculator_logger->should_log(spdlog::level::debug)) { auto vlmExecutionContext = std::static_pointer_cast(executionContext); - auto perfMetrics = tryGetVlmPerfMetrics(vlmExecutionContext->generationHandle); - if (perfMetrics) - logPerfMetrics(*perfMetrics); + auto perfMetrics = vlmExecutionContext->generationHandle->get_vlm_perf_metrics(); + logPerfMetrics(perfMetrics); } return status; } @@ -113,9 +96,8 @@ absl::Status VisualLanguageModelServable::preparePartialResponse(std::shared_ptr !executionContext->sendLoopbackSignal && llm_calculator_logger->should_log(spdlog::level::debug)) { auto vlmExecutionContext = std::static_pointer_cast(executionContext); - auto perfMetrics = tryGetVlmPerfMetrics(vlmExecutionContext->generationHandle); - if (perfMetrics) - logPerfMetrics(*perfMetrics); + auto perfMetrics = vlmExecutionContext->generationHandle->get_vlm_perf_metrics(); + logPerfMetrics(perfMetrics); } return status; }