From b187f56437c6016be629d1d24598d3f80768d787 Mon Sep 17 00:00:00 2001
From: mzegla <milosz.zeglarski@intel.com>
Date: Mon, 10 Aug 2026 15:32:07 +0200
Subject: [PATCH] init

---
 .../continuous_batching/servable.cpp          | 25 +++++++++++++++---
 .../continuous_batching/servable.cpp          | 26 ++++++++++++++++---
 2 files changed, 43 insertions(+), 8 deletions(-)

diff --git a/src/llm/language_model/continuous_batching/servable.cpp b/src/llm/language_model/continuous_batching/servable.cpp
index ff7a435524..7b878a4060 100644
--- a/src/llm/language_model/continuous_batching/servable.cpp
+++ b/src/llm/language_model/continuous_batching/servable.cpp
@@ -17,6 +17,7 @@
 #include <memory>
 #include <stdexcept>
 #include <string>
+#include <thread>
 #include <vector>
 
 #include "../../../logging.hpp"
@@ -59,6 +60,20 @@ void ContinuousBatchingServable::logPerfMetrics(ov::genai::PerfMetrics& perfMetr
         prefillSpeedTps);
 }
 
+// CB stepping thread writes metrics in _free_non_running_requests() slightly after
+// pushing the final output. Yield briefly to close the race window.
+// TODO: remove once GenAI's get_perf_metrics() blocks instead of asserting (fix in generation_stream.hpp)
+static std::optional<ov::genai::PerfMetrics> tryGetPerfMetrics(const ov::genai::GenerationHandle& handle) {
+    for (int i = 0; i < 1000; ++i) {
+        try {
+            return handle->get_perf_metrics();
+        } catch (const ov::Exception&) {
+            std::this_thread::yield();
+        }
+    }
+    return std::nullopt;
+}
+
 void ContinuousBatchingServable::notifyExecutorThread() {
     SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Notifying executor thread");
     if (properties->llmExecutorWrapper == nullptr) {
@@ -165,8 +180,9 @@ absl::Status ContinuousBatchingServable::prepareCompleteResponse(std::shared_ptr
     auto status = GenAiServable::prepareCompleteResponse(executionContext);
     if (status.ok() && llm_calculator_logger->should_log(spdlog::level::debug)) {
         auto cbExecutionContext = std::static_pointer_cast<ContinuousBatchingServableExecutionContext>(executionContext);
-        auto perfMetrics = cbExecutionContext->generationHandle->get_perf_metrics();
-        logPerfMetrics(perfMetrics);
+        auto perfMetrics = tryGetPerfMetrics(cbExecutionContext->generationHandle);
+        if (perfMetrics)
+            logPerfMetrics(*perfMetrics);
     }
     return status;
 }
@@ -177,8 +193,9 @@ absl::Status ContinuousBatchingServable::preparePartialResponse(std::shared_ptr<
         !executionContext->sendLoopbackSignal &&
         llm_calculator_logger->should_log(spdlog::level::debug)) {
         auto cbExecutionContext = std::static_pointer_cast<ContinuousBatchingServableExecutionContext>(executionContext);
-        auto perfMetrics = cbExecutionContext->generationHandle->get_perf_metrics();
-        logPerfMetrics(perfMetrics);
+        auto perfMetrics = tryGetPerfMetrics(cbExecutionContext->generationHandle);
+        if (perfMetrics)
+            logPerfMetrics(*perfMetrics);
     }
     return status;
 }
diff --git a/src/llm/visual_language_model/continuous_batching/servable.cpp b/src/llm/visual_language_model/continuous_batching/servable.cpp
index ca52a3907a..5f6b79b8ae 100644
--- a/src/llm/visual_language_model/continuous_batching/servable.cpp
+++ b/src/llm/visual_language_model/continuous_batching/servable.cpp
@@ -19,6 +19,7 @@
 #include <memory>
 #include <stdexcept>
 #include <string>
+#include <thread>
 #include <vector>
 
 #include "src/port/rapidjson_document.hpp"
@@ -35,6 +36,21 @@
 
 namespace ovms {
 
+// CB stepping thread writes base perf metrics in _free_non_running_requests() slightly
+// after pushing the final output; get_vlm_perf_metrics() calls get_perf_metrics() internally.
+// Yield briefly to close the race window.
+// TODO: remove once GenAI's get_perf_metrics() blocks instead of asserting (fix in generation_stream.hpp)
+static std::optional<ov::genai::VLMPerfMetrics> tryGetVlmPerfMetrics(const ov::genai::GenerationHandle& handle) {
+    for (int i = 0; i < 1000; ++i) {
+        try {
+            return handle->get_vlm_perf_metrics();
+        } catch (const ov::Exception&) {
+            std::this_thread::yield();
+        }
+    }
+    return std::nullopt;
+}
+
 void VisualLanguageModelServable::logPerfMetrics(ov::genai::VLMPerfMetrics& perfMetrics) {
     const size_t inputTokenCount = perfMetrics.get_num_input_tokens();
     const size_t outputTokenCount = perfMetrics.get_num_generated_tokens();
@@ -84,8 +100,9 @@ absl::Status VisualLanguageModelServable::prepareCompleteResponse(std::shared_pt
     auto status = GenAiServable::prepareCompleteResponse(executionContext);
     if (status.ok() && llm_calculator_logger->should_log(spdlog::level::debug)) {
         auto vlmExecutionContext = std::static_pointer_cast<VisualLanguageModelServableExecutionContext>(executionContext);
-        auto perfMetrics = vlmExecutionContext->generationHandle->get_vlm_perf_metrics();
-        logPerfMetrics(perfMetrics);
+        auto perfMetrics = tryGetVlmPerfMetrics(vlmExecutionContext->generationHandle);
+        if (perfMetrics)
+            logPerfMetrics(*perfMetrics);
     }
     return status;
 }
@@ -96,8 +113,9 @@ absl::Status VisualLanguageModelServable::preparePartialResponse(std::shared_ptr
         !executionContext->sendLoopbackSignal &&
         llm_calculator_logger->should_log(spdlog::level::debug)) {
         auto vlmExecutionContext = std::static_pointer_cast<VisualLanguageModelServableExecutionContext>(executionContext);
-        auto perfMetrics = vlmExecutionContext->generationHandle->get_vlm_perf_metrics();
-        logPerfMetrics(perfMetrics);
+        auto perfMetrics = tryGetVlmPerfMetrics(vlmExecutionContext->generationHandle);
+        if (perfMetrics)
+            logPerfMetrics(*perfMetrics);
     }
     return status;
 }