1717import csv
1818import threading
1919from abc import ABC , abstractmethod
20+ from datetime import datetime
2021from pathlib import Path
2122
2223import numpy as np
24+ import psutil
2325from dateutil import parser
2426
2527from tests .functional .utils .logger import get_logger
@@ -32,6 +34,7 @@ class ResourceMonitor(threading.Thread, ABC):
3234 def __init__ (self ):
3335 threading .Thread .__init__ (self )
3436 self ._stop_event = threading .Event ()
37+ self .stop_reason = None
3538
3639 def stop (self ):
3740 self ._stop_event .set ()
@@ -42,6 +45,7 @@ def run(self):
4245 try :
4346 self .check_resources ()
4447 except StopIteration as e :
48+ self .stop_reason = str (e )
4549 self ._stop_event .set ()
4650 break
4751 self .save_data ()
@@ -55,13 +59,33 @@ def save_data(self):
5559 pass
5660
5761
62+ def _cgroup_cache_bytes (cgroup_memory_stats ):
63+ if "cache" in cgroup_memory_stats :
64+ return float (cgroup_memory_stats .get ("cache" , 0 ))
65+ return float (cgroup_memory_stats .get ("file" , 0 ))
66+
67+
5868class DockerResourceMonitor (ResourceMonitor ):
5969 MEMORY_USAGE = "MEMORY_USAGE"
60- FIELDS = ["DATE" , "PIDS_COUNT" , MEMORY_USAGE ] # + ["CPU_USAGE"] # Enable in further releases
70+ PRIVATE_MEMORY = "PRIVATE_MEMORY"
71+ MEMORY_CACHE = "MEMORY_CACHE"
72+ FIELDS = ["DATE" , "PIDS_COUNT" , MEMORY_USAGE , PRIVATE_MEMORY , MEMORY_CACHE ] # + ["CPU_USAGE"] # Enable in further releases
73+ VALIDATED_FIELDS = [MEMORY_USAGE , PRIVATE_MEMORY ]
74+ LOGGED_MEMORY_FIELDS = [MEMORY_CACHE ]
75+ COUNTER_FIELDS = ["PIDS_COUNT" ]
76+ LOGGED_FIELDS = LOGGED_MEMORY_FIELDS + COUNTER_FIELDS
6177 FIELDS_TO_STATS = {
6278 "DATE" : lambda x : x ["read" ],
6379 "PIDS_COUNT" : lambda x : int (x ["pids_stats" ].get ("current" , "0" )),
6480 MEMORY_USAGE : lambda x : "{:.2f}M" .format (float (x ["memory_stats" ].get ("usage" , "0.0" )) / (2 ** 20 )),
81+ PRIVATE_MEMORY : lambda x : "{:.2f}M" .format (
82+ float (x ["memory_stats" ].get ("stats" , {}).get (
83+ "anon" , x ["memory_stats" ].get ("stats" , {}).get ("rss" , 0 )
84+ )) / (2 ** 20 )
85+ ),
86+ MEMORY_CACHE : lambda x : "{:.2f}M" .format (
87+ _cgroup_cache_bytes (x ["memory_stats" ].get ("stats" , {})) / (2 ** 20 )
88+ ),
6589 # Enable after debug & fixing
6690 # "CPU_USAGE": lambda x:
6791 # [cpu / x['cpu_stats']['cpu_usage']['total_usage'] for cpu in x['cpu_stats']['cpu_usage']['percpu_usage']],
@@ -140,9 +164,142 @@ def _get_resource_data(self):
140164 'networks' = {dict: 1} {'eth0': {'rx_bytes': 90, 'rx_packets': 1, 'rx_errors': 0, ...
141165 """
142166 stats = self .container .stats (stream = False , decode = False )
167+ # Prevent reading zeroes from stopped container
168+ pids = int (stats .get ("pids_stats" , {}).get ("current" , 0 ) or 0 )
169+ usage = float (stats .get ("memory_stats" , {}).get ("usage" , 0 ) or 0 )
170+ if pids <= 0 or usage <= 0 :
171+ reason = (
172+ f"container { self .container .name } reports no live process "
173+ f"(pids_stats.current={ pids } , memory_stats.usage={ usage } bytes) - the container "
174+ f"stopped, crashed or was removed while the test was still sampling"
175+ )
176+ logger .warning (f"Stopping Docker resource monitor: { reason } " )
177+ raise StopIteration (reason )
143178 return stats
144179
145180 def get_stats_by_field (self , field ):
146181 result = self ._get_resource_data ()
147182 self ._docker_stats_data_raw .append (result )
148183 return self .get_field_data (field , result )
184+
185+ def sample_all (self ):
186+ """Read one stats snapshot and return all tracked metrics as floats (MB / counts).
187+
188+ A single snapshot keeps every metric in the returned sample mutually
189+ consistent (same instant) and avoids one docker stats call per metric.
190+ """
191+ stats = self ._get_resource_data ()
192+ self ._docker_stats_data_raw .append (stats )
193+ return {
194+ field : float (str (self .get_field_data (field , stats )).replace ("M" , "" ))
195+ for field in self .get_validated_metric_names () + self .get_logged_metric_names ()
196+ }
197+
198+ @classmethod
199+ def get_validated_metric_names (cls ):
200+ return cls .VALIDATED_FIELDS
201+
202+ @classmethod
203+ def get_logged_metric_names (cls ):
204+ return cls .LOGGED_FIELDS
205+
206+ @classmethod
207+ def get_memory_metric_names (cls ):
208+ return cls .VALIDATED_FIELDS + cls .LOGGED_MEMORY_FIELDS
209+
210+ @classmethod
211+ def get_counter_metric_names (cls ):
212+ return cls .COUNTER_FIELDS
213+
214+
215+ class WindowsResourceMonitor (ResourceMonitor ):
216+ WORKING_SET_SIZE = "WORKING_SET_SIZE"
217+ PRIVATE_BYTES = "PRIVATE_BYTES"
218+ PAGE_FAULTS = "PAGE_FAULTS"
219+
220+ MEMORY_USAGE = WORKING_SET_SIZE
221+
222+ FIELDS = ["DATE" , WORKING_SET_SIZE , PRIVATE_BYTES , PAGE_FAULTS ]
223+ VALIDATED_FIELDS = [PRIVATE_BYTES ]
224+ LOGGED_MEMORY_FIELDS = [WORKING_SET_SIZE ]
225+ COUNTER_FIELDS = [PAGE_FAULTS ]
226+ LOGGED_FIELDS = LOGGED_MEMORY_FIELDS + COUNTER_FIELDS
227+ SAMPLE_INTERVAL_SEC = 1.0
228+ # Optional callback invoked after save_data with (log_path).
229+ on_data_saved = None
230+
231+ def __init__ (self , ovms_pid ):
232+ super ().__init__ ()
233+ self .ovms_pid = ovms_pid
234+ self ._process = psutil .Process (int (ovms_pid ))
235+ self ._stats_data_raw = []
236+
237+ def cleanup (self ):
238+ if not self ._stop_event .is_set ():
239+ if self .is_alive ():
240+ self .stop ()
241+ self .save_data ()
242+
243+ def _get_resource_data (self ):
244+ stats = {"DATE" : datetime .now ().isoformat ()}
245+ try :
246+ info = self ._process .memory_info ()
247+ except psutil .Error as error :
248+ reason = (f"OVMS process pid={ self .ovms_pid } is no longer readable ({ error } ) - it "
249+ f"exited, crashed or was killed while the test was still sampling" )
250+ logger .warning (f"Stopping Windows resource monitor: { reason } " )
251+ raise StopIteration (reason ) from error
252+ # wset/private are the Win32 counters .NET exposes as WorkingSet64/PrivateMemorySize64.
253+ stats [self .WORKING_SET_SIZE ] = float (info .wset ) / (1024 * 1024 )
254+ stats [self .PRIVATE_BYTES ] = float (info .private ) / (1024 * 1024 )
255+ stats [self .PAGE_FAULTS ] = int (info .num_page_faults )
256+ return stats
257+
258+ def check_resources (self ):
259+ result = self ._get_resource_data ()
260+ self ._stats_data_raw .append (result )
261+ self ._stop_event .wait (self .SAMPLE_INTERVAL_SEC )
262+
263+ def save_data (self ):
264+ self .rows = list (self ._stats_data_raw )
265+ log_path = Path (artifacts_dir , f"windows_stats_pid_{ self .ovms_pid } .log" )
266+ with log_path .open ("w" ) as csvfile :
267+ writer = csv .DictWriter (csvfile , fieldnames = self .FIELDS )
268+ writer .writeheader ()
269+ writer .writerows (self .rows )
270+ if WindowsResourceMonitor .on_data_saved :
271+ WindowsResourceMonitor .on_data_saved (log_path )
272+ return log_path
273+
274+ def get_stats_by_field (self , field ):
275+ result = self ._get_resource_data ()
276+ self ._stats_data_raw .append (result )
277+ value = result [field ]
278+ if field in (self .WORKING_SET_SIZE , self .PRIVATE_BYTES ):
279+ return f"{ value :.2f} M"
280+ return str (value )
281+
282+ def sample_all (self ):
283+ """Read one process snapshot and return all tracked metrics as floats (MB / counts)."""
284+ stats = self ._get_resource_data ()
285+ self ._stats_data_raw .append (stats )
286+ return {
287+ field : float (stats [field ])
288+ for field in self .get_validated_metric_names () + self .get_logged_metric_names ()
289+ }
290+
291+ @classmethod
292+ def get_validated_metric_names (cls ):
293+ return cls .VALIDATED_FIELDS
294+
295+ @classmethod
296+ def get_logged_metric_names (cls ):
297+ return cls .LOGGED_FIELDS
298+
299+ @classmethod
300+ def get_memory_metric_names (cls ):
301+ return cls .VALIDATED_FIELDS + cls .LOGGED_MEMORY_FIELDS
302+
303+ @classmethod
304+ def get_counter_metric_names (cls ):
305+ return cls .COUNTER_FIELDS
0 commit comments