From cebdb93c600d8c7fa583c9b05d118dcb615e0b47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Otto=20Kr=C3=B6pke?= Date: Mon, 18 Aug 2025 19:16:10 +0200 Subject: [PATCH 1/4] GetRawCounterArray_size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan-Otto Kröpke --- internal/pdh/collector.go | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/internal/pdh/collector.go b/internal/pdh/collector.go index ef7e639bc..d8435727d 100644 --- a/internal/pdh/collector.go +++ b/internal/pdh/collector.go @@ -338,31 +338,27 @@ func (c *Collector) collectWorkerRaw() { for _, counter := range c.counters { for _, instance := range counter.Instances { - // Get the info with the current buffer size - bytesNeeded = uint32(cap(buf)) + bytesNeeded = 0 - for { - ret := GetRawCounterArray(instance, &bytesNeeded, &itemCount, &buf[0]) - - if ret == ErrorSuccess { + // Get the info with the current buffer size + ret := GetRawCounterArray(instance, &bytesNeeded, &itemCount, nil) + if err := NewPdhError(ret); ret != MoreData { + if isKnownCounterDataError(err) { break } - if err := NewPdhError(ret); ret != MoreData { - if isKnownCounterDataError(err) { - break - } - - return fmt.Errorf("GetRawCounterArray: %w", err) - } - - if bytesNeeded <= uint32(cap(buf)) { - return fmt.Errorf("GetRawCounterArray reports buffer too small (%d), but buffer is large enough (%d): %w", uint32(cap(buf)), bytesNeeded, NewPdhError(ret)) - } + return fmt.Errorf("GetRawCounterArray: %w", err) + } + if bytesNeeded > uint32(cap(buf)) { buf = make([]byte, bytesNeeded) } + ret = GetRawCounterArray(instance, &bytesNeeded, &itemCount, &buf[0]) + if ret != ErrorSuccess { + return fmt.Errorf("GetRawCounterArray: %w", err) + } + items = unsafe.Slice((*RawCounterItem)(unsafe.Pointer(&buf[0])), itemCount) var ( From 2ba863be1589ceaa80b3818ed7eb1980f9e3846f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Otto=20Kr=C3=B6pke?= Date: Sun, 24 Aug 2025 11:04:35 +0200 Subject: [PATCH 2/4] optimize pdh syscalls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan-Otto Kröpke --- internal/pdh/collector.go | 70 ++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/internal/pdh/collector.go b/internal/pdh/collector.go index d8435727d..12ba23ae6 100644 --- a/internal/pdh/collector.go +++ b/internal/pdh/collector.go @@ -338,25 +338,36 @@ func (c *Collector) collectWorkerRaw() { for _, counter := range c.counters { for _, instance := range counter.Instances { + // First call: get required buffer size (as per PDH documentation) bytesNeeded = 0 - - // Get the info with the current buffer size ret := GetRawCounterArray(instance, &bytesNeeded, &itemCount, nil) - if err := NewPdhError(ret); ret != MoreData { - if isKnownCounterDataError(err) { + if ret != MoreData { + if err := NewPdhError(ret); isKnownCounterDataError(err) { break } - - return fmt.Errorf("GetRawCounterArray: %w", err) + return fmt.Errorf("GetRawCounterArray size query: %w", NewPdhError(ret)) } + // Optimize buffer allocation: grow with headroom to reduce future reallocations if bytesNeeded > uint32(cap(buf)) { - buf = make([]byte, bytesNeeded) + // Grow buffer with some headroom (minimum 1KB, or 50% larger than needed) + newSize := bytesNeeded + if newSize < 1024 { + newSize = 1024 + } else if newSize < 8192 { // For buffers under 8KB, add 50% headroom + newSize = (newSize * 3) / 2 + } + buf = make([]byte, newSize) } - ret = GetRawCounterArray(instance, &bytesNeeded, &itemCount, &buf[0]) + // Second call: get the actual data (as per PDH documentation) + actualBytesNeeded := bytesNeeded + ret = GetRawCounterArray(instance, &actualBytesNeeded, &itemCount, &buf[0]) if ret != ErrorSuccess { - return fmt.Errorf("GetRawCounterArray: %w", err) + if err := NewPdhError(ret); isKnownCounterDataError(err) { + break + } + return fmt.Errorf("GetRawCounterArray data retrieval: %w", NewPdhError(ret)) } items = unsafe.Slice((*RawCounterItem)(unsafe.Pointer(&buf[0])), itemCount) @@ -501,29 +512,36 @@ func (c *Collector) collectWorkerFormatted() { for _, counter := range c.counters { for _, instance := range counter.Instances { - // Get the info with the current buffer size - bytesNeeded = uint32(cap(buf)) - - for { - ret := GetFormattedCounterArrayDouble(instance, &bytesNeeded, &itemCount, &buf[0]) - - if ret == ErrorSuccess { + // First call: get required buffer size (as per PDH documentation) + bytesNeeded = 0 + ret := GetFormattedCounterArrayDouble(instance, &bytesNeeded, &itemCount, nil) + if ret != MoreData { + if err := NewPdhError(ret); isKnownCounterDataError(err) { break } + return fmt.Errorf("GetFormattedCounterArrayDouble size query: %w", NewPdhError(ret)) + } - if err := NewPdhError(ret); ret != MoreData { - if isKnownCounterDataError(err) { - break - } - - return fmt.Errorf("GetFormattedCounterArrayDouble: %w", err) + // Optimize buffer allocation: grow with headroom to reduce future reallocations + if bytesNeeded > uint32(cap(buf)) { + // Grow buffer with some headroom (minimum 1KB, or 50% larger than needed) + newSize := bytesNeeded + if newSize < 1024 { + newSize = 1024 + } else if newSize < 8192 { // For buffers under 8KB, add 50% headroom + newSize = (newSize * 3) / 2 } + buf = make([]byte, newSize) + } - if bytesNeeded <= uint32(cap(buf)) { - return fmt.Errorf("GetFormattedCounterArrayDouble reports buffer too small (%d), but buffer is large enough (%d): %w", uint32(cap(buf)), bytesNeeded, NewPdhError(ret)) + // Second call: get the actual data (as per PDH documentation) + actualBytesNeeded := bytesNeeded + ret = GetFormattedCounterArrayDouble(instance, &actualBytesNeeded, &itemCount, &buf[0]) + if ret != ErrorSuccess { + if err := NewPdhError(ret); isKnownCounterDataError(err) { + break } - - buf = make([]byte, bytesNeeded) + return fmt.Errorf("GetFormattedCounterArrayDouble data retrieval: %w", NewPdhError(ret)) } items = unsafe.Slice((*FmtCounterValueItemDouble)(unsafe.Pointer(&buf[0])), itemCount) From 44e2257c6f2352fe79bde69376bda32667aa6b74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Otto=20Kr=C3=B6pke?= Date: Sun, 24 Aug 2025 11:20:23 +0200 Subject: [PATCH 3/4] optimize pdh syscalls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan-Otto Kröpke --- .idea/dictionaries/project.xml | 3 +- internal/pdh/collector.go | 48 ++++++++++++------------ internal/pdh/collector_bench_test.go | 56 ++++++++++++++-------------- 3 files changed, 55 insertions(+), 52 deletions(-) diff --git a/.idea/dictionaries/project.xml b/.idea/dictionaries/project.xml index 74d07f8aa..cd6050e1e 100644 --- a/.idea/dictionaries/project.xml +++ b/.idea/dictionaries/project.xml @@ -6,6 +6,7 @@ gochecknoglobals lpwstr luid + perfdata operationoptions setupapi spdx @@ -13,4 +14,4 @@ vmcompute - \ No newline at end of file + diff --git a/internal/pdh/collector.go b/internal/pdh/collector.go index 12ba23ae6..a76fdff9b 100644 --- a/internal/pdh/collector.go +++ b/internal/pdh/collector.go @@ -301,6 +301,8 @@ func (c *Collector) collectWorkerRaw() { ) buf := make([]byte, 1) + // Reuse string cache across collection cycles to reduce allocations + stringCache := make(map[*uint16]string, 64) for data := range c.collectCh { err = (func() error { @@ -334,7 +336,10 @@ func (c *Collector) collectWorkerRaw() { elemValue := reflect.ValueOf(reflect.New(elemType).Interface()).Elem() indexMap := map[string]int{} - stringMap := map[*uint16]string{} + // Clear and reuse string cache instead of creating new one each time + for k := range stringCache { + delete(stringCache, k) + } for _, counter := range c.counters { for _, instance := range counter.Instances { @@ -372,11 +377,6 @@ func (c *Collector) collectWorkerRaw() { items = unsafe.Slice((*RawCounterItem)(unsafe.Pointer(&buf[0])), itemCount) - var ( - instanceName string - ok bool - ) - for _, item := range items { if item.RawValue.CStatus != CstatusValidData && item.RawValue.CStatus != CstatusNewData { c.logger.Debug("skipping counter item with invalid data status", @@ -388,9 +388,14 @@ func (c *Collector) collectWorkerRaw() { continue } - if instanceName, ok = stringMap[item.SzName]; !ok { + var ( + instanceName string + ok bool + ) + + if instanceName, ok = stringCache[item.SzName]; !ok { instanceName = windows.UTF16PtrToString(item.SzName) - stringMap[item.SzName] = instanceName + stringCache[item.SzName] = instanceName } if strings.HasSuffix(instanceName, InstanceTotal) && !c.totalCounterRequested { @@ -401,12 +406,8 @@ func (c *Collector) collectWorkerRaw() { instanceName = InstanceEmpty } - var ( - index int - ok bool - ) - - if index, ok = indexMap[instanceName]; !ok { + index, indexExists := indexMap[instanceName] + if !indexExists { index = dv.Len() indexMap[instanceName] = index @@ -475,6 +476,8 @@ func (c *Collector) collectWorkerFormatted() { ) buf := make([]byte, 1) + // Reuse string cache across collection cycles to reduce allocations + stringCache := make(map[*uint16]string, 64) for data := range c.collectCh { err = (func() error { @@ -508,7 +511,10 @@ func (c *Collector) collectWorkerFormatted() { elemValue := reflect.ValueOf(reflect.New(elemType).Interface()).Elem() indexMap := map[string]int{} - stringMap := map[*uint16]string{} + // Clear and reuse string cache instead of creating new one each time + for k := range stringCache { + delete(stringCache, k) + } for _, counter := range c.counters { for _, instance := range counter.Instances { @@ -556,9 +562,9 @@ func (c *Collector) collectWorkerFormatted() { continue } - if instanceName, ok = stringMap[item.SzName]; !ok { + if instanceName, ok = stringCache[item.SzName]; !ok { instanceName = windows.UTF16PtrToString(item.SzName) - stringMap[item.SzName] = instanceName + stringCache[item.SzName] = instanceName } if strings.HasSuffix(instanceName, InstanceTotal) && !c.totalCounterRequested { @@ -569,12 +575,8 @@ func (c *Collector) collectWorkerFormatted() { instanceName = InstanceEmpty } - var ( - index int - ok bool - ) - - if index, ok = indexMap[instanceName]; !ok { + index, indexExists := indexMap[instanceName] + if !indexExists { index = dv.Len() indexMap[instanceName] = index diff --git a/internal/pdh/collector_bench_test.go b/internal/pdh/collector_bench_test.go index 666a2af1c..43712f1e8 100644 --- a/internal/pdh/collector_bench_test.go +++ b/internal/pdh/collector_bench_test.go @@ -28,34 +28,34 @@ import ( type processFull struct { Name string - ProcessorTime float64 `pdh:"% Processor Time"` - PrivilegedTime float64 `pdh:"% Privileged Time"` - UserTime float64 `pdh:"% User Time"` - CreatingProcessID float64 `pdh:"Creating Process ID"` - ElapsedTime float64 `pdh:"Elapsed Time"` - HandleCount float64 `pdh:"Handle Count"` - IDProcess float64 `pdh:"ID Process"` - IODataBytesSec float64 `pdh:"IO Data Bytes/sec"` - IODataOperationsSec float64 `pdh:"IO Data Operations/sec"` - IOOtherBytesSec float64 `pdh:"IO Other Bytes/sec"` - IOOtherOperationsSec float64 `pdh:"IO Other Operations/sec"` - IOReadBytesSec float64 `pdh:"IO Read Bytes/sec"` - IOReadOperationsSec float64 `pdh:"IO Read Operations/sec"` - IOWriteBytesSec float64 `pdh:"IO Write Bytes/sec"` - IOWriteOperationsSec float64 `pdh:"IO Write Operations/sec"` - PageFaultsSec float64 `pdh:"Page Faults/sec"` - PageFileBytesPeak float64 `pdh:"Page File Bytes Peak"` - PageFileBytes float64 `pdh:"Page File Bytes"` - PoolNonpagedBytes float64 `pdh:"Pool Nonpaged Bytes"` - PoolPagedBytes float64 `pdh:"Pool Paged Bytes"` - PriorityBase float64 `pdh:"Priority Base"` - PrivateBytes float64 `pdh:"Private Bytes"` - ThreadCount float64 `pdh:"Thread Count"` - VirtualBytesPeak float64 `pdh:"Virtual Bytes Peak"` - VirtualBytes float64 `pdh:"Virtual Bytes"` - WorkingSetPrivate float64 `pdh:"Working Set - Private"` - WorkingSetPeak float64 `pdh:"Working Set Peak"` - WorkingSet float64 `pdh:"Working Set"` + ProcessorTime float64 `perfdata:"% Processor Time"` + PrivilegedTime float64 `perfdata:"% Privileged Time"` + UserTime float64 `perfdata:"% User Time"` + CreatingProcessID float64 `perfdata:"Creating Process ID"` + ElapsedTime float64 `perfdata:"Elapsed Time"` + HandleCount float64 `perfdata:"Handle Count"` + IDProcess float64 `perfdata:"ID Process"` + IODataBytesSec float64 `perfdata:"IO Data Bytes/sec"` + IODataOperationsSec float64 `perfdata:"IO Data Operations/sec"` + IOOtherBytesSec float64 `perfdata:"IO Other Bytes/sec"` + IOOtherOperationsSec float64 `perfdata:"IO Other Operations/sec"` + IOReadBytesSec float64 `perfdata:"IO Read Bytes/sec"` + IOReadOperationsSec float64 `perfdata:"IO Read Operations/sec"` + IOWriteBytesSec float64 `perfdata:"IO Write Bytes/sec"` + IOWriteOperationsSec float64 `perfdata:"IO Write Operations/sec"` + PageFaultsSec float64 `perfdata:"Page Faults/sec"` + PageFileBytesPeak float64 `perfdata:"Page File Bytes Peak"` + PageFileBytes float64 `perfdata:"Page File Bytes"` + PoolNonpagedBytes float64 `perfdata:"Pool Nonpaged Bytes"` + PoolPagedBytes float64 `perfdata:"Pool Paged Bytes"` + PriorityBase float64 `perfdata:"Priority Base"` + PrivateBytes float64 `perfdata:"Private Bytes"` + ThreadCount float64 `perfdata:"Thread Count"` + VirtualBytesPeak float64 `perfdata:"Virtual Bytes Peak"` + VirtualBytes float64 `perfdata:"Virtual Bytes"` + WorkingSetPrivate float64 `perfdata:"Working Set - Private"` + WorkingSetPeak float64 `perfdata:"Working Set Peak"` + WorkingSet float64 `perfdata:"Working Set"` } func BenchmarkTestCollector(b *testing.B) { From 74d8bc7afe832555aa32c68fd3e8a9f4ca99c5d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Otto=20Kr=C3=B6pke?= Date: Sun, 24 Aug 2025 14:22:33 +0200 Subject: [PATCH 4/4] collector: call PdhGetRawCounterArrayW twice to get the correct amount of bytes needed. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan-Otto Kröpke --- .idea/dictionaries/project.xml | 4 ++-- internal/pdh/collector.go | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.idea/dictionaries/project.xml b/.idea/dictionaries/project.xml index cd6050e1e..719a7631a 100644 --- a/.idea/dictionaries/project.xml +++ b/.idea/dictionaries/project.xml @@ -6,12 +6,12 @@ gochecknoglobals lpwstr luid - perfdata operationoptions + perfdata setupapi spdx textfile vmcompute - + \ No newline at end of file diff --git a/internal/pdh/collector.go b/internal/pdh/collector.go index a76fdff9b..d34b1ae6b 100644 --- a/internal/pdh/collector.go +++ b/internal/pdh/collector.go @@ -348,8 +348,15 @@ func (c *Collector) collectWorkerRaw() { ret := GetRawCounterArray(instance, &bytesNeeded, &itemCount, nil) if ret != MoreData { if err := NewPdhError(ret); isKnownCounterDataError(err) { + c.logger.Debug("no data for counter instance", + slog.String("counter", counter.Name), + slog.String("object", c.object), + slog.Any("err", err), + ) + break } + return fmt.Errorf("GetRawCounterArray size query: %w", NewPdhError(ret)) } @@ -370,8 +377,15 @@ func (c *Collector) collectWorkerRaw() { ret = GetRawCounterArray(instance, &actualBytesNeeded, &itemCount, &buf[0]) if ret != ErrorSuccess { if err := NewPdhError(ret); isKnownCounterDataError(err) { + c.logger.Debug("no data for counter instance", + slog.String("counter", counter.Name), + slog.String("object", c.object), + slog.Any("err", err), + ) + break } + return fmt.Errorf("GetRawCounterArray data retrieval: %w", NewPdhError(ret)) } @@ -525,6 +539,7 @@ func (c *Collector) collectWorkerFormatted() { if err := NewPdhError(ret); isKnownCounterDataError(err) { break } + return fmt.Errorf("GetFormattedCounterArrayDouble size query: %w", NewPdhError(ret)) } @@ -547,6 +562,7 @@ func (c *Collector) collectWorkerFormatted() { if err := NewPdhError(ret); isKnownCounterDataError(err) { break } + return fmt.Errorf("GetFormattedCounterArrayDouble data retrieval: %w", NewPdhError(ret)) }