Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
name: CI

on:
push:
branches: [main]
pull_request:
branches: [main]

jobs:
format-check:
name: clang-format check
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- name: Install clang-format
run: sudo apt-get update && sudo apt-get install -y clang-format
- name: Check formatting
run: make check-format

test:
name: Unit tests
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y libnvidia-ml-dev libgtest-dev
- name: Build and run unit tests
run: make test
45 changes: 37 additions & 8 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,38 +1,67 @@
CXX = g++
CXXFLAGS = -std=c++11 -Wall -Wextra -O2
NVML_LIB = /usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1
CUDA_INCLUDE = -I/usr/local/cuda-12.9/targets/x86_64-linux/include
CUDA_LIB = -L/usr/local/cuda-12.9/targets/x86_64-linux/lib -lcudart
CUDA_INCLUDE = -I/usr/local/cuda/targets/x86_64-linux/include
CUDA_LIB = -L/usr/local/cuda/targets/x86_64-linux/lib -lcudart
GTEST_LIBS = -lgtest -lgtest_main -lpthread

BUILD_DIR = build
MONITOR_TARGET = $(BUILD_DIR)/nvlink_monitor
EXAMPLE_TARGET = $(BUILD_DIR)/nvlink_bw_test
TEST_TARGET = $(BUILD_DIR)/run_tests

MONITOR_SOURCES = monitor/nvlink_monitor.cpp
MONITOR_HEADERS = monitor/nvlink_monitor.h
EXAMPLE_SOURCES = example/nvlink_bw_test.cpp
MONITOR_SOURCES = monitor/nvlink_monitor.cpp monitor/bandwidth_calc.cpp monitor/arg_parser.cpp
MONITOR_HEADERS = monitor/nvlink_monitor.h monitor/bandwidth_calc.h monitor/arg_parser.h
EXAMPLE_SOURCES = example/nvlink_bw_test.cpp example/bw_stats.cpp example/arg_parser.cpp
EXAMPLE_HEADERS = example/bw_stats.h example/arg_parser.h

TEST_SOURCES = test/test_bandwidth_calc.cpp test/test_arg_parser.cpp test/test_bw_stats.cpp \
test/test_bw_test_args.cpp \
monitor/bandwidth_calc.cpp monitor/arg_parser.cpp example/bw_stats.cpp example/arg_parser.cpp
TEST_HEADERS = monitor/bandwidth_calc.h monitor/arg_parser.h example/bw_stats.h example/arg_parser.h monitor/nvlink_monitor.h

# Sources checked by clang-format (all C++ sources and headers)
FORMAT_SOURCES = $(MONITOR_SOURCES) $(MONITOR_HEADERS) \
$(EXAMPLE_SOURCES) $(EXAMPLE_HEADERS) test/*.cpp

# Default target
all: $(MONITOR_TARGET) $(EXAMPLE_TARGET)

# Compile the monitor program
$(MONITOR_TARGET): $(MONITOR_SOURCES) $(MONITOR_HEADERS)
@mkdir -p $(BUILD_DIR)
$(CXX) $(CXXFLAGS) -o $(MONITOR_TARGET) $(MONITOR_SOURCES) $(NVML_LIB)
$(CXX) $(CXXFLAGS) $(CUDA_INCLUDE) -o $(MONITOR_TARGET) $(MONITOR_SOURCES) $(NVML_LIB)

# Compile the example program
$(EXAMPLE_TARGET): $(EXAMPLE_SOURCES)
$(EXAMPLE_TARGET): $(EXAMPLE_SOURCES) $(EXAMPLE_HEADERS)
@mkdir -p $(BUILD_DIR)
$(CXX) $(CXXFLAGS) $(CUDA_INCLUDE) -o $(EXAMPLE_TARGET) $(EXAMPLE_SOURCES) $(CUDA_LIB)

# Compile and run unit tests (no GPU/NVML/CUDA required to link)
$(TEST_TARGET): $(TEST_SOURCES) $(TEST_HEADERS)
@mkdir -p $(BUILD_DIR)
$(CXX) $(CXXFLAGS) -I. $(CUDA_INCLUDE) -o $(TEST_TARGET) $(TEST_SOURCES) $(GTEST_LIBS)

# Build only monitor
monitor: $(MONITOR_TARGET)

# Build only example
example: $(EXAMPLE_TARGET)

# Build and run unit tests
test: $(TEST_TARGET)
./$(TEST_TARGET)

# Clean build artifacts
clean:
rm -rf $(BUILD_DIR)

.PHONY: all monitor example clean
# Apply clang-format in place to all sources
format:
clang-format -i $(FORMAT_SOURCES)

# Fail if any source is not clang-format-clean (for CI)
check-format:
@clang-format --dry-run --Werror $(FORMAT_SOURCES)

.PHONY: all monitor example test format check-format clean
42 changes: 32 additions & 10 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,18 @@ A comprehensive toolkit for monitoring and testing NVIDIA NVLink bandwidth and s
```
NvLinkMonitor/
├── monitor/ # NVLink monitoring tool
│ ├── nvlink_monitor.cpp # Main monitoring implementation
│ └── nvlink_monitor.h # Monitoring tool headers
│ ├── nvlink_monitor.{cpp,h} # Monitor class + main()
│ ├── bandwidth_calc.{cpp,h} # Bandwidth calculation logic (extracted)
│ └── arg_parser.{cpp,h} # CLI argument parsing (extracted)
├── example/ # NVLink bandwidth testing tool
│ └── nvlink_bw_test.cpp # Bandwidth test implementation
├── build/ # Build output directory
├── Makefile # Main build configuration
│ ├── nvlink_bw_test.cpp # Bandwidth test main()
│ ├── bw_stats.{cpp,h} # Bandwidth stats aggregation (extracted)
│ └── arg_parser.{cpp,h} # CLI argument parsing (extracted)
├── test/ # GoogleTest unit tests (no GPU needed)
├── .github/workflows/ci.yml # CI: format check + unit tests
├── Makefile # Build configuration
├── install-deps.sh # Dependency installation script
├── .clang-format # clang-format style (Google, 4-space)
└── README.md # This file
```

Expand Down Expand Up @@ -69,6 +74,23 @@ The executables will be created in the `build/` directory:
- `build/nvlink_monitor` - NVLink monitoring tool
- `build/nvlink_bw_test` - NVLink bandwidth test tool

### 🧪 Testing

Unit tests use [GoogleTest](https://github.com/google/googletest) and run **without a GPU** — the test binary links only the extracted pure-logic modules (`bandwidth_calc`, `arg_parser`, `bw_stats`) plus gtest, with no NVML/CUDA library dependencies.

```bash
# Run unit tests (builds the test binary first)
make test

# Apply clang-format in place to all sources
make format

# Fail if any source is not clang-format-clean (CI gate)
make check-format
```

CI (`.github/workflows/ci.yml`) runs `make check-format` and `make test` on ubuntu-22.04 for every push and pull request. The GPU binaries themselves are not built in CI (they require a CUDA toolkit and driver).

## 🧩 Components

### 1. 📊 NVLink Monitor (`monitor/`)
Expand All @@ -82,27 +104,27 @@ A real-time monitoring tool for NVLink bandwidth and status.

#### Continuous monitoring:
```bash
./build/nvlink_monitor -continuous true
./build/nvlink_monitor --continuous true
```

#### Single monitoring:
```bash
./build/nvlink_monitor -continuous false
./build/nvlink_monitor --continuous false
```

#### Custom interval (e.g., 0.5 seconds):
```bash
./build/nvlink_monitor -interval 0.5
./build/nvlink_monitor --interval 0.5
```

#### Detailed NvLink output:
```bash
./build/nvlink_monitor -verbose
./build/nvlink_monitor --verbose
```

#### Combined options:
```bash
./build/nvlink_monitor -continuous false -interval 0.5 -verbose
./build/nvlink_monitor --continuous false --interval 0.5 --verbose
```

#### Output to file:
Expand Down
129 changes: 129 additions & 0 deletions example/arg_parser.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
#include "arg_parser.h"

#include <getopt.h>
#include <unistd.h>

#include <cerrno>
#include <cstdlib>
#include <string>

namespace {

// Parses a base-10 integer from `s` into `out`. Returns false and sets `err`
// on parse failure (non-numeric trailing chars) or ERANGE overflow.
bool parseLong(const char* s, long& out, std::string& err, const char* name) {
char* end = nullptr;
errno = 0;
out = std::strtol(s, &end, 10);
if (errno == ERANGE) {
err = std::string(name) + " out of range";
return false;
}
if (end == s || *end != '\0') {
err = std::string("invalid ") + name + ": " + s;
return false;
}
return true;
}

} // namespace

TestConfig parseBwTestArgs(int argc, char* argv[]) {
TestConfig config;

static struct option long_options[] = {
{"iterations", required_argument, 0, 'i'},
{"buffer-size", required_argument, 0, 'b'},
{"src-gpu", required_argument, 0, 's'},
{"dst-gpu", required_argument, 0, 'd'},
{"help", no_argument, 0, 'h'},
{0, 0, 0, 0}};

// Reset getopt's global state so repeated calls (e.g. in tests) re-parse
// from the beginning. glibc resets internal state when optind is set to 0.
optind = 0;

int opt;
int option_index = 0;

while ((opt = getopt_long(argc, argv, "i:b:s:d:h", long_options,
&option_index)) != -1) {
switch (opt) {
case 'i': {
long val;
if (!parseLong(optarg, val, config.errorMessage,
"iterations")) {
config.ok = false;
return config;
}
if (val <= 0) {
config.ok = false;
config.errorMessage = "iterations must be positive";
return config;
}
config.iterations = static_cast<int>(val);
break;
}
case 'b': {
long val;
if (!parseLong(optarg, val, config.errorMessage,
"buffer size")) {
config.ok = false;
return config;
}
if (val <= 0) {
config.ok = false;
config.errorMessage = "buffer size must be positive";
return config;
}
config.buffer_size_mb = static_cast<size_t>(val);
break;
}
case 's': {
long val;
if (!parseLong(optarg, val, config.errorMessage,
"source GPU ID")) {
config.ok = false;
return config;
}
if (val < 0) {
config.ok = false;
config.errorMessage = "source GPU ID must be non-negative";
return config;
}
config.src_gpu_id = static_cast<int>(val);
break;
}
case 'd': {
long val;
if (!parseLong(optarg, val, config.errorMessage,
"destination GPU ID")) {
config.ok = false;
return config;
}
if (val < 0) {
config.ok = false;
config.errorMessage =
"destination GPU ID must be non-negative";
return config;
}
config.dst_gpu_id = static_cast<int>(val);
break;
}
case 'h':
config.help = true;
break;
case '?':
// getopt already printed an error message to stderr.
config.ok = false;
config.errorMessage = "invalid command-line arguments";
return config;
default:
config.ok = false;
config.errorMessage = "unexpected option";
return config;
}
}

return config;
}
22 changes: 22 additions & 0 deletions example/arg_parser.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#ifndef NVLINK_BW_TEST_ARG_PARSER_H
#define NVLINK_BW_TEST_ARG_PARSER_H

#include <cstddef>
#include <string>

// Parsed command-line arguments for nvlink_bw_test.
struct TestConfig {
int iterations = 100;
size_t buffer_size_mb = 1000;
int src_gpu_id = 0;
int dst_gpu_id = 1;
bool help = false;
bool ok = true;
std::string errorMessage;
};

// Parses nvlink_bw_test command-line arguments.
// On error, sets ok=false and errorMessage (does not print or exit).
TestConfig parseBwTestArgs(int argc, char* argv[]);

#endif // NVLINK_BW_TEST_ARG_PARSER_H
28 changes: 28 additions & 0 deletions example/bw_stats.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#include "bw_stats.h"

#include <algorithm>
#include <numeric>

BandwidthStats computeBandwidthStats(const std::vector<double>& copyTimesMs,
size_t bufferSizeMb) {
BandwidthStats stats{0.0, 0.0, 0.0, 0.0, false};

if (copyTimesMs.empty()) {
return stats;
}

double avgTime =
std::accumulate(copyTimesMs.begin(), copyTimesMs.end(), 0.0) /
copyTimesMs.size();
double minTime = *std::min_element(copyTimesMs.begin(), copyTimesMs.end());
double maxTime = *std::max_element(copyTimesMs.begin(), copyTimesMs.end());

// bufferSizeMb is MiB; /1024.0 gives GiB. Time is ms; /1000.0 gives s.
stats.avgGiBps = (bufferSizeMb / 1024.0) / (avgTime / 1000.0);
stats.minGiBps = (bufferSizeMb / 1024.0) / (maxTime / 1000.0);
stats.maxGiBps = (bufferSizeMb / 1024.0) / (minTime / 1000.0);
stats.avgLatencyMs = avgTime;
stats.valid = true;

return stats;
}
24 changes: 24 additions & 0 deletions example/bw_stats.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#ifndef NVLINK_BW_TEST_STATS_H
#define NVLINK_BW_TEST_STATS_H

#include <cstddef>
#include <vector>

// Aggregated bandwidth statistics for a series of copy timings.
// Values are GiB/s (bufferSizeMb is MiB, divided by 1024 to get GiB).
struct BandwidthStats {
double avgGiBps;
double minGiBps;
double maxGiBps;
double avgLatencyMs;
bool valid;
};

// Computes avg/min/max bandwidth (GiB/s) and avg latency (ms) from a
// vector of per-iteration copy times (in ms). Returns valid=false on
// empty input. min/max bandwidth are derived from max/min time
// respectively (slowest copy = lowest bandwidth).
BandwidthStats computeBandwidthStats(const std::vector<double>& copyTimesMs,
size_t bufferSizeMb);

#endif // NVLINK_BW_TEST_STATS_H
Loading
Loading