diff --git a/README.md b/README.md index 05e3f19..ac3eeaf 100644 --- a/README.md +++ b/README.md @@ -136,11 +136,30 @@ A real-time monitoring tool for NVLink bandwidth and status. ./build/nvlink_monitor -v -o detailed.log ``` +#### Machine-readable output (CSV / JSON): +```bash +# CSV: header + one row per (gpu, link) per sample. A per-GPU "total" row +# (link_id=total) is always emitted so a GPU with no active links still appears. +./build/nvlink_monitor -f csv -o data.csv + +# JSONL: one self-contained JSON object per sample (streaming-friendly for +# continuous mode). Each object carries per-GPU totals and per-link arrays. +./build/nvlink_monitor --format json -o data.jsonl +``` + +Status/diagnostic messages (GPU discovery, "Starting continuous monitoring", +etc.) are written to **stderr**, so stdout only contains clean CSV/JSON data — +pipe or redirect stdout directly for analysis: +```bash +./build/nvlink_monitor -f csv 2>/dev/null | awk -F, '$4=="total"{print}' +``` + #### 📋 Available options: - `-c, --continuous [true|false]`: Run in continuous mode (default: true) - `-i, --interval `: Set custom monitoring interval in seconds (supports decimals, default: 1.0) - `-v, --verbose`: Enable detailed NvLink output (shows individual link bandwidth) - `-o, --output `: Redirect output to file +- `-f, --format text|csv|json`: Output format (default: text). CSV and JSON are machine-readable; `--verbose` only affects the text path - `-h, --help`: Show help information **Note:** The interval parameter supports decimal values (e.g., 0.5 for 500ms, 0.1 for 100ms). The minimum practical interval is 1 microsecond (0.000001s), but very small intervals may affect system performance. diff --git a/monitor/arg_parser.cpp b/monitor/arg_parser.cpp index 1870549..be90ef9 100644 --- a/monitor/arg_parser.cpp +++ b/monitor/arg_parser.cpp @@ -39,6 +39,25 @@ MonitorCliArgs parseMonitorArgs(int argc, char* argv[]) { return args; } args.outputFilename = argv[++i]; + } else if (arg == "-f" || arg == "--format") { + if (i + 1 >= argc) { + args.ok = false; + args.errorMessage = "Missing value for " + arg; + return args; + } + std::string f = argv[++i]; + if (f == "text") { + args.format = OutputFormat::Text; + } else if (f == "csv") { + args.format = OutputFormat::CSV; + } else if (f == "json") { + args.format = OutputFormat::JSON; + } else { + args.ok = false; + args.errorMessage = + "Invalid format: " + f + " (expected text, csv, or json)"; + return args; + } } else if (arg == "-c" || arg == "--continuous") { if (i + 1 < argc && (std::string(argv[i + 1]) == "true" || std::string(argv[i + 1]) == "false")) { diff --git a/monitor/arg_parser.h b/monitor/arg_parser.h index f5a43ff..d245239 100644 --- a/monitor/arg_parser.h +++ b/monitor/arg_parser.h @@ -3,11 +3,19 @@ #include +// Output format for nvlink_monitor. +// Text - human-readable (default) +// CSV - machine-readable: header + one row per (gpu, link) per sample +// JSON - JSONL: one self-contained JSON object per sample +// (streaming-friendly) +enum class OutputFormat { Text, CSV, JSON }; + // Parsed command-line arguments for nvlink_monitor. struct MonitorCliArgs { double interval = 1.0; bool continuous = true; bool verbose = false; + OutputFormat format = OutputFormat::Text; std::string outputFilename; bool helpRequested = false; bool ok = true; diff --git a/monitor/nvlink_monitor.cpp b/monitor/nvlink_monitor.cpp index dc0cbc4..304f966 100644 --- a/monitor/nvlink_monitor.cpp +++ b/monitor/nvlink_monitor.cpp @@ -2,11 +2,23 @@ #include +#include #include #include "arg_parser.h" #include "bandwidth_calc.h" +// Formats the current local time per the given strftime-style format string. +// Used by the CSV/JSON formatters for ISO 8601 timestamps. +static std::string formatNow(const char* fmt) { + auto now = std::chrono::system_clock::now(); + auto time_t = std::chrono::system_clock::to_time_t(now); + auto tm = *std::localtime(&time_t); + std::ostringstream oss; + oss << std::put_time(&tm, fmt); + return oss.str(); +} + // Global flag for signal handling. sig_atomic_t guarantees that writes from // the signal handler are well-defined. The handler only flips this flag — it // must NOT do any I/O (std::cout/cerr are not async-signal-safe and can @@ -22,8 +34,12 @@ void signal_handler(int signal) { } // NvLinkMonitor constructor -NvLinkMonitor::NvLinkMonitor(bool verbose, const std::string& outputFilename) - : verboseOutput(verbose), fileOutput(!outputFilename.empty()) { +NvLinkMonitor::NvLinkMonitor(bool verbose, OutputFormat format, + const std::string& outputFilename) + : verboseOutput(verbose), + outputFormat(format), + csvHeaderPrinted(false), + fileOutput(!outputFilename.empty()) { // Initialize NVML nvmlReturn_t result = nvmlInit(); if (result != NVML_SUCCESS) { @@ -63,7 +79,7 @@ void NvLinkMonitor::discoverGPUs() { return; } - std::cout << "Found " << deviceCount << " GPU(s)" << std::endl; + std::cerr << "Found " << deviceCount << " GPU(s)" << std::endl; for (unsigned int i = 0; i < deviceCount; i++) { nvmlDevice_t device; @@ -108,7 +124,7 @@ void NvLinkMonitor::discoverGPUs() { gpu.nvLinkCount = nvLinkCount; gpus.push_back(gpu); - std::cout << "GPU " << i << ": " << name << " (UUID: " << uuid << ") - " + std::cerr << "GPU " << i << ": " << name << " (UUID: " << uuid << ") - " << nvLinkCount << " NvLinks" << std::endl; } } @@ -266,10 +282,59 @@ void NvLinkMonitor::formatDetailedGPUResult( } } +void NvLinkMonitor::formatCsvResult( + const std::vector& results, double interval) { + auto& os = getOutputStream(); + if (!csvHeaderPrinted) { + os << "timestamp,interval_s,gpu_id,link_id,tx_gibps,rx_gibps\n"; + csvHeaderPrinted = true; + } + // ISO 8601 (no spaces) so the field never needs quoting. + std::string ts = formatNow("%Y-%m-%dT%H:%M:%S"); + os << std::fixed; + for (const auto& gpu : results) { + // Per-GPU total row (link_id=total); always emitted so a GPU with no + // active links still appears in the output. + os << ts << "," << std::setprecision(6) << interval << "," << gpu.gpuId + << ",total," << std::setprecision(3) << gpu.totalTxGiBps << "," + << gpu.totalRxGiBps << "\n"; + for (const auto& link : gpu.links) { + os << ts << "," << std::setprecision(6) << interval << "," + << gpu.gpuId << "," << link.linkId << "," << std::setprecision(3) + << link.txGiBps << "," << link.rxGiBps << "\n"; + } + } +} + +void NvLinkMonitor::formatJsonResult( + const std::vector& results, double interval) { + // One self-contained JSON object per line (JSONL) for streaming. + auto& os = getOutputStream(); + std::string ts = formatNow("%Y-%m-%dT%H:%M:%S"); + os << std::fixed; + os << "{\"ts\":\"" << ts << "\",\"interval_s\":" << std::setprecision(6) + << interval << ",\"gpus\":["; + for (size_t i = 0; i < results.size(); i++) { + const auto& gpu = results[i]; + if (i > 0) os << ","; + os << "{\"id\":\"" << gpu.gpuId + << "\",\"tx_gibps\":" << std::setprecision(3) << gpu.totalTxGiBps + << ",\"rx_gibps\":" << gpu.totalRxGiBps << ",\"links\":["; + for (size_t j = 0; j < gpu.links.size(); j++) { + const auto& link = gpu.links[j]; + if (j > 0) os << ","; + os << "{\"id\":" << link.linkId << ",\"tx_gibps\":" << link.txGiBps + << ",\"rx_gibps\":" << link.rxGiBps << "}"; + } + os << "]}"; + } + os << "]}\n"; +} + void NvLinkMonitor::runContinuousMonitoring(double interval) { - std::cout << "Starting continuous monitoring, interval: " << interval << "s" + std::cerr << "Starting continuous monitoring, interval: " << interval << "s" << std::endl; - std::cout << "Press Ctrl+C to stop monitoring" << std::endl; + std::cerr << "Press Ctrl+C to stop monitoring" << std::endl; // Set high priority for more accurate timing #ifdef _GNU_SOURCE @@ -277,7 +342,7 @@ void NvLinkMonitor::runContinuousMonitoring(double interval) { struct sched_param param; param.sched_priority = sched_get_priority_max(SCHED_FIFO); if (sched_setscheduler(0, SCHED_FIFO, ¶m) == 0) { - std::cout << "Set real-time scheduling priority for improved accuracy" + std::cerr << "Set real-time scheduling priority for improved accuracy" << std::endl; } #endif @@ -325,14 +390,27 @@ void NvLinkMonitor::runContinuousMonitoring(double interval) { auto results = calculateBandwidth(lastSnapshot, currentSnapshot, actualInterval); - // Add timing precision information in verbose mode - if (verboseOutput) { - formatDetailedGPUResult(results); - getOutputStream() - << " [Timing: " << std::fixed << std::setprecision(6) - << actualInterval << "s]" << std::endl; - } else { - formatGPUResult(results); + // Dispatch on output format. CSV/JSON always emit per-link data + // (structured output is already "detailed"); --verbose only affects + // the text path. + switch (outputFormat) { + case OutputFormat::CSV: + formatCsvResult(results, actualInterval); + break; + case OutputFormat::JSON: + formatJsonResult(results, actualInterval); + break; + case OutputFormat::Text: + default: + if (verboseOutput) { + formatDetailedGPUResult(results); + getOutputStream() + << " [Timing: " << std::fixed << std::setprecision(6) + << actualInterval << "s]" << std::endl; + } else { + formatGPUResult(results); + } + break; } lastSnapshot = currentSnapshot; @@ -341,10 +419,10 @@ void NvLinkMonitor::runContinuousMonitoring(double interval) { } void NvLinkMonitor::runSingleMonitoring(double interval) { - std::cout << "Getting first snapshot..." << std::endl; + std::cerr << "Getting first snapshot..." << std::endl; auto snapshot1 = getNvLinkData(); - std::cout << "Waiting " << interval << "s to get second snapshot..." + std::cerr << "Waiting " << interval << "s to get second snapshot..." << std::endl; // Use microsecond precision for sleep to improve timing accuracy std::this_thread::sleep_for( @@ -353,10 +431,21 @@ void NvLinkMonitor::runSingleMonitoring(double interval) { auto snapshot2 = getNvLinkData(); auto results = calculateBandwidth(snapshot1, snapshot2, interval); - if (verboseOutput) { - formatDetailedGPUResult(results); - } else { - formatGPUResult(results); + switch (outputFormat) { + case OutputFormat::CSV: + formatCsvResult(results, interval); + break; + case OutputFormat::JSON: + formatJsonResult(results, interval); + break; + case OutputFormat::Text: + default: + if (verboseOutput) { + formatDetailedGPUResult(results); + } else { + formatGPUResult(results); + } + break; } } @@ -376,6 +465,9 @@ void printHelp(const char* programName) { << std::endl; std::cout << " -o, --output : Redirect output to file" << std::endl; + std::cout + << " -f, --format text|csv|json : Output format (default: text)" + << std::endl; std::cout << " -h, --help : Show this help message" << std::endl; std::cout << std::endl; @@ -400,6 +492,11 @@ void printHelp(const char* programName) { std::cout << " " << programName << " -v -o detailed.log # Verbose output to file" << std::endl; + std::cout << " " << programName + << " -f csv -o data.csv # CSV output to file" << std::endl; + std::cout << " " << programName + << " --format json -o data.jsonl # JSONL output to file" + << std::endl; } int main(int argc, char* argv[]) { @@ -420,7 +517,7 @@ int main(int argc, char* argv[]) { signal(SIGTERM, signal_handler); try { - NvLinkMonitor monitor(args.verbose, args.outputFilename); + NvLinkMonitor monitor(args.verbose, args.format, args.outputFilename); if (args.continuous) { monitor.runContinuousMonitoring(args.interval); @@ -432,11 +529,12 @@ int main(int argc, char* argv[]) { return 1; } - // Printed from the main thread (not the signal handler) because std::cout + // Printed from the main thread (not the signal handler) because std::cerr // is not async-signal-safe. The handler only flips g_running; this covers - // both continuous and single monitoring modes uniformly. + // both continuous and single monitoring modes uniformly. Goes to stderr so + // it never corrupts CSV/JSON data on stdout. if (!g_running) { - std::cout << "\nReceived stop signal, exiting..." << std::endl; + std::cerr << "\nReceived stop signal, exiting..." << std::endl; } return 0; diff --git a/monitor/nvlink_monitor.h b/monitor/nvlink_monitor.h index 9933ece..847f29c 100644 --- a/monitor/nvlink_monitor.h +++ b/monitor/nvlink_monitor.h @@ -12,6 +12,8 @@ #include #include +#include "arg_parser.h" + // Global flag for signal handling. Uses sig_atomic_t (not bool) so that // writes from the async signal handler are well-defined per the C/C++ // standard; the handler must stay async-signal-safe (no I/O, no allocations). @@ -59,20 +61,25 @@ class NvLinkMonitor { private: std::vector gpus; std::vector lastResults; - bool verboseOutput; // Flag for detailed NvLink output - std::ofstream outputFile; // Output file stream - bool fileOutput; // Flag for file output + bool verboseOutput; // Flag for detailed NvLink output + OutputFormat outputFormat; // Output format (text/csv/json) + bool csvHeaderPrinted; // Whether the CSV header has been emitted + std::ofstream outputFile; // Output file stream + bool fileOutput; // Flag for file output public: /** * @brief Constructor - initializes NVML and discovers GPUs * @param verbose Enable detailed NvLink output + * @param format Output format (text/csv/json) * @param outputFilename Optional output file name (empty for console * output) * @throws std::runtime_error if NVML initialization fails or file cannot be * opened */ - NvLinkMonitor(bool verbose = false, const std::string& outputFilename = ""); + NvLinkMonitor(bool verbose = false, + OutputFormat format = OutputFormat::Text, + const std::string& outputFilename = ""); /** * @brief Destructor - shuts down NVML @@ -113,6 +120,24 @@ class NvLinkMonitor { */ void formatDetailedGPUResult(const std::vector& results); + /** + * @brief Formats results as CSV (header once, then one row per gpu/link). + * Emits a per-gpu "total" row (link_id=total) plus one row per link. + * @param results Vector of GPU monitoring results to display + * @param interval Actual sampling interval in seconds + */ + void formatCsvResult(const std::vector& results, + double interval); + + /** + * @brief Formats results as a single JSONL line (one JSON object per + * sample). Streaming-friendly for continuous mode. + * @param results Vector of GPU monitoring results to display + * @param interval Actual sampling interval in seconds + */ + void formatJsonResult(const std::vector& results, + double interval); + /** * @brief Gets the output stream (file or console) * @return Reference to the output stream diff --git a/test/test_arg_parser.cpp b/test/test_arg_parser.cpp index 56ab787..105678c 100644 --- a/test/test_arg_parser.cpp +++ b/test/test_arg_parser.cpp @@ -36,6 +36,7 @@ TEST(MonitorArgs, Defaults) { EXPECT_DOUBLE_EQ(args.interval, 1.0); EXPECT_TRUE(args.continuous); EXPECT_FALSE(args.verbose); + EXPECT_EQ(args.format, OutputFormat::Text); EXPECT_TRUE(args.outputFilename.empty()); EXPECT_FALSE(args.helpRequested); } @@ -152,6 +153,54 @@ TEST(MonitorArgs, ContinuousTrue) { EXPECT_TRUE(args.continuous); } +TEST(MonitorArgs, FormatDefaultIsText) { + Argv a{"nvlink_monitor"}; + auto args = parseMonitorArgs(a.argc(), a.argv()); + EXPECT_TRUE(args.ok); + EXPECT_EQ(args.format, OutputFormat::Text); +} + +TEST(MonitorArgs, FormatTextExplicit) { + Argv a{"nvlink_monitor", "--format", "text"}; + auto args = parseMonitorArgs(a.argc(), a.argv()); + EXPECT_TRUE(args.ok); + EXPECT_EQ(args.format, OutputFormat::Text); +} + +TEST(MonitorArgs, FormatCsvShort) { + Argv a{"nvlink_monitor", "-f", "csv"}; + auto args = parseMonitorArgs(a.argc(), a.argv()); + EXPECT_TRUE(args.ok); + EXPECT_EQ(args.format, OutputFormat::CSV); +} + +TEST(MonitorArgs, FormatCsvLong) { + Argv a{"nvlink_monitor", "--format", "csv"}; + auto args = parseMonitorArgs(a.argc(), a.argv()); + EXPECT_TRUE(args.ok); + EXPECT_EQ(args.format, OutputFormat::CSV); +} + +TEST(MonitorArgs, FormatJsonLong) { + Argv a{"nvlink_monitor", "--format", "json"}; + auto args = parseMonitorArgs(a.argc(), a.argv()); + EXPECT_TRUE(args.ok); + EXPECT_EQ(args.format, OutputFormat::JSON); +} + +TEST(MonitorArgs, FormatInvalidRejected) { + Argv a{"nvlink_monitor", "--format", "yaml"}; + auto args = parseMonitorArgs(a.argc(), a.argv()); + EXPECT_FALSE(args.ok); + EXPECT_FALSE(args.errorMessage.empty()); +} + +TEST(MonitorArgs, FormatMissingValueRejected) { + Argv a{"nvlink_monitor", "--format"}; + auto args = parseMonitorArgs(a.argc(), a.argv()); + EXPECT_FALSE(args.ok); +} + TEST(MonitorArgs, UnknownOptionRejected) { Argv a{"nvlink_monitor", "--bogus"}; auto args = parseMonitorArgs(a.argc(), a.argv()); @@ -159,10 +208,12 @@ TEST(MonitorArgs, UnknownOptionRejected) { } TEST(MonitorArgs, CombinedOptions) { - Argv a{"nvlink_monitor", "-v", "-i", "0.5", "-o", "out.log"}; + Argv a{"nvlink_monitor", "-v", "-i", "0.5", "-o", + "out.log", "--format", "csv"}; auto args = parseMonitorArgs(a.argc(), a.argv()); EXPECT_TRUE(args.ok); EXPECT_TRUE(args.verbose); EXPECT_DOUBLE_EQ(args.interval, 0.5); EXPECT_EQ(args.outputFilename, "out.log"); + EXPECT_EQ(args.format, OutputFormat::CSV); }