From c54ce55edd3b773caa333e1ca1d4da30d2d8e9c9 Mon Sep 17 00:00:00 2001 From: Michael K Date: Mon, 4 May 2026 17:35:21 -0500 Subject: [PATCH 1/3] fix header and add --head flag to lfd --- MIDAS/src/midas_shell_commands.h | 41 ++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/MIDAS/src/midas_shell_commands.h b/MIDAS/src/midas_shell_commands.h index 2f29e6bb..152630e8 100644 --- a/MIDAS/src/midas_shell_commands.h +++ b/MIDAS/src/midas_shell_commands.h @@ -574,7 +574,18 @@ MCommandExecutionResult cmd_lfd(const MShellContext& ctx) { // // - if (ctx.argc != 2) { return MCommandExecutionResult::ERR_INVAL_ARGC; } + if (ctx.argc != 2 && ctx.argc != 3) { } + bool stop_after_header_print = false; + if(ctx.argc == 3) { + if(!strcmp(ctx.argv[2], "--head")) { + stop_after_header_print = true; + } else { + return MCommandExecutionResult::ERR_INVAL_ARGUMENT; + } + } else if (ctx.argc != 2) { + return MCommandExecutionResult::ERR_INVAL_ARGC; + } + String bin_path = "/" + String(ctx.argv[1]) + ".bin"; String meta_path = "/" + String(ctx.argv[1]) + ".meta"; @@ -590,15 +601,35 @@ MCommandExecutionResult cmd_lfd(const MShellContext& ctx) { Serial.println("Failed to open .meta file!"); return MCommandExecutionResult::ERR_FS_FAIL_OPEN; } + + size_t size_ptr = LOG_FMT_VERSION; // We write multiple uint32_ts, so for serial.write to print it properly we need to store it // Start writing header - Serial.write("LAUNCH "); Serial.write(LOG_FMT_VERSION); Serial.write('\n'); + Serial.write("LAUNCH "); Serial.write((uint8_t*)size_ptr, sizeof(size_t)); Serial.write('\n'); + + // File name Serial.write("FILE "); Serial.write(ctx.argv[1]); Serial.write('\n'); - Serial.write("CHECKSUM "); Serial.write(LOG_CHECKSUM); Serial.write(" "); Serial.write(EEPROM_CHECKSUM); Serial.write('\n'); - Serial.write("META "); Serial.write(meta_file.size()); Serial.write('\n'); - Serial.write("BIN "); Serial.write(bin_file.size()); Serial.write('\n'); + + // Log checksum + size_ptr = LOG_CHECKSUM; + Serial.write("CHECKSUM "); Serial.write((uint8_t*)size_ptr, sizeof(size_t)); Serial.write(" "); + + // EEPROM checksum + size_ptr = EEPROM_CHECKSUM; + Serial.write((uint8_t*)size_ptr, sizeof(size_t)); Serial.write('\n'); + + // Meta file size + size_ptr = meta_file.size(); + Serial.write("META "); Serial.write((uint8_t*)size_ptr, sizeof(size_t)); Serial.write('\n'); + + // Binary file size + size_ptr = bin_file.size(); + Serial.write("BIN "); Serial.write((uint8_t*)size_ptr, sizeof(size_t)); Serial.write('\n'); + Serial.flush(); + if(stop_after_header_print) { return MCommandExecutionResult::OK; } + // Then dump meta and bin files while(meta_file.available()) { Serial.write(meta_file.read()); From 6e403fd9ec09fae183b91d4c778fbd5e565b8c71 Mon Sep 17 00:00:00 2001 From: Michael K Date: Mon, 4 May 2026 20:39:58 -0500 Subject: [PATCH 2/3] fix lfd --- MIDAS/src/midas_shell_commands.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/MIDAS/src/midas_shell_commands.h b/MIDAS/src/midas_shell_commands.h index 152630e8..d95640bd 100644 --- a/MIDAS/src/midas_shell_commands.h +++ b/MIDAS/src/midas_shell_commands.h @@ -605,26 +605,26 @@ MCommandExecutionResult cmd_lfd(const MShellContext& ctx) { size_t size_ptr = LOG_FMT_VERSION; // We write multiple uint32_ts, so for serial.write to print it properly we need to store it // Start writing header - Serial.write("LAUNCH "); Serial.write((uint8_t*)size_ptr, sizeof(size_t)); Serial.write('\n'); + Serial.write("LAUNCH "); Serial.write((uint8_t*)&size_ptr, sizeof(size_t)); Serial.write('\n'); // File name Serial.write("FILE "); Serial.write(ctx.argv[1]); Serial.write('\n'); // Log checksum size_ptr = LOG_CHECKSUM; - Serial.write("CHECKSUM "); Serial.write((uint8_t*)size_ptr, sizeof(size_t)); Serial.write(" "); + Serial.write("CHECKSUM "); Serial.write((uint8_t*)&size_ptr, sizeof(size_t)); Serial.write(" "); // EEPROM checksum size_ptr = EEPROM_CHECKSUM; - Serial.write((uint8_t*)size_ptr, sizeof(size_t)); Serial.write('\n'); + Serial.write((uint8_t*)&size_ptr, sizeof(size_t)); Serial.write('\n'); // Meta file size size_ptr = meta_file.size(); - Serial.write("META "); Serial.write((uint8_t*)size_ptr, sizeof(size_t)); Serial.write('\n'); + Serial.write("META "); Serial.write((uint8_t*)&size_ptr, sizeof(size_t)); Serial.write('\n'); // Binary file size size_ptr = bin_file.size(); - Serial.write("BIN "); Serial.write((uint8_t*)size_ptr, sizeof(size_t)); Serial.write('\n'); + Serial.write("BIN "); Serial.write((uint8_t*)&size_ptr, sizeof(size_t)); Serial.write('\n'); Serial.flush(); From e50628c1686e7037ae7517549a43366a7dfb934a Mon Sep 17 00:00:00 2001 From: Michael K Date: Thu, 7 May 2026 16:46:55 -0500 Subject: [PATCH 3/3] my evil plan is coming to fruition --- MIDAS/platformio.ini | 26 +-- MIDAS/src/log_decode/main.cpp | 8 - MIDAS/{ => tools}/extra_script.py | 0 MIDAS/tools/lfd/lfdconvert.py | 333 ++++++++++++++++++++++++++++++ MIDAS/tools/lfd/lfdoffload.py | 88 ++++++++ MIDAS/tools/lfd/lfdplot.py | 43 ++++ MIDAS/{ => tools}/log_enc.py | 0 7 files changed, 470 insertions(+), 28 deletions(-) delete mode 100644 MIDAS/src/log_decode/main.cpp rename MIDAS/{ => tools}/extra_script.py (100%) create mode 100644 MIDAS/tools/lfd/lfdconvert.py create mode 100644 MIDAS/tools/lfd/lfdoffload.py create mode 100644 MIDAS/tools/lfd/lfdplot.py rename MIDAS/{ => tools}/log_enc.py (100%) diff --git a/MIDAS/platformio.ini b/MIDAS/platformio.ini index 69d49e14..fe8903c8 100644 --- a/MIDAS/platformio.ini +++ b/MIDAS/platformio.ini @@ -17,8 +17,8 @@ board = esp32-s3-devkitc-1 framework = arduino monitor_speed = 115200 extra_scripts = - pre:extra_script.py - pre:log_enc.py + pre:tools/extra_script.py + pre:tools/log_enc.py build_flags = -DARDUINO_USB_CDC_ON_BOOT=1 -DCONFIG_DISABLE_HAL_LOCKS=1 @@ -37,7 +37,7 @@ platform = espressif32@6.7.0 board = adafruit_feather_esp32s3 framework = arduino extra_scripts = - pre:extra_script.py + pre:tools/extra_script.py build_flags = -DARDUINO_USB_CDC_ON_BOOT=1 -DHILSIM=1 @@ -52,7 +52,7 @@ platform = espressif32@6.7.0 board = adafruit_feather_esp32s3 framework = arduino extra_scripts = - pre:extra_script.py + pre:tools/extra_script.py build_flags = -DARDUINO_USB_CDC_ON_BOOT=1 -DHILSIM=1 @@ -62,25 +62,11 @@ build_unflags = -std=gnu++11 lib_deps = sparkfun/SparkFun MMC5983MA Magnetometer Arduino Library@^1.1.4 -[env:logdecoder] -platform = native -build_type = debug -extra_scripts = - pre:extra_script.py - pre:log_enc.py -build_flags = - -g3 - -std=gnu++2a -build_src_filter = + - - - -build_unflags = - -std=gnu++11 -lib_deps = sparkfun/SparkFun MMC5983MA Magnetometer Arduino Library@^1.1.4 - [env:mcu_silsim_sustainer] platform = native build_type = debug extra_scripts = - pre:extra_script.py + pre:tools/extra_script.py build_flags = -DSILSIM -g3 @@ -112,7 +98,7 @@ extra_scripts = pre:test/fsm_test/srcinc.py platform = native build_type = debug extra_scripts = - pre:extra_script.py + pre:tools/extra_script.py build_flags = -DSILSIM -g3 diff --git a/MIDAS/src/log_decode/main.cpp b/MIDAS/src/log_decode/main.cpp deleted file mode 100644 index 1e13a72e..00000000 --- a/MIDAS/src/log_decode/main.cpp +++ /dev/null @@ -1,8 +0,0 @@ -#include "log_format_AUTOGEN.h" -#include "log_checksum.h" -#include - -int main() { - std::cout << "to be done" << std::endl; - return 0; -} diff --git a/MIDAS/extra_script.py b/MIDAS/tools/extra_script.py similarity index 100% rename from MIDAS/extra_script.py rename to MIDAS/tools/extra_script.py diff --git a/MIDAS/tools/lfd/lfdconvert.py b/MIDAS/tools/lfd/lfdconvert.py new file mode 100644 index 00000000..49f62e00 --- /dev/null +++ b/MIDAS/tools/lfd/lfdconvert.py @@ -0,0 +1,333 @@ +"""MIDAS .launch v2 file decoder + +Usage: + python lfdconvert.py [-o prefix] + +Michael Karpov, 2026 +""" +import argparse, csv, struct +from pathlib import Path + + +T_STRUCT, T_FLOAT32, T_FLOAT64 = 0, 1, 2 +T_INT8, T_UINT8, T_INT16, T_UINT16 = 3, 4, 5, 6 +T_INT32, T_UINT32, T_INT64, T_UINT64 = 7, 8, 9, 10 +T_BOOL, T_ENUM, T_ARRAY, T_UNION = 11, 12, 13, 14 + +LEAVES = { + T_FLOAT32: (" len(blob): + raise ValueError(f"unexpected EOF at offset {pos}") + out = blob[pos:pos+n]; pos += n; return out + def expect(lit): + got = take(len(lit)) + if got != lit: raise ValueError(f"expected {lit}, got {got}") + def u32(): + return struct.unpack(" len(bin_blob): + print(f"missing data for {vname}\n") + break + data = bin_blob[pos:pos+vsize]; pos += vsize + + row = [str(ts), vname] + [""] * len(columns) + for i, (col_disc, _, (_, tid, off)) in enumerate(columns): + if col_disc == disc: + sz = LEAVES[tid][1] + row[2+i] = fmt_value(tid, data[off:off+sz]) + decoded_rows.append(row) + + #backfill + next_seen = [""] * len(columns) + for row in reversed(decoded_rows): + for i in range(len(columns)): + col = 2 + i + if row[col] == "": + row[col] = next_seen[i] + else: + next_seen[i] = row[col] + + with open(path, "w", newline="") as f: + w = csv.writer(f) + w.writerow(["timestamp_ms", "sensor"] + [f"{vname}.{leaf[0]}" for _, vname, leaf in columns]) + w.writerows(decoded_rows) + + return len(decoded_rows), len(columns) + + +def write_eeprom_csv(eeprom_data, eeprom_entries, path): + root, _ = build_tree(eeprom_entries) + if root.size > len(eeprom_data): + print(f"warning: EEPROM struct ({root.size}B) > blob ({len(eeprom_data)}B); truncating\n") + + rows = 0 + with open(path, "w", newline="") as f: + w = csv.writer(f) + w.writerow(["field", "value"]) + for field, tid, off in leaves_of(root): + sz = LEAVES[tid][1] + if off + sz > len(eeprom_data): + break + w.writerow([field, fmt_value(tid, eeprom_data[off:off+sz])]) + rows += 1 + return rows + + +def write_meta_csv(metalog_tail, path): + pos, rows = 0, 0 + with open(path, "w", newline="") as f: + w = csv.writer(f) + w.writerow(["code", "value"]) + while pos + 4 <= len(metalog_tail): + code = struct.unpack(" len(metalog_tail): + print(f"truncated metalog entry for {name}\n") + break + value = fmt_value(tid, metalog_tail[pos:pos+sz]) + pos += sz + if metalog_tail[pos:pos+1] != b"\n": + print(f"expected \\n after metalog {name}; aborting\n") + break + pos += 1 + w.writerow([name, value]) + rows += 1 + return rows + + +def main(): + p = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("input", type=Path) + p.add_argument("-o", "--output", type=Path, default=None, help="output prefix (default: input with .launch stripped)") + args = p.parse_args() + + prefix = args.output or args.input.with_suffix("") + flight_path = prefix.with_name(prefix.name + "_flight.csv") + eeprom_path = prefix.with_name(prefix.name + "_eeprom.csv") + meta_path = prefix.with_name(prefix.name + "_meta.csv") + + hdr, meta_blob, bin_blob = read_launch(args.input) + print(f"decoding {hdr['name']} v{hdr['version']} (log_csum 0x{hdr['log_csum']:08x}, eep_csum 0x{hdr['eep_csum']:08x}, meta={len(meta_blob)}B, bin={len(bin_blob)}B)") + + + log_entries, disc_entries, eeprom_data, eeprom_entries, metalog_tail = read_meta(meta_blob) + write_eeprom_csv(eeprom_data, eeprom_entries, eeprom_path) + write_meta_csv(metalog_tail, meta_path) + write_flight_csv(bin_blob, hdr["log_csum"], log_entries, disc_entries, flight_path) + + print(f"Done.\n") + +if __name__ == "__main__": + main() diff --git a/MIDAS/tools/lfd/lfdoffload.py b/MIDAS/tools/lfd/lfdoffload.py new file mode 100644 index 00000000..76d4ce57 --- /dev/null +++ b/MIDAS/tools/lfd/lfdoffload.py @@ -0,0 +1,88 @@ +""" +LFD offload script + +Michael Karpov, 2026 +""" + +import serial +import sys +from enum import IntEnum + +p_s = sys.argv[1] + +port = serial.Serial(p_s, 115200, timeout=0.1) + +class MShellCommand: + class Result(IntEnum): + OK = 0 + ERR_UNSPECIFIED = 1 + ERR_INVALID_CMD = 2 + ERR_NO_CMD = 3 + ERR_INVAL_ARGC = 4 + ERR_INVAL_ARGUMENT = 5 + ERR_INVAL_ARG_RANGE = 6 + ERR_INVAL_FSM = 7 + + def __init__(self, cmd, long=False): + self.cmd = cmd + self.long = long + + def enc(self) -> bytes: + return f"{self.cmd}\n".encode() + +class MShellExecutor: + def __init__(self, cmd_list: list[MShellCommand], port: serial.Serial): + self.cmds = cmd_list + self.port: serial.Serial = port + + class Output: + def __init__(self, cmd_res: MShellCommand.Result, out: str): + self.res = cmd_res + self.out = out + + def get_response(port: serial.Serial) -> Output: + dat = port.read_until(b" ").decode()[:-len(" ")].strip() + rval = port.read_until(b"\n").decode() + return MShellExecutor.Output(MShellCommand.Result(int(rval)), dat) + + + def run(self): + for cmd in self.cmds: + print(f">> {cmd.enc()}") + self.port.write(cmd.enc()) + out = MShellExecutor.get_response(self.port) + print(f"<< {out.out}") + + if(out.res != MShellCommand.Result.OK): + print("ERR: Command failed!") + return + + def run_single(cmd: MShellCommand, port: serial.Serial) -> Output: + print(f">> {cmd.enc()}") + port.write(cmd.enc()) + out = MShellExecutor.get_response(port) + print(f"<< {out.out}") + return out + + +MShellExecutor.run_single(MShellCommand("echo 0"), port) + +port.write(f"lfd {sys.argv[2]}\n".encode()) + +i = 0 +size_out = 0 +debug_buf = bytearray() # capture first chunk to compare against the .launch on disk + +with open(sys.argv[3], "wb+") as outf: + while True: + a = port.read() + outf.write(a) + if len(debug_buf) < 256 and a: + debug_buf.extend(a) + if len(debug_buf) >= 256: + print(f"\n[raw first {len(debug_buf)} bytes from port]: {bytes(debug_buf)!r}") + size_out += len(a) + i += 1 + if i % 100 == 0: + print(f"Outputting to {sys.argv[3]}... {size_out}/? B", end="\r") + diff --git a/MIDAS/tools/lfd/lfdplot.py b/MIDAS/tools/lfd/lfdplot.py new file mode 100644 index 00000000..de0828f8 --- /dev/null +++ b/MIDAS/tools/lfd/lfdplot.py @@ -0,0 +1,43 @@ +""" +python lfdplot.py data8_flight.csv imu.highg_acceleration.ax barometer.altitude + +Michael Karpov, 2026 +""" +import argparse, sys +from pathlib import Path + +import pandas as pd +import matplotlib.pyplot as plt + + +def plot_columns(df, columns, time_col="timestamp_ms"): + fig, axes = plt.subplots(len(columns), 1, sharex=True, figsize=(10, 2.5 * len(columns))) + if len(columns) == 1: + axes = [axes] + t = df[time_col] + for ax, col in zip(axes, columns): + ax.plot(t, df[col], linewidth=0.8) + ax.set_ylabel(col) + ax.grid(True, alpha=0.3) + axes[-1].set_xlabel(time_col) + fig.tight_layout() + return fig + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("csv", type=Path, help="flight csv from lfdconvert") + p.add_argument("columns", nargs="+", help="column names to plot") + args = p.parse_args() + + df = pd.read_csv(args.csv) + missing = [c for c in args.columns if c not in df.columns] + if missing: + print(f"unknown column(s): {missing}\n") + sys.exit(1) + + plot_columns(df, args.columns) + plt.show() + + +if __name__ == "__main__": + main() diff --git a/MIDAS/log_enc.py b/MIDAS/tools/log_enc.py similarity index 100% rename from MIDAS/log_enc.py rename to MIDAS/tools/log_enc.py