Fix nested Antelope parameter history serialization

python/mspasspy/history.py

@@ -1,3 +1,5 @@
+from collections import OrderedDict
+
 import pymongo
 
 
@@ -25,6 +27,18 @@ def get_jobid(db):
         return maxdoc["jobid"] + 1
 
 
+def _antelope_pf_to_dict(pf):
+    """Return one complete AntelopePf node without changing the source."""
+    result = OrderedDict()
+    for simple_key in pf.keys():
+        result[simple_key] = pf.get(simple_key)
+    for table_key in pf.tbl_keys():
+        result[table_key] = pf.get_tbl(table_key)
+    for branch_key in pf.arr_keys():
+        result[branch_key] = _antelope_pf_to_dict(pf.get_branch(branch_key))
+    return result
+
+
 def pfbranch_to_dict(pf, key):
     """
     Recursive function to convert a single branch in an AntelopePf to a python dict.
@@ -49,23 +63,7 @@ def pfbranch_to_dict(pf, key):
     :return:  python dict translation of AntelopePf branch structure
     :raise:  RunTime errors are possible from the ccore methods that are called.
     """
-    brkeys = pf.arr_keys()
-    if len(brkeys) > 0:
-        # This loads simple parameters at this level
-        allbrdata = pf.todict()
-        # This loads any Tbl& data at this level of the hierarchy
-        tblkeys = pf.tbl_keys()
-        for k in tblkeys:
-            tvals = pf.get_tbl(k)
-            allbrdata[k] = tvals
-        for k in brkeys:
-            pfbranch = pf.get_branch(k)
-            bk = pfbranch_to_dict(pfbranch, k)
-            allbrdata[k] = bk
-        return allbrdata
-    else:
-        brdata = pf.todict()
-        return brdata
+    return _antelope_pf_to_dict(pf.get_branch(key))
 
 
 class basic_history_data:
@@ -135,23 +133,7 @@ def __init__(self, job, alg, pf):
         self.jobid = job
         self.algorithm = alg
         self.param_type = "AntelopePf"
-        # This works because Metadata2dict calls pf.keys() which only returns
-        # simple name:value pair parameters.  We use branch and tbl calls later
-        self.params = pf.todict()
-        # tbl's next - simpler than a Arr which requires recursion
-        tblkeys = pf.tbl_keys()
-        for k in tblkeys:
-            tvals = pf.get_tbl(k)
-            # testing suggests tvals is a regular python list so this
-            # should work cleanly
-            self.params[k] = tvals
-        # Arr's are harder if we want to allow them to be arbitrarily deep.
-        # Hence we use this recursive function defined above.
-        arrkeys = pf.arr_keys()
-        for k in arrkeys:
-            branchval = {}
-            branchval = pfbranch_to_dict(pf, k)
-            self.params[k] = branchval
+        self.params = _antelope_pf_to_dict(pf)
 
 
 class HistoryLogger:

python/tests/test_history_pf_contract.py

@@ -0,0 +1,129 @@
+from collections import OrderedDict
+from copy import deepcopy
+
+import pytest
+
+from mspasspy.ccore.utility import AntelopePf
+from mspasspy.ccore.utility import MsPASSError
+from mspasspy.history import pf_history_data, pfbranch_to_dict
+from mspasspy.util.converter import AntelopePf2dict
+
+PF_TEXT = """
+root_value root
+root_integer 7
+root_table &Tbl{
+root row one
+root row two
+}
+selected &Arr{
+    selected_value level_one
+    internal_real 1.5
+    internal_table &Tbl{
+internal row one
+internal row two
+    }
+    middle &Arr{
+        middle_value level_two
+        middle_table &Tbl{
+middle row
+        }
+        leaf &Arr{
+            leaf_value level_three
+            leaf_flag true
+            leaf_table &Tbl{
+leaf row one
+leaf row two
+            }
+        }
+    }
+    selected_sibling &Arr{
+        selected_sibling_value retained
+    }
+}
+unselected &Arr{
+    unselected_value do_not_leak
+    unselected_table &Tbl{
+unrelated row
+    }
+}
+"""
+
+
+SELECTED = {
+    "selected_value": "level_one",
+    "internal_real": 1.5,
+    "internal_table": ["internal row one", "internal row two"],
+    "middle": {
+        "middle_value": "level_two",
+        "middle_table": ["middle row"],
+        "leaf": {
+            "leaf_value": "level_three",
+            "leaf_flag": True,
+            "leaf_table": ["leaf row one", "leaf row two"],
+        },
+    },
+    "selected_sibling": {"selected_sibling_value": "retained"},
+}
+
+
+def _pf(tmp_path):
+    path = tmp_path / "nested_history.pf"
+    path.write_text(PF_TEXT)
+    return AntelopePf(str(path))
+
+
+def test_pfbranch_to_dict_decodes_only_the_requested_branch(tmp_path):
+    pf = _pf(tmp_path)
+    before = deepcopy(AntelopePf2dict(pf))
+
+    result = pfbranch_to_dict(pf, "selected")
+
+    assert result == SELECTED
+    assert type(result) is OrderedDict
+    assert type(result["middle"]) is OrderedDict
+    assert type(result["middle"]["leaf"]) is OrderedDict
+    assert type(result["internal_real"]) is float
+    assert type(result["middle"]["leaf"]["leaf_flag"]) is bool
+    result_keys = list(result)
+    assert set(result_keys[:2]) == {"selected_value", "internal_real"}
+    assert result_keys[2:] == ["internal_table", "middle", "selected_sibling"]
+    assert "unselected" not in result
+    result["internal_table"].append("returned-value mutation")
+    result["middle"]["leaf"]["leaf_table"][0] = "returned-value mutation"
+    assert AntelopePf2dict(pf) == before
+
+
+def test_pfbranch_to_dict_missing_branch_preserves_pf_and_raises(tmp_path):
+    pf = _pf(tmp_path)
+    before = deepcopy(AntelopePf2dict(pf))
+
+    with pytest.raises(MsPASSError, match="missing"):
+        pfbranch_to_dict(pf, "missing")
+
+    assert AntelopePf2dict(pf) == before
+
+
+def test_pf_history_data_preserves_simple_tbl_and_nested_arr_values(tmp_path):
+    pf = _pf(tmp_path)
+    before = deepcopy(AntelopePf2dict(pf))
+
+    history = pf_history_data(17, "nested_algorithm", pf)
+
+    assert history.jobid == 17
+    assert history.algorithm == "nested_algorithm"
+    assert history.param_type == "AntelopePf"
+    assert type(history.params) is OrderedDict
+    assert type(history.params["selected"]) is OrderedDict
+    assert type(history.params["selected"]["middle"]["leaf"]) is OrderedDict
+    assert type(history.params["root_integer"]) is int
+    assert history.params == {
+        "root_value": "root",
+        "root_integer": 7,
+        "root_table": ["root row one", "root row two"],
+        "selected": SELECTED,
+        "unselected": {
+            "unselected_value": "do_not_leak",
+            "unselected_table": ["unrelated row"],
+        },
+    }
+    assert AntelopePf2dict(pf) == before