diff --git a/python/mspasspy/db/normalize.py b/python/mspasspy/db/normalize.py index f773b8cf92..c374915537 100755 --- a/python/mspasspy/db/normalize.py +++ b/python/mspasspy/db/normalize.py @@ -2393,10 +2393,10 @@ class OriginTimeDBMatcher(DatabaseMatcher): source.time - tolerance <= test_time <= source.time + tolerance The test_time value for matching from a datum can come through - one of two methods driven by the constructor argument "time_key". - When time_key is a None (default) the algorithm assumes all input + one of two methods driven by the constructor argument ``data_time_key``. + When ``data_time_key`` is None (default) the algorithm assumes all input are mspass atomic data objects that have the start time defined by - the attribute "t0" (mspass_object.t0). If time_key is a string + the attribute "t0" (mspass_object.t0). If ``data_time_key`` is a string it is assumed to be a Metadata key used to fetch an epoch time to use for the test. The most likely use of that feature would be for ensemble processing where test_time is set as a field in the @@ -2460,6 +2460,15 @@ class OriginTimeDBMatcher(DatabaseMatcher): is not unique. When False find_one returns the first document found and logs a complaint message. (default is False) :type require_unique_match: boolean + + :param data_time_key: data object Metadata key used instead of the + object's ``t0`` to compute the test origin time. The default, None, + uses ``t0``. + :type data_time_key: string or None + + :param source_time_key: source collection field used for the origin-time + interval query. The default, None, selects ``"time"``. + :type source_time_key: string or None """ def __init__( @@ -2489,7 +2498,7 @@ def __init__( self.t0offset = t0offset self.tolerance = tolerance self.data_time_key = data_time_key - self.source_time_key = source_time_key + self.source_time_key = "time" if source_time_key is None else source_time_key if query is None: query = {} if isinstance(query, dict): @@ -2509,21 +2518,21 @@ def query_generator(self, mspass_object) -> dict: This algorithm implements the time test described in detail in docstring for this class. Note the fundamental change in how the test time is computed that depends on the internal (self) - attribute time_key. When None we use the data's t0 attribute. - Otherwise self.time_key is assumed to be a string key to + attribute ``data_time_key``. When None we use the data's t0 attribute. + Otherwise ``self.data_time_key`` is assumed to be a string key to fetch the test time from the object's Metadata container. :param mspass_object: MsPASS defined data object that contains data to be used for this match (t0 attribute or content of - self.time_key). + ``self.data_time_key``). :type mspass_object: Any valid MsPASS data object. :return: query python dictionary on sucess. Return None if a query could not be constructed. That happens two ways here. 1. If the input is not a valid mspass data object or marked dead. - 2. If the time_key algorithm is used and time_key isn't defined - in the input datum. + 2. If the data_time_key algorithm is used and data_time_key isn't + defined in the input datum. """ # This could generate mysterious results if a user messes up # badly, but it makes the code more stable - otherwise @@ -2550,7 +2559,7 @@ def query_generator(self, mspass_object) -> dict: # as python dictionary query = copy.deepcopy(self.query) - query["time"] = { + query[self.source_time_key] = { "$gte": test_time - self.tolerance, "$lte": test_time + self.tolerance, } @@ -2581,10 +2590,10 @@ class OriginTimeMatcher(DataFrameCacheMatcher): source.time - tolerance <= test_time <= source.time + tolerance The test_time value for matching from a datum can come through - one of two methods driven by the constructor argument "time_key". - When time_key is a None (default) the algorithm assumes all input + one of two methods driven by the constructor argument ``data_time_key``. + When ``data_time_key`` is None (default) the algorithm assumes all input are mspass atomic data objects that have the start time defined by - the attribute "t0" (mspass_object.t0). If time_key is a string + the attribute "t0" (mspass_object.t0). If ``data_time_key`` is a string it is assumed to be a Metadata key used to fetch an epoch time to use for the test. The most likely use of that feature would be for ensemble processing where test_time is set as a field in the @@ -2718,6 +2727,16 @@ def __init__( message += "Required for matching with waveform start times" raise MsPASSError(message, ErrorSeverity.Fatal) + def _get_test_time(self, mspass_object): + """Return the configured data time adjusted by ``t0offset``.""" + if self.data_time_key is None: + test_time = mspass_object.t0 + elif mspass_object.is_defined(self.data_time_key): + test_time = mspass_object[self.data_time_key] + else: + return None + return test_time - self.t0offset + def subset(self, mspass_object) -> pd.DataFrame: """ Implementation of subset method requried by inheritance from @@ -2735,16 +2754,9 @@ def subset(self, mspass_object) -> pd.DataFrame: if mspass_object.dead(): return pd.DataFrame() - if self.data_time_key is None: - # this maybe should have a test to assure UTC time standard - # but will defer for now - test_time = mspass_object.t0 - else: - if mspass_object.is_defined(self.data_time_key): - test_time = mspass_object[self.data_time_key] - else: - return pd.DataFrame() - test_time -= self.t0offset + test_time = self._get_test_time(mspass_object) + if test_time is None: + return pd.DataFrame() tmin = test_time - self.tolerance tmax = test_time + self.tolerance @@ -2840,17 +2852,7 @@ def _nearest_time_source(self, mspass_object, mdlist): for md in mdlist: dt[i] = md[time_key] i += 1 - # always use t0 if possile. - # this logic, however, allows mspass_object to be a - # plain Metadata container or a python dictionary - # intentinally let this throw an exception for Metadata if the - # required key is missing. If t0 is not defined it tries to - # use self.data_time_key (normaly "startttme") - if hasattr(mspass_object, "t0"): - test_time = mspass_object.t0 - else: - test_time = mspass_object[self.data_time_key] - test_time -= self.t0offset + test_time = self._get_test_time(mspass_object) dt -= test_time dt = np.abs(dt) component_to_use = np.argmin(dt) diff --git a/python/tests/db/test_origin_time_key_contract.py b/python/tests/db/test_origin_time_key_contract.py new file mode 100644 index 0000000000..75751b61e1 --- /dev/null +++ b/python/tests/db/test_origin_time_key_contract.py @@ -0,0 +1,131 @@ +import pandas as pd +import pytest + +from mspasspy.ccore.seismic import TimeSeries +import mspasspy.db.normalize as normalize_module +from mspasspy.db.normalize import OriginTimeDBMatcher, OriginTimeMatcher + + +def _datum(t0, data_key=None, data_time=None): + datum = TimeSeries(1) + datum.t0 = t0 + datum.set_live() + if data_key is not None: + datum[data_key] = data_time + return datum + + +@pytest.mark.parametrize( + "source_key,data_key", + [("time", None), ("origin_epoch", "pick_epoch")], +) +@pytest.mark.parametrize( + "data_time,tolerance,expected_count", + [(200.0, 0.25, 0), (100.1, 0.25, 1), (101.0, 1.1, 2)], +) +def test_default_and_custom_keys_cover_zero_one_and_multiple_candidates( + source_key, data_key, data_time, tolerance, expected_count +): + sources = pd.DataFrame( + [{source_key: 100.0, "label": "early"}, {source_key: 102.0, "label": "late"}] + ) + matcher = OriginTimeMatcher( + sources, + tolerance=tolerance, + attributes_to_load=[source_key, "label"], + load_if_defined=[], + prepend_collection_name=False, + data_time_key=data_key, + source_time_key=source_key, + ) + datum = _datum( + data_time if data_key is None else 999.0, + data_key=data_key, + data_time=data_time, + ) + + matches, elog = matcher.find(datum) + + if expected_count == 0: + assert matches is None + assert elog is not None + else: + assert elog is None + assert len(matches) == expected_count + + +@pytest.mark.parametrize( + "data_key,t0,data_time,expected_label", + [ + (None, 101.9, None, "t0"), + ("pick_epoch", 101.9, 100.1, "configured-key"), + ], +) +def test_configured_data_time_controls_candidate_filtering_and_nearest_match( + data_key, t0, data_time, expected_label +): + sources = pd.DataFrame( + [ + {"origin_epoch": 100.0, "label": "configured-key"}, + {"origin_epoch": 102.0, "label": "t0"}, + ] + ) + matcher = OriginTimeMatcher( + sources, + tolerance=3.0, + attributes_to_load=["origin_epoch", "label"], + load_if_defined=[], + prepend_collection_name=False, + data_time_key=data_key, + source_time_key="origin_epoch", + ) + datum = _datum(t0, data_key=data_key, data_time=data_time) + + result, elog = matcher.find_one(datum) + + assert elog is None + assert result["label"] == expected_label + + +@pytest.mark.parametrize( + "source_key,data_key,t0,data_time", + [(None, None, 100.0, None), ("origin_epoch", "pick_epoch", 999.0, 100.0)], +) +def test_database_query_uses_configured_source_and_data_keys( + source_key, data_key, t0, data_time +): + matcher = object.__new__(OriginTimeDBMatcher) + matcher.t0offset = 2.0 + matcher.tolerance = 0.5 + matcher.query = {"kind": "event"} + matcher.data_time_key = data_key + matcher.source_time_key = "time" if source_key is None else source_key + datum = _datum(t0, data_key=data_key, data_time=data_time) + + query = matcher.query_generator(datum) + + expected_time = (t0 if data_key is None else data_time) - 2.0 + assert query == { + "kind": "event", + matcher.source_time_key: { + "$gte": expected_time - 0.5, + "$lte": expected_time + 0.5, + }, + } + + +@pytest.mark.parametrize( + "source_time_key,expected", [(None, "time"), ("origin_epoch", "origin_epoch")] +) +def test_database_matcher_constructor_normalizes_the_default_source_key( + monkeypatch, source_time_key, expected +): + monkeypatch.setattr( + normalize_module.DatabaseMatcher, + "__init__", + lambda self, *args, **kwargs: None, + ) + + matcher = OriginTimeDBMatcher(object(), source_time_key=source_time_key) + + assert matcher.source_time_key == expected