Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 36 additions & 34 deletions python/mspasspy/db/normalize.py
Original file line number Diff line number Diff line change
Expand Up @@ -2393,10 +2393,10 @@ class OriginTimeDBMatcher(DatabaseMatcher):
source.time - tolerance <= test_time <= source.time + tolerance

The test_time value for matching from a datum can come through
one of two methods driven by the constructor argument "time_key".
When time_key is a None (default) the algorithm assumes all input
one of two methods driven by the constructor argument ``data_time_key``.
When ``data_time_key`` is None (default) the algorithm assumes all input
are mspass atomic data objects that have the start time defined by
the attribute "t0" (mspass_object.t0). If time_key is a string
the attribute "t0" (mspass_object.t0). If ``data_time_key`` is a string
it is assumed to be a Metadata key used to fetch an epoch time
to use for the test. The most likely use of that feature would be
for ensemble processing where test_time is set as a field in the
Expand Down Expand Up @@ -2460,6 +2460,15 @@ class OriginTimeDBMatcher(DatabaseMatcher):
is not unique. When False find_one returns the first document
found and logs a complaint message. (default is False)
:type require_unique_match: boolean

:param data_time_key: data object Metadata key used instead of the
object's ``t0`` to compute the test origin time. The default, None,
uses ``t0``.
:type data_time_key: string or None

:param source_time_key: source collection field used for the origin-time
interval query. The default, None, selects ``"time"``.
:type source_time_key: string or None
"""

def __init__(
Expand Down Expand Up @@ -2489,7 +2498,7 @@ def __init__(
self.t0offset = t0offset
self.tolerance = tolerance
self.data_time_key = data_time_key
self.source_time_key = source_time_key
self.source_time_key = "time" if source_time_key is None else source_time_key
if query is None:
query = {}
if isinstance(query, dict):
Expand All @@ -2509,21 +2518,21 @@ def query_generator(self, mspass_object) -> dict:
This algorithm implements the time test described in detail in
docstring for this class. Note the fundamental change in how the
test time is computed that depends on the internal (self)
attribute time_key. When None we use the data's t0 attribute.
Otherwise self.time_key is assumed to be a string key to
attribute ``data_time_key``. When None we use the data's t0 attribute.
Otherwise ``self.data_time_key`` is assumed to be a string key to
fetch the test time from the object's Metadata container.

:param mspass_object: MsPASS defined data object that contains
data to be used for this match (t0 attribute or content of
self.time_key).
``self.data_time_key``).
:type mspass_object: Any valid MsPASS data object.

:return: query python dictionary on sucess. Return None if
a query could not be constructed. That happens two ways here.

1. If the input is not a valid mspass data object or marked dead.
2. If the time_key algorithm is used and time_key isn't defined
in the input datum.
2. If the data_time_key algorithm is used and data_time_key isn't
defined in the input datum.
"""
# This could generate mysterious results if a user messes up
# badly, but it makes the code more stable - otherwise
Expand All @@ -2550,7 +2559,7 @@ def query_generator(self, mspass_object) -> dict:
# as python dictionary
query = copy.deepcopy(self.query)

query["time"] = {
query[self.source_time_key] = {
"$gte": test_time - self.tolerance,
"$lte": test_time + self.tolerance,
}
Expand Down Expand Up @@ -2581,10 +2590,10 @@ class OriginTimeMatcher(DataFrameCacheMatcher):
source.time - tolerance <= test_time <= source.time + tolerance

The test_time value for matching from a datum can come through
one of two methods driven by the constructor argument "time_key".
When time_key is a None (default) the algorithm assumes all input
one of two methods driven by the constructor argument ``data_time_key``.
When ``data_time_key`` is None (default) the algorithm assumes all input
are mspass atomic data objects that have the start time defined by
the attribute "t0" (mspass_object.t0). If time_key is a string
the attribute "t0" (mspass_object.t0). If ``data_time_key`` is a string
it is assumed to be a Metadata key used to fetch an epoch time
to use for the test. The most likely use of that feature would be
for ensemble processing where test_time is set as a field in the
Expand Down Expand Up @@ -2718,6 +2727,16 @@ def __init__(
message += "Required for matching with waveform start times"
raise MsPASSError(message, ErrorSeverity.Fatal)

def _get_test_time(self, mspass_object):
"""Return the configured data time adjusted by ``t0offset``."""
if self.data_time_key is None:
test_time = mspass_object.t0
elif mspass_object.is_defined(self.data_time_key):
test_time = mspass_object[self.data_time_key]
else:
return None
return test_time - self.t0offset

def subset(self, mspass_object) -> pd.DataFrame:
"""
Implementation of subset method requried by inheritance from
Expand All @@ -2735,16 +2754,9 @@ def subset(self, mspass_object) -> pd.DataFrame:
if mspass_object.dead():
return pd.DataFrame()

if self.data_time_key is None:
# this maybe should have a test to assure UTC time standard
# but will defer for now
test_time = mspass_object.t0
else:
if mspass_object.is_defined(self.data_time_key):
test_time = mspass_object[self.data_time_key]
else:
return pd.DataFrame()
test_time -= self.t0offset
test_time = self._get_test_time(mspass_object)
if test_time is None:
return pd.DataFrame()

tmin = test_time - self.tolerance
tmax = test_time + self.tolerance
Expand Down Expand Up @@ -2840,17 +2852,7 @@ def _nearest_time_source(self, mspass_object, mdlist):
for md in mdlist:
dt[i] = md[time_key]
i += 1
# always use t0 if possile.
# this logic, however, allows mspass_object to be a
# plain Metadata container or a python dictionary
# intentinally let this throw an exception for Metadata if the
# required key is missing. If t0 is not defined it tries to
# use self.data_time_key (normaly "startttme")
if hasattr(mspass_object, "t0"):
test_time = mspass_object.t0
else:
test_time = mspass_object[self.data_time_key]
test_time -= self.t0offset
test_time = self._get_test_time(mspass_object)
dt -= test_time
dt = np.abs(dt)
component_to_use = np.argmin(dt)
Expand Down
131 changes: 131 additions & 0 deletions python/tests/db/test_origin_time_key_contract.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
import pandas as pd
import pytest

from mspasspy.ccore.seismic import TimeSeries
import mspasspy.db.normalize as normalize_module
from mspasspy.db.normalize import OriginTimeDBMatcher, OriginTimeMatcher


def _datum(t0, data_key=None, data_time=None):
datum = TimeSeries(1)
datum.t0 = t0
datum.set_live()
if data_key is not None:
datum[data_key] = data_time
return datum


@pytest.mark.parametrize(
"source_key,data_key",
[("time", None), ("origin_epoch", "pick_epoch")],
)
@pytest.mark.parametrize(
"data_time,tolerance,expected_count",
[(200.0, 0.25, 0), (100.1, 0.25, 1), (101.0, 1.1, 2)],
)
def test_default_and_custom_keys_cover_zero_one_and_multiple_candidates(
source_key, data_key, data_time, tolerance, expected_count
):
sources = pd.DataFrame(
[{source_key: 100.0, "label": "early"}, {source_key: 102.0, "label": "late"}]
)
matcher = OriginTimeMatcher(
sources,
tolerance=tolerance,
attributes_to_load=[source_key, "label"],
load_if_defined=[],
prepend_collection_name=False,
data_time_key=data_key,
source_time_key=source_key,
)
datum = _datum(
data_time if data_key is None else 999.0,
data_key=data_key,
data_time=data_time,
)

matches, elog = matcher.find(datum)

if expected_count == 0:
assert matches is None
assert elog is not None
else:
assert elog is None
assert len(matches) == expected_count


@pytest.mark.parametrize(
"data_key,t0,data_time,expected_label",
[
(None, 101.9, None, "t0"),
("pick_epoch", 101.9, 100.1, "configured-key"),
],
)
def test_configured_data_time_controls_candidate_filtering_and_nearest_match(
data_key, t0, data_time, expected_label
):
sources = pd.DataFrame(
[
{"origin_epoch": 100.0, "label": "configured-key"},
{"origin_epoch": 102.0, "label": "t0"},
]
)
matcher = OriginTimeMatcher(
sources,
tolerance=3.0,
attributes_to_load=["origin_epoch", "label"],
load_if_defined=[],
prepend_collection_name=False,
data_time_key=data_key,
source_time_key="origin_epoch",
)
datum = _datum(t0, data_key=data_key, data_time=data_time)

result, elog = matcher.find_one(datum)

assert elog is None
assert result["label"] == expected_label


@pytest.mark.parametrize(
"source_key,data_key,t0,data_time",
[(None, None, 100.0, None), ("origin_epoch", "pick_epoch", 999.0, 100.0)],
)
def test_database_query_uses_configured_source_and_data_keys(
source_key, data_key, t0, data_time
):
matcher = object.__new__(OriginTimeDBMatcher)
matcher.t0offset = 2.0
matcher.tolerance = 0.5
matcher.query = {"kind": "event"}
matcher.data_time_key = data_key
matcher.source_time_key = "time" if source_key is None else source_key
datum = _datum(t0, data_key=data_key, data_time=data_time)

query = matcher.query_generator(datum)

expected_time = (t0 if data_key is None else data_time) - 2.0
assert query == {
"kind": "event",
matcher.source_time_key: {
"$gte": expected_time - 0.5,
"$lte": expected_time + 0.5,
},
}


@pytest.mark.parametrize(
"source_time_key,expected", [(None, "time"), ("origin_epoch", "origin_epoch")]
)
def test_database_matcher_constructor_normalizes_the_default_source_key(
monkeypatch, source_time_key, expected
):
monkeypatch.setattr(
normalize_module.DatabaseMatcher,
"__init__",
lambda self, *args, **kwargs: None,
)

matcher = OriginTimeDBMatcher(object(), source_time_key=source_time_key)

assert matcher.source_time_key == expected
Loading