From e355f8d9c57201bcc5fe00db0eafb566c2ecc0cf Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Thu, 16 Feb 2023 14:33:47 -0800 Subject: [PATCH 1/2] Improvements on parsing time from tiffs. --- weather_mv/loader_pipeline/sinks.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/weather_mv/loader_pipeline/sinks.py b/weather_mv/loader_pipeline/sinks.py index 47873d2..a4fb5a3 100644 --- a/weather_mv/loader_pipeline/sinks.py +++ b/weather_mv/loader_pipeline/sinks.py @@ -163,8 +163,11 @@ def _get_band_data(i): ds = xr.merge(band_data_list) ds.attrs['is_normalized'] = ds_is_normalized_attr - end_time = None - if initialization_time_regex and forecast_time_regex: + start_time, end_time = None, None + time_attrs_present = 'start_time' in ds.attrs and 'end_time' in ds.attrs + if time_attrs_present: + start_time, end_time = ds.attrs['start_time'], ds.attrs['end_time'] + elif initialization_time_regex and forecast_time_regex: try: start_time = match_datetime(uri, initialization_time_regex) except Exception: @@ -178,15 +181,17 @@ def _get_band_data(i): # TODO(#159): Explore ways to capture required metadata using xarray. with rasterio.open(filename) as f: - datetime_value_ms = None + datetime_value_s = None try: - datetime_value_s = (int(end_time.timestamp()) if end_time is not None - else int(f.tags()[tif_metadata_for_datetime]) / 1000.0) + datetime_value_s = ( + int(end_time.timestamp()) if end_time is not None + else int(f.tags()[tif_metadata_for_datetime]) / 1000.0 + ) ds = ds.assign_coords({'time': datetime.datetime.utcfromtimestamp(datetime_value_s)}) except KeyError: raise RuntimeError(f"Invalid datetime metadata of tif: {tif_metadata_for_datetime}.") except ValueError: - raise RuntimeError(f"Invalid datetime value in tif's metadata: {datetime_value_ms}.") + raise RuntimeError(f"Invalid datetime value in tif's metadata: {datetime_value_s}.") return ds From 88d18f1794c63a5a5070ad12272e20bb65415d92 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Mon, 13 Mar 2023 13:18:13 -0700 Subject: [PATCH 2/2] Flipped logic: Now, what we had before (regex parsing) is the default. --- weather_mv/loader_pipeline/sinks.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/weather_mv/loader_pipeline/sinks.py b/weather_mv/loader_pipeline/sinks.py index a4fb5a3..bf3abe9 100644 --- a/weather_mv/loader_pipeline/sinks.py +++ b/weather_mv/loader_pipeline/sinks.py @@ -164,10 +164,7 @@ def _get_band_data(i): ds.attrs['is_normalized'] = ds_is_normalized_attr start_time, end_time = None, None - time_attrs_present = 'start_time' in ds.attrs and 'end_time' in ds.attrs - if time_attrs_present: - start_time, end_time = ds.attrs['start_time'], ds.attrs['end_time'] - elif initialization_time_regex and forecast_time_regex: + if initialization_time_regex and forecast_time_regex: try: start_time = match_datetime(uri, initialization_time_regex) except Exception: @@ -178,6 +175,8 @@ def _get_band_data(i): raise RuntimeError("Wrong regex passed in --forecast_time_regex.") ds.attrs['start_time'] = start_time ds.attrs['end_time'] = end_time + elif 'start_time' in ds.attrs and 'end_time' in ds.attrs: + start_time, end_time = ds.attrs['start_time'], ds.attrs['end_time'] # TODO(#159): Explore ways to capture required metadata using xarray. with rasterio.open(filename) as f: