From 1783504d0b8c43ce74e392a3a690a564e930da73 Mon Sep 17 00:00:00 2001 From: Ou Ku Date: Mon, 7 Sep 2026 11:20:26 +0200 Subject: [PATCH 1/2] initial implementation of vce --- arc_estimation.py | 1782 ++++++++++++++++++++++++++++++++++++++ constants.py | 11 + network.py | 2054 ++++++++++++++++++++++++++++++++++++++++++++ stochastic.py | 160 ++++ test_network.py | 531 ++++++++++++ test_stochastic.py | 81 ++ utils.py | 841 ++++++++++++++++++ 7 files changed, 5460 insertions(+) create mode 100644 arc_estimation.py create mode 100644 constants.py create mode 100644 network.py create mode 100644 stochastic.py create mode 100644 test_network.py create mode 100644 test_stochastic.py create mode 100644 utils.py diff --git a/arc_estimation.py b/arc_estimation.py new file mode 100644 index 00000000..5a4d9ad8 --- /dev/null +++ b/arc_estimation.py @@ -0,0 +1,1782 @@ +"""arc estimation algorithms.""" + +from typing import Literal + +import dask.array as da +import numpy as np +import xarray as xr +from scipy.optimize import curve_fit + +import depsi.model_definition as md +import depsi.stats as est +from depsi.utils import get_distance, wrap_phase + +# Constants +STOP_HEIGHT = 1e-4 # Stop search step for height [m] +STOP_VEL = 1e-7 # Stop search step for velocity [m/y] +MAX_COUNT = 10 # Maximum number of search iterations +THRES_TEMP_COH_MEMORY = 500 # unit MB. Threshold to use dask array for temporal coherence calculation in periodogram. + + +def _compute_dd(sd_complex_i, sd_complex_j, sd_quality_i, sd_quality_j): + """Compute the Double-Difference (DD) phase observation given complex time series of points i and j. + + Point i serves as the reference point and is subtracted from point j: + - `sd_complex_conj_i = sd_complex_i.conj()` + - `dd_arc = sd_complex_j * sd_complex_conj_i` + + The function also calculates the diagonal of the Variance-Covariance Matrix (VCM) of the DD phase + (`Q_yy_diagonal_sigma`), which represents the standard deviation (sigma) of the DD phase quality. + + Parameters + ---------- + sd_complex_i : np.ndarray + Complex time series of the reference point i. Shape (n, ). + sd_complex_j : np.ndarray + Complex time series of point j. Shape (n, ). + sd_quality_i : np.ndarray + Quality values (sigma) of the single-difference (SD) phase for point i. Shape (n, ). + sd_quality_j : np.ndarray + Quality values (sigma) of the single-difference (SD) phase for point j. Shape (n, ). + + Returns + ------- + tuple + - dd_arc : np.ndarray + Double-difference (DD) phase observations. Shape (n, ). + - Q_yy_diagonal_sigma : np.ndarray + Diagonal values of the Variance-Covariance Matrix (VCM) of the DD phases. Shape (n, ). + + Example + ------- + >>> dd_arc, Q_yy_diagonal_sigma = _compute_dd(sd_complex_i, sd_complex_j, + sd_quality_i, sd_quality_j) + """ + sd_complex_conj_i = sd_complex_i.conj() # Compute the complex conjugate for the reference point i + dd_arc = sd_complex_j * sd_complex_conj_i + + # Compute the diagonal of the VCM of the dd phases + Q_yy_diagonal_sigma = np.sqrt((sd_quality_i) ** 2 + (sd_quality_j) ** 2) + + return dd_arc, Q_yy_diagonal_sigma + + +def _unwrap_phases_filter(filter_length, arc_dd, phase_arc, jump): + """Filter and unwrap double-difference (DD) arc phase observations in the complex domain. + + This function filters the real and imaginary components of the DD arc observations, computes the filtered phase + (angle), and unwraps the phase based on the filtered function. It simplifies the detection of 2π jumps and applies + corrections to produce an unwrapped phase time series. + + Parameters + ---------- + filter_length : int + Length of the moving average filter used to smooth the real and imaginary components of the arc observation. + arc_dd : np.ndarray + Double-difference arc observations in the complex domain. Shape (n, ). + phase_arc : np.ndarray + Wrapped phase observations for the arc. Shape (n, ). + jump : float + Threshold as a fraction of 2π used to detect and correct phase jumps during unwrapping. + + Returns + ------- + tuple + - phase_arc_unwrap : np.ndarray + Unwrapped phase observations for the arc. Shape (n, ). + - pi_diff : np.ndarray + Integer multiple of 2π differences between the unwrapped filtered phase and the original phase. Shape (n, ). + - filtered_phase_wrap : np.ndarray + Filtered and wrapped phase observations. Shape (n, ). + - filter_real : np.ndarray + Filtered real component of the arc observations. Shape (n, ). + - filter_imag : np.ndarray + Filtered imaginary component of the arc observations. Shape (n, ). + + Notes + ----- + - The function applies a moving average filter to smooth the real and imaginary parts of the input arc observations. + - The filtered phase is used to detect 2π phase jumps, and a correction is applied to unwrap the phase. + - Corrections are based on the difference between the original phase and the unwrapped filtered phase. + """ + ## Filter in Re and Im domain to detect ambiguity levels easily + filter_real = np.convolve(arc_dd.real, np.ones(filter_length) / filter_length, mode="same") + filter_imag = np.convolve(arc_dd.imag, np.ones(filter_length) / filter_length, mode="same") + complex_filtered = filter_real + filter_imag * 1j + filtered_phase_wrap = np.angle(complex_filtered) + + # Detect 2pi jumps in the filtered double difference and calculate shift of time series + abs_diff = np.zeros(len(filtered_phase_wrap) - 1) + for k in range(len(filtered_phase_wrap) - 1): + abs_diff[k] = filtered_phase_wrap[k + 1] - filtered_phase_wrap[k] + + shift = np.zeros(len(arc_dd)) + for k in range(len(abs_diff) - 1): + if abs_diff[k] < -jump * 2 * np.pi or abs_diff[k] > jump * 2 * np.pi: + if abs_diff[k] < -jump * 2 * np.pi: + shift[k + 1 :] = shift[k + 1 :] + 2 * np.pi + if abs_diff[k] > jump * 2 * np.pi: + shift[k + 1 :] = shift[k + 1 :] - 2 * np.pi + else: + shift[k + 1 : 0] = 0 + + # Unwrap the filtered phase function + filtered_phase_unwrap = filtered_phase_wrap + shift + + # Unwrap the DD phase observation for the arc based on the unwraped filtered function + # Correct for integer pi values differences between the filtered unwraped function and the DD observations + pi_diff = np.around((phase_arc - filtered_phase_unwrap) / (2 * np.pi), 0) + phase_arc_unwrap = phase_arc - 2 * np.pi * pi_diff + + return phase_arc_unwrap, pi_diff, filtered_phase_wrap, filter_real, filter_imag + + +def _scipy_fit_partition_2nd_order_bounds(breakpoints, x_data, arc_obs, initial_guess, bounds, vcm, n_max_iter): + """Estimate the parameters for an arc using a partitioned second-order polynomial fit. + + This function splits the time series of the arc observations into multiple partitions at the specified breakpoints. + For each partition, a 2nd order polynomial is fit, and the function ensures that the time series is continuous + at the breakpoints. The fitting process uses the `curve_fit` function with specified bounds and variance-covariance + matrix (vcm). + + Parameters + ---------- + breakpoints : list of int + Indices where the time series is divided into partitions. + x_data : np.ndarray + The input data for the model, typically including time-related variables. Shape (m, n). + arc_obs : np.ndarray + The observed arc, including both real and imaginary parts. Shape (m,). + initial_guess : np.ndarray + Initial guess for the unknown parameters. Shape (n,). + bounds : tuple of (lower_bounds, upper_bounds) + The bounds for the parameters during fitting. Each bound is an array of length n. + vcm : np.ndarray + The variance-covariance matrix of the observations. Shape (m, m). + n_max_iter : np.ndarray + The maximum nr of iterations for non-linear lsq per arc + + Returns + ------- + estimated_params : np.ndarray + The estimated parameters after fitting. Shape (n,). + pcov : np.ndarray + The covariance matrix of the estimated parameters. Shape (n, n). + + """ + + def _model_arc_2nd_order(x_data, *model_params, bkps=breakpoints): + """Model for an arc using a second-order polynomial for each partition in the time series. + + The arc time series is divided into partitions at the specified breakpoints, and for each partition, a 2nd + polynomial is fit. The model accounts for amplitude variations, displacement, cross range, and temperature. + + This function needs to be defined inside _scipy_fit_partition_2nd_order_bounds since the model + uses breakpoints. + + Parameters + ---------- + x_data : np.ndarray + The input data for the model, including time (t), temperature (T), and CR (cr2ph). Shape (m, n). + model_params : list of float + The parameters for the model. These include amplitude, displacement model parameters, and cross range + and temperature. The exact number and order of parameters depend on the number of breakpoints. + bkps : list of int, optional + Breakpoints at which the time series is divided into partitions. Default is breakpoints. + + Returns + ------- + np.ndarray + The modeled arc, which includes both the real and imaginary parts of the arc observations. + Shape (2 * m,). + """ + t, temp, cr2ph = x_data + + # Define the amplitude per partition + aa = model_params[0 : len(bkps)] + + # Define the parameters for the displacement model (third order polynomial) + intercept = model_params[len(bkps)] + p1 = model_params[len(bkps) + 1 : 2 * len(bkps) + 1] + p2 = model_params[2 * len(bkps) + 1 : 3 * len(bkps) + 1] + + # Define parameters for the cross range and temperature + height = model_params[-2] + expansion = model_params[-1] + + # Define the displacement phase and ampltiudes (they vary per partition) + displ = np.zeros(len(t)) + ampl = np.zeros(len(t)) + + # c is the start of a new partition + c = 0 + for i in range(len(bkps)): + # Define till what index the function should go (which is the end of the partition) + idx = int(bkps[i]) + 1 + + # Define the displacement values + displ[c:idx] = intercept - (p1[i] * t[c] + p2[i] * t[c] ** 2) + (p1[i] * t[c:idx] + p2[i] * t[c:idx] ** 2) + ampl[c:idx] = aa[i] + + # Define the 'intercept' of the new partition (that is the end of the next partition) + intercept = displ[idx - 1] + + # c is the starting point of a new partition + c = idx - 1 + + # Define the Real complex observation + real_part = ampl * np.cos(height * cr2ph + expansion * temp + displ) + imag_part = ampl * np.sin(height * cr2ph + expansion * temp + displ) + return np.append(real_part, imag_part) + + estimated_params, pcov, infodict, _, _ = curve_fit( + f=_model_arc_2nd_order, + xdata=x_data, + ydata=arc_obs, + p0=initial_guess, + bounds=bounds, + sigma=vcm, + absolute_sigma=True, + full_output=True, + max_nfev=n_max_iter, + ) + + return estimated_params, pcov + + +def _scipy_fit_partition_2nd_order_bounds_derivative( + breakpoints, x_data, arc_obs, initial_guess, bounds, vcm, n_max_iter +): + """Estimate the parameters for an arc using a partitioned second-order polynomial fit. + + This function splits the time series of the arc observations into multiple partitions at the specified breakpoints. + For each partition, a 2nd order polynomial is fit, and the function ensures that the time series is continuous + at the breakpoints. The fitting process uses the `curve_fit` function with specified bounds and variance-covariance + matrix (vcm). + + Parameters + ---------- + breakpoints : list of int + Indices where the time series is divided into partitions. + x_data : np.ndarray + The input data for the model, typically including time-related variables. Shape (m, n). + arc_obs : np.ndarray + The observed arc, including both real and imaginary parts. Shape (m,). + initial_guess : np.ndarray + Initial guess for the unknown parameters. Shape (n,). + bounds : tuple of (lower_bounds, upper_bounds) + The bounds for the parameters during fitting. Each bound is an array of length n. + vcm : np.ndarray + The variance-covariance matrix of the observations. Shape (m, m). + n_max_iter : np.ndarray + The maximum nr of iterations for non-linear lsq per arc + + Returns + ------- + estimated_params : np.ndarray + The estimated parameters after fitting. Shape (n,). + pcov : np.ndarray + The covariance matrix of the estimated parameters. Shape (n, n). + + """ + + def _model_arc_2nd_order_derivative(x_data, *model_params, bkps=breakpoints): + """Model for an arc using a second-order polynomial for each partition in the time series. + + The arc time series is divided into partitions at the specified breakpoints, and for each partition, a 2nd + polynomial is fit. The model accounts for amplitude variations, displacement, cross range, and temperature. + + This function needs to be defined inside _scipy_fit_partition_2nd_order_bounds since the model + uses breakpoints. + + Parameters + ---------- + x_data : np.ndarray + The input data for the model, including time (`t`), temperature (`T`), and CR (`cr2ph`). Shape (m, n). + model_params : list of float + The parameters for the model. These include amplitude, displacement model parameters, and cross range + and temperature. The exact number and order of parameters depend on the number of breakpoints. + bkps : list of int, optional + Breakpoints at which the time series is divided into partitions. Default is `breakpoints`. + + Returns + ------- + np.ndarray + The modeled arc, which includes both the real and imaginary parts of the arc observations. + Shape (2 * m,). + """ + t, temp, cr2ph = x_data + + nr_bkps = len(bkps) + + # Define the amplitude per partition + aa = model_params[0:nr_bkps] + + # Define the parameters for the displacement model (third order polynomial) + intercept = model_params[nr_bkps] + p1 = np.array(model_params[nr_bkps + 1 : 2 * nr_bkps + 1]) + p2 = np.array(model_params[2 * nr_bkps + 1 : 3 * nr_bkps + 1]) + + # Define parameters for the cross range and temperature + height = model_params[-2] + expansion = model_params[-1] + + # Define the displacement phase and ampltiudes (they vary per partition) + displ = np.zeros(len(t)) + ampl = np.zeros(len(t)) + + # c is the start of a new partition + c = 0 + prev_slope = 0 + + for i in range(nr_bkps): + # Define till what index the function should go (which is the end of the partition) + idx = int(bkps[i]) + 1 + if i > 0: + p1[i] = prev_slope # Zorg dat de eerste afgeleide overeenkomt met de vorige + intercept = displ[c] + + # Define the displacement values + displ[c:idx] = intercept - (p1[i] * t[c] + p2[i] * t[c] ** 2) + (p1[i] * t[c:idx] + p2[i] * t[c:idx] ** 2) + ampl[c:idx] = aa[i] + + # Define the 'intercept' of the new partition (that is the end of the next partition) + intercept = displ[idx - 1] + + prev_slope = p1[i] + 2 * p2[i] * t[idx - 1] + + # c is the starting point of a new partition + c = idx - 1 + + # Define the Real complex observation + real_part = ampl * np.cos(height * cr2ph + expansion * temp + displ) + imag_part = ampl * np.sin(height * cr2ph + expansion * temp + displ) + return np.append(real_part, imag_part) + + estimated_params, pcov, infodict, _, _ = curve_fit( + f=_model_arc_2nd_order_derivative, + xdata=x_data, + ydata=arc_obs, + p0=initial_guess, + bounds=bounds, + sigma=vcm, + absolute_sigma=True, + full_output=True, + max_nfev=n_max_iter, + ) + + return estimated_params, pcov + + +def _model_arc_partitions_2nd_order_phases(x_data, model_params): + """Calculate the forward model of the total, displacement, cross range, and thermal phase for an arc. + + The function divides the time series into different partitions. For each partition, a 2nd order polynomial is used. + + Parameters + ---------- + x_data : tuple + Contains the following elements: + - bkps (list of int): Breakpoints that divide the time series into partitions. + - t (np.ndarray): Time data for the arc. + - temp (np.ndarray): Temperature data for the arc. + - cr2ph (np.ndarray): Cross-range data for the arc. + model_params : list of float + Model parameters for the arc: + - Amplitudes per partition (A). + - Displacement model parameters (intercept, p1, p2). + - Parameters for cross-range (H) and temperature (expansion). + + Returns + ------- + tuple + Contains the following elements: + - phase_total (np.ndarray): The total phase (displacement + cross range + thermal). + - phase_thermal (np.ndarray): The thermal phase. + - phase_cross_range (np.ndarray): The cross-range phase. + - phase_displacement (np.ndarray): The displacement phase. + - real_part (np.ndarray): The real part of the arc observation. + - imag_part (np.ndarray): The imaginary part of the arc observation. + """ + bkps, t, temp, cr2ph = x_data + + # Define the amplitude per partition + aa = model_params[0 : len(bkps)] + + # Define the parameters for the displacement model (third order polynomial) + intercept = model_params[len(bkps)] + p1 = model_params[len(bkps) + 1 : 2 * len(bkps) + 1] + p2 = model_params[2 * len(bkps) + 1 : 3 * len(bkps) + 1] + + # Define parameters for the cross range and temperature + height = model_params[-2] + expansion = model_params[-1] + + # Define the displacement phase and ampltiudes (they vary per partition) + displ = np.zeros(len(t)) + ampl = np.zeros(len(t)) + + # c is the start of a new partition + c = 0 + for i in range(len(bkps)): + # Define till what index the function should go (which is the end of the partition) + idx = int(bkps[i]) + 1 + + # Define the displacement values + displ[c:idx] = intercept - (p1[i] * t[c] + p2[i] * t[c] ** 2) + (p1[i] * t[c:idx] + p2[i] * t[c:idx] ** 2) + ampl[c:idx] = aa[i] + + # Define the 'intercept' of the new partition (that is the end of the next partition) + intercept = displ[idx - 1] + + # c is the starting point of a new partition + c = idx - 1 + + # Define the Real complex observation + real_part = ampl * np.cos(height * cr2ph + expansion * temp + displ) + imag_part = ampl * np.sin(height * cr2ph + expansion * temp + displ) + + # Define the phases + phase_thermal = expansion * temp + phase_cross_range = height * cr2ph + phase_displacement = displ + phase_total = phase_thermal + phase_cross_range + phase_displacement + + return phase_total, phase_thermal, phase_cross_range, phase_displacement, real_part, imag_part + + +def _unwrap_phases(observed_phase, estimated_phase): + """Unwrap the observed phases based on modeled/estimated phase. + + This function corrects phase jumps by adjusting the observed phase values with respect to the estimated phase. + + Parameters + ---------- + observed_phase : np.ndarray + The observed phase values (typically in radians). + estimated_phase : np.ndarray + The estimated or modeled phase values based on the model parameters. + + Returns + ------- + np.ndarray + The unwrapped phase values, which are corrected to account for phase wrapping. + """ + pi_diff = np.around((estimated_phase - observed_phase) / (2 * np.pi), 0) + phase_unwrap = observed_phase + 2 * np.pi * pi_diff + + return phase_unwrap + + +def _compute_residuals_per_partition_stm(y_arc, y_est, Q_dd, bkps): + """Compute the rmse and std of the residuals per partition. + + This function can be used to compare values with a predefined Q matrix. + + Parameters + ---------- + y_arc : np.ndarray + The observed (real) arc values (should be in a flattened array). + y_est : np.ndarray + The estimated arc values (should be in a flattened array). + Q_dd : np.ndarray + The variance-covariance matrix (Q_dd) of the residuals. + bkps : list or np.ndarray + Breakpoints indicating where the arc is divided into different partitions. + + Returns + ------- + tuple + - rmse_partition (list): List of RMSE values computed per partition. + - std_est_partition (list): List of standard deviations of the residuals per partition. + - q_per_partition (list): List of standard deviations from the Q_dd matrix for each partition. + """ + start = 0 + rmse_partition = [] + std_est_partition = [] + q_per_partition = [] + + y_arc = y_arc.flatten() + y_est = y_est.flatten() + + # Loop over the partitions to comput residues per partition + # and add the values to lists + for i in range(len(bkps)): + n = bkps[i] - start + + rmse_s = np.sqrt((np.sum((y_arc[start : bkps[i]] - y_est[start : bkps[i]]) ** 2)) / n) + std_s = np.sqrt(np.var(y_arc[start : bkps[i]] - y_est[start : bkps[i]])) + q_per_s = np.sqrt(Q_dd[start, start]) # the apriori defined quality per partition + + rmse_partition.append(rmse_s) + std_est_partition.append(std_s) + q_per_partition.append(q_per_s) + + start = bkps[i] + + return rmse_partition, std_est_partition, q_per_partition + + +def _flatten_arrays_in_dict(dictionary): + """Flatten arrays in a dictionary. + + Function is required and used in arc_estimation_functions + + Args: + dictionary (dict): dictionary with arrays + + Returns: + dict: dictionary with flattend arrays + """ + dictionary = {key: np.array(value) for key, value in dictionary.items()} + + for key, value in dictionary.items(): + if isinstance(value, np.ndarray): + if value.ndim > 1: # Only flatten the array if the dimension is larger than 1 + dictionary[key] = value.ravel() # get 1D + return dictionary + + +def arc_estimation_xarray_input( + stm_pnt_i, + stm_pnt_j, + bounds, + m2ph, + n_max_iter, + partition_quality_label: str, + x_crd_label: str = "rd_x", + y_crd_label: str = "rd_y", + coordinate_type: Literal["euclidean", "geographic"] = "euclidean", + filter_length_complex=30, + jump_percentage_2pi=0.85, + vcm_complex_method="mad_median", + test_stochastics=False, + print_output=False, +): + """Estimate parameters for the arc defined between the connection point j and control point i. + + This function performs a series of computations for and arc, + including variance-covariance matrix computation, double-difference phase estimation, and parameter fitting + in both the phase and complex domains. The results of these calculations are stored in structured dictionaries. + + Parameters + ---------- + stm_pnt_i : Xarray.Dataset + Input space time matrix for the reference point i + stm_pnt_j : Xarray.Dataset + Input space time matrix for connection point j + bounds : tuple of lists + Bounds for parameter estimation in the format (lower_bounds, upper_bounds). + m2ph : float + Conversion factor from meters to phase. + n_max_iter : np.ndarray + The maximum nr of iterations for non-linear lsq per arc + partition_quality_label: str + Layer name in the STM of the SLC quality + x_crd_label: str, default "rd_x" + Label of the x-coordinate in the STMs (for geographic, this is 'lon') + y_crd_label: str, default "rd_y" + Label of the y-coordinate in the STMs (for geographic, this is 'lat') + coordinate_type: Literal["euclidean", "geographic"], default "euclidean" + Whether to compute distances in Euclidean space (for RD) or geographic distance (for lat/lon) + filter_length_complex : int, optional + Length of the filter for phase unwrapping (default: 30). + jump_percentage_2pi : float, optional + Threshold for unwrapping phase jumps in terms of 2π (default: 0.85). + vcm_complex_method : str, optional + Method for variance-covariance matrix estimation in the complex domain. + Options are "sigma_mean" or "mad_median" (default: "mad_median"). + test_stochastics : bool, optional + Flag for performing stochastic testing (default: False). + print_output : bool, optional + Flag for enabling or disabling print statements (default: False). + + Returns + ------- + results : dict + Dictionary containing results for each arc, including: + - 'unwrap_phases_arc': Unwrapped phases for each arc. + - 'sigma_phases_arc': Phase variances for each arc. + - 'estimated_phase': Estimated phases for each arc. + - 'estimated_displ_phase': Displacement-related phases for each arc. + - 'estimated_thermal': Estimated thermal expansion coefficients. + - 'estimated_cross_range': Estimated cross-range components. + - 'estimated_cross_range_sigma': Uncertainties of cross-range estimates. + - 'estimated_thermal_sigma': Uncertainties of thermal estimates. + - 'estimated_thermal_phase': Thermal-related phases for each arc. + - 'estimated_cross_range_phase': Cross-range related phases for each arc. + - 'cr2ph_arcs': Cross-range-to-phase conversion factors for each arc. + - 'succeeded_arcs': List of arcs where parameter estimation succeeded. + + stochastic_results : dict, optional + Dictionary containing stochastic testing results (if `test_stochastics=True`), including: + - 'q_per_partition': Quality metrics for each partition. + - 'std_residuals_partition': Standard deviations of residuals for each partition. + - 'rmse_residuals_partition': RMSE of residuals for each partition. + - 'mean_sigma_post_arc': Mean post-fit sigma values for each arc. + - 'mean_a_priori_sigma_arc': Mean a priori sigma values for each arc. + - 'arc_length': Lengths of the arcs. + - 'mean_sigma_p_i': Mean quality metrics for the first point in each arc. + - 'mean_sigma_p_j': Mean quality metrics for the second point in each arc. + + Notes + ----- + 1. The function uses deterministic assignment for the CR component, setting it to zero for one of the points. + 2. The estimation process includes fallback mechanisms to handle cases where optimal parameters cannot be found. + 3. Requires external utility functions for phase unwrapping, functional model construction, and lsq estimation. + + Raises + ------ + ValueError + If an unknown `vcm_complex_method` is specified. + RuntimeError, ValueError + If parameter estimation fails for an arc during optimization. + """ + # If we want to do some tests on the stochastics + if test_stochastics: + stochastic_results = { + "q_per_partition": [], + "std_residuals_partition": [], + "rmse_residuals_partition": [], + "mean_sigma_post_arc": [], + "mean_a_priori_sigma_arc": [], + "arc_length": [], + "mean_sigma_p_i": [], + "mean_sigma_p_j": [], + } + + # Dictionary to store results for one arc + results = { + "unwrap_phases_arc": [], + "sigma_phases_arc": [], + "estimated_phase": [], + "estimated_displ_phase": [], + "estimated_thermal": [], + "estimated_cross_range": [], + "estimated_cross_range_sigma": [], + "estimated_thermal_sigma": [], + "estimated_thermal_phase": [], + "estimated_cross_range_phase": [], + "cr2ph_arcs": [], + "succeeded_arcs": [], + } + + print(f"idx pnt i: {int(stm_pnt_i['space'].values)}") + print(f"idx pnt j: {int(stm_pnt_j['space'].values)}") + + dates = stm_pnt_i["time"].values + Btemporal = stm_pnt_i["years_since_first_img"].values + temp = stm_pnt_i["temperature"].values + + # Extract information of the two points of the arc + pnt_i_idx = int(stm_pnt_i["space"].values) + sd_complex_i = stm_pnt_i["sd_complex"].values + slc_quality_i = stm_pnt_i[partition_quality_label].values + bkps_stm_i = stm_pnt_i["breakpoints"].values + sigma_ampl_sd_i = stm_pnt_i["partition_sd_amplitude_sigma"].values + mean_ampl_sd_i = stm_pnt_i["partition_sd_amplitude_mean"].values + mad_ampl_sd_i = stm_pnt_i["partition_sd_mad"].values + median_ampl_sd_i = stm_pnt_i["partition_sd_amplitude_median"].values + + pnt_j_idx = int(stm_pnt_j["space"].values) + sd_complex_j = stm_pnt_j["sd_complex"].values + slc_quality_j = stm_pnt_j[partition_quality_label].values + cr2ph_j = stm_pnt_j["sd_cr2ph"].values + bkps_stm_j = stm_pnt_j["breakpoints"].values + sigma_ampl_sd_j = stm_pnt_j["partition_sd_amplitude_sigma"].values + mean_ampl_sd_j = stm_pnt_j["partition_sd_amplitude_mean"].values + mad_ampl_sd_j = stm_pnt_j["partition_sd_mad"].values + median_ampl_sd_j = stm_pnt_j["partition_sd_amplitude_median"].values + + # Step 1: extract information on the arc + # Compute the arc length + arc_length = get_distance( + [stm_pnt_i[x_crd_label], stm_pnt_i[y_crd_label]], + [stm_pnt_j[x_crd_label], stm_pnt_j[y_crd_label]], + mode=coordinate_type, + ) + + # Combine breakpoints to have breakpoints per arc + bkps_arc_stm = bkps_stm_i + bkps_stm_j + # Define the indexes of the breakpoints for the arc + bkps = [index for index, value in enumerate(bkps_arc_stm) if value > 0] + bkps.append(len(dates) - 1) + + # Compute the cr2ph for the arc, equals to point j since we determinsitcally set the value for point i to zero + cr2ph_arc = cr2ph_j + + # Step 2. Compute the DD phases for the arc + # point i is the reference point and is subtracted from point j: + # Note that the output of Qyy_diagonal are actually sigmas and NO variances. Therefore we need to square the values + dd_arc, Qyy_diagonal = _compute_dd(sd_complex_i, sd_complex_j, slc_quality_i, slc_quality_j) + + # Compute the variance covariance matrix of the DD based on the NMAD for the arc + Qyy = np.identity(len(dates)) * Qyy_diagonal**2 + Qyy_inv = np.linalg.inv(Qyy) + + # Step 3. Estimate parameters in the phase domain + # This step is required to get proper intial estimates for the parameter estimation in the complex domain + # Unwrap the phases based on the filtered real and imaginary part + phase_arc_unwrap, _, _, _, _ = _unwrap_phases_filter( + filter_length_complex, dd_arc, np.angle(dd_arc), jump_percentage_2pi + ) + + # Compute the A matrix. Column order (cross_range, temperature, offset, velocity) must match how + # x_hat_arc_ph is indexed positionally below (and elsewhere in this function). + A_arc = md.construct_design_matrix( + ["cross_range", "temperature", "offset", "velocity"], + m2ph, + cross_range=cr2ph_arc, + temperature=temp, + n_epochs=len(Btemporal), + time=Btemporal, + ) + + # Define the observation vector for the arc, which is based on the 'unwrapped' phase based on the filter + y_arc = np.reshape(phase_arc_unwrap, (len(phase_arc_unwrap), 1)) + + # Estimate parameters in the phase domain + x_hat_arc_ph, Qx_hat_arc_ph = est.blue_q_yy_inv(A_arc, y_arc, Qyy_inv) + + # Step 4. Create VCM in the complex domain + # Here we will compute the VCM for the complex domain. + # It is possible to choose between the mean and sigma or mad and median amplitude per partition. + + # Estimate sigma of the DD phases + if vcm_complex_method == "sigma_mean": + sigma_dd = np.abs(mean_ampl_sd_i * mean_ampl_sd_j) * np.sqrt( + (sigma_ampl_sd_i / mean_ampl_sd_i) ** 2 + (sigma_ampl_sd_j / mean_ampl_sd_j) ** 2 + ) + + elif vcm_complex_method == "mad_median": + sigma_dd = np.abs(median_ampl_sd_i * median_ampl_sd_j) * np.sqrt( + (mad_ampl_sd_i * 1.4826 / median_ampl_sd_i) ** 2 + (mad_ampl_sd_j * 1.4826 / median_ampl_sd_j) ** 2 + ) + else: + raise ValueError( + f"You specified an unknown vcm complex method. The method -- {vcm_complex_method} -- does not exist" + ) + + # Compute VCM in the complex domain + sigma_complex = np.append( + sigma_dd, sigma_dd + ) # Real and Imag are stacked together since we use both of the observations + Q_dd_cmplx = np.identity(len(sigma_complex)) + np.fill_diagonal(Q_dd_cmplx, sigma_complex**2) + + # Step 5. Parameter estimation in the complex domain + # Complex data preparation for the arc + re_arc = dd_arc.real + im_arc = dd_arc.imag + arc_obs = np.append(re_arc, im_arc) + + # Combine all the independent variables in one independent variable + x_data = (bkps, Btemporal, temp, cr2ph_arc) # used in phase estimation + X_data = (Btemporal, temp, cr2ph_arc) # used in curve fit + + # Create arrays with initial values + x0_2_p = np.zeros(3 * len(bkps) + 3) # Create empty array for the bounds + x0_2_p[0 : len(bkps)] = np.ones(len(bkps)) * np.max(re_arc) # The amplitude to be estimated + x0_2_p[len(bkps) + 1] = x_hat_arc_ph[2, 0] # Interception of the dispalcement polynomial + x0_2_p[len(bkps) + 1 : 2 * len(bkps) + 1] = ( + np.ones(len(bkps)) * x_hat_arc_ph[3, 0] + ) # Value related displacement velocity + x0_2_p[2 * len(bkps) + 1 : 3 * len(bkps) + 1] = np.zeros( + len(bkps) + ) # Value related to the second compont of polynomial + x0_2_p[-2] = x_hat_arc_ph[0, 0] # Cross range + x0_2_p[-1] = x_hat_arc_ph[1, 0] # Thermal expansion + + # Define the bounds + ( + A_lower, + a_lower, + b_lower, + c_lower, + CR_lower, + exp_lower, + A_upper, + a_upper, + b_upper, + c_upper, + CR_upper, + exp_upper, + ) = bounds + + # define bounds for second order polynomial with partitions + bounds_upper_2_p = np.ones(len(bkps) * 3 + 3) + bounds_upper_2_p[0 : len(bkps)] = A_upper + bounds_upper_2_p[len(bkps)] = a_upper + bounds_upper_2_p[len(bkps) + 1 : 2 * len(bkps) + 1] = np.ones(len(bkps)) * b_upper + bounds_upper_2_p[2 * len(bkps) + 1 : 3 * len(bkps) + 1] = np.ones(len(bkps)) * c_upper + bounds_upper_2_p[-2] = CR_upper + bounds_upper_2_p[-1] = exp_upper + + bounds_lower_2_p = np.ones(len(bkps) * 3 + 3) + bounds_lower_2_p[0 : len(bkps)] = A_lower + bounds_lower_2_p[len(bkps)] = a_lower + bounds_lower_2_p[len(bkps) + 1 : 2 * len(bkps) + 1] = np.ones(len(bkps)) * b_lower + bounds_lower_2_p[2 * len(bkps) + 1 : 3 * len(bkps) + 1] = np.ones(len(bkps)) * c_lower + bounds_lower_2_p[-2] = CR_lower + bounds_lower_2_p[-1] = exp_lower + + bounds_2_p = (list(bounds_lower_2_p), list(bounds_upper_2_p)) + + # Curvefit with 2nd order displacement polynomial, partitions, and bounds + # Estimate the unknown parameters: + try: + x_hat_2_p_b, pcov_2_p_b = _scipy_fit_partition_2nd_order_bounds( + bkps, + X_data, + arc_obs, + x0_2_p, + bounds_2_p, + Q_dd_cmplx, + n_max_iter, + ) + + except (RuntimeError, ValueError) as e: + print(f"Optimal parameters not found. Skipping arc {(pnt_i_idx, pnt_j_idx)}") + print(f"Encountered error: {e}") + + # Fill everything with nans + ts_length = len(Btemporal) + + for key in [ + "unwrap_phases_arc", + "sigma_phases_arc", + "estimated_phase", + "estimated_displ_phase", + "estimated_thermal_phase", + "estimated_cross_range_phase", + "cr2ph_arcs", + ]: + results[key].append(np.full([ts_length], np.nan)) + for key in [ + "estimated_thermal", + "estimated_cross_range", + "estimated_cross_range_sigma", + "estimated_thermal_sigma", + ]: + results[key].append(np.nan) + + results["succeeded_arcs"].append((np.nan, np.nan)) + + if test_stochastics: + for key in [ + "q_per_partition", + "std_residuals_partition", + "rmse_residuals_partition", + "mean_sigma_post_arc", + ]: + stochastic_results[key].append(np.nan) + stochastic_results["mean_a_priori_sigma_arc"].append(np.mean(Qyy_diagonal)) + stochastic_results["arc_length"].append(arc_length) + stochastic_results["mean_sigma_p_i"].append(np.mean(slc_quality_i)) + stochastic_results["mean_sigma_p_j"].append(np.mean(slc_quality_j)) + + else: + # Estimate the phases: + phase_est_2_p_b, phase_th_2_p_b, phase_cross_range_2_p_b, phase_disp_2_p_b, _, _ = ( + _model_arc_partitions_2nd_order_phases(x_data, x_hat_2_p_b) + ) + # Unwrap the observed phases: + phase_unwrap_2_p_b = _unwrap_phases(np.angle(dd_arc), phase_est_2_p_b) + # Estimate 'residual' phase: + phase_res_2_p_b = phase_unwrap_2_p_b - phase_est_2_p_b + + # Add the results for the arc to the dictionary + results["unwrap_phases_arc"].append(phase_unwrap_2_p_b) + results["sigma_phases_arc"].append(Qyy_diagonal) + results["estimated_phase"].append(phase_est_2_p_b) + results["estimated_displ_phase"].append(phase_disp_2_p_b) + results["estimated_thermal"].append(x_hat_2_p_b[-1] * 1000 / m2ph) + results["estimated_cross_range"].append(x_hat_2_p_b[-2]) + results["estimated_cross_range_sigma"].append(np.sqrt(pcov_2_p_b[-2, -2])) + results["estimated_thermal_sigma"].append(np.sqrt(pcov_2_p_b[-1, -1]) * 1000 / m2ph) + results["estimated_thermal_phase"].append(phase_th_2_p_b) + results["estimated_cross_range_phase"].append(phase_cross_range_2_p_b) + results["cr2ph_arcs"].append(cr2ph_arc) + results["succeeded_arcs"].append((pnt_i_idx, pnt_j_idx)) + # Get the dictionaries in the right shape and format + results = _flatten_arrays_in_dict(results) + + if test_stochastics: + rmse_res_partition, std_res_partition, q_per_part = _compute_residuals_per_partition_stm( + phase_unwrap_2_p_b, phase_est_2_p_b, Qyy, bkps + ) + stochastic_results["q_per_partition"].append(q_per_part) + stochastic_results["std_residuals_partition"].append(std_res_partition) + stochastic_results["rmse_residuals_partition"].append(rmse_res_partition) + stochastic_results["mean_sigma_post_arc"].append(np.std(phase_res_2_p_b)) + stochastic_results["mean_a_priori_sigma_arc"].append(np.mean(Qyy_diagonal)) + stochastic_results["arc_length"].append(arc_length) + stochastic_results["mean_sigma_p_i"].append(np.mean(slc_quality_i)) + stochastic_results["mean_sigma_p_j"].append(np.mean(slc_quality_j)) + + # Get the dictionaries in the right shape and format + stochastic_results = _flatten_arrays_in_dict(stochastic_results) + + # Step 6. Printing + + if print_output: + print( + "Estimated cross_range (phase domain NMAD):", + np.around(x_hat_arc_ph[0, 0], 2), + "+/-", + np.around((np.sqrt(Qx_hat_arc_ph[0, 0])) / (-1 * m2ph), 2), + ) + print( + "Estimated cross_range (2nd order + partitions and bounds):", + np.around(x_hat_2_p_b[-2], 2), + "+/-", + np.around((np.sqrt(pcov_2_p_b[-2, -2])) / (-1 * m2ph), 2), + ) + print( + "Estimated thermal expansion (phase domain NMAD):", + np.around(x_hat_arc_ph[1, 0] * 1000 / m2ph, 4), + "+/-", + np.around(np.sqrt(Qx_hat_arc_ph[1, 1]) * 1000 / m2ph, 2), + ) + print( + "Estimated thermal expansion (2nd order + partitions and bounds):", + np.around(x_hat_2_p_b[-1] * 1000 / m2ph, 4), + np.around(np.sqrt(pcov_2_p_b[-1, -1]) * 1000 / m2ph, 2), + ) + print("") + print("") + + if test_stochastics: + return results, stochastic_results + + else: + return results + + +def arc_estimation_control_network( + arcs_to_analyse, + bounds, + m2ph, + n_max_iter, + Btemporal, + dates, + temp, + sd_complex, + slc_quality, + cr2ph, + ampl_ts, + bkps_stm, + mean_ampl_sd, + sigma_ampl_sd, + mad_ampl_sd, + median_ampl_sd, + x_coordinates, + y_coordinates, + coordinate_type: Literal["euclidean", "geographic"] = "euclidean", + filter_length_complex=30, + jump_percentage_2pi=0.85, + vcm_complex_method="mad_median", + test_stochastics=False, + print_output=False, +): + """Estimate parameters for arcs in a control network based on input time series and geodetic measurements. + + This function performs a series of computations for each arc in the control network, + including variance-covariance matrix computation, double-difference phase estimation, and parameter fitting + in both the phase and complex domains. The results of these calculations are stored in structured dictionaries. + + Parameters + ---------- + arcs_to_analyse : list of tuples + List of arcs, where each arc is defined as a tuple (i, j) representing indices of two points. + bounds : tuple of lists + Bounds for parameter estimation in the format (lower_bounds, upper_bounds). + m2ph : float + Conversion factor from meters to phase. + n_max_iter : np.ndarray + The maximum nr of iterations for non-linear lsq per arc + Btemporal : numpy.ndarray + Array of decimal years corresponding to the time series epochs. + dates : numpy.ndarray + Array of date indices or timestamps corresponding to the time series. + temp : numpy.ndarray + Array of temperature values for thermal expansion modeling. + sd_complex : numpy.ndarray + Complex-valued standard deviations of the signal for all points. + slc_quality : numpy.ndarray + Quality metric for single-look complex (SLC) data. + cr2ph : numpy.ndarray + Cross-range to phase conversion factors for the points. + ampl_ts : numpy.ndarray + Amplitude time series for each point. + bkps_stm : numpy.ndarray + Breakpoints for state transition modeling. + mean_ampl_sd : numpy.ndarray + Mean amplitudes for each point, used in variance modeling. + sigma_ampl_sd : numpy.ndarray + Standard deviations of amplitudes for each point. + mad_ampl_sd : numpy.ndarray + Median absolute deviations (MAD) of amplitudes. + median_ampl_sd : numpy.ndarray + Median amplitudes for each point. + x_coordinates : numpy.ndarray + X-coordinates of the points in the control network. + y_coordinates : numpy.ndarray + Y-coordinates of the points in the control network. + coordinate_type: Literal["euclidean", "geographic"], default "euclidean" + Whether the given coordinates are in Euclidean space (such as RD) or in geographic space (such as lon/lat) + filter_length_complex : int, optional + Length of the filter for phase unwrapping (default: 30). + jump_percentage_2pi : float, optional + Threshold for unwrapping phase jumps in terms of 2π (default: 0.85). + vcm_complex_method : str, optional + Method for variance-covariance matrix estimation in the complex domain. + Options are "sigma_mean" or "mad_median" (default: "mad_median"). + test_stochastics : bool, optional + Flag for performing stochastic testing (default: False). + print_output : bool, optional + Flag for enabling or disabling print statements (default: False). + + Returns + ------- + results : dict + Dictionary containing results for each arc, including: + - 'unwrap_phases_arc': Unwrapped phases for each arc. + - 'sigma_phases_arc': Phase variances for each arc. + - 'estimated_phase': Estimated phases for each arc. + - 'estimated_displ_phase': Displacement-related phases for each arc. + - 'estimated_thermal': Estimated thermal expansion coefficients. + - 'estimated_cross_range': Estimated cross-range components. + - 'estimated_cross_range_sigma': Uncertainties of cross-range estimates. + - 'estimated_thermal_sigma': Uncertainties of thermal estimates. + - 'estimated_thermal_phase': Thermal-related phases for each arc. + - 'estimated_cross_range_phase': Cross-range related phases for each arc. + - 'cr2ph_arcs': Cross-range-to-phase conversion factors for each arc. + - 'succeeded_arcs': List of arcs where parameter estimation succeeded. + + stochastic_results : dict, optional + Dictionary containing stochastic testing results (if `test_stochastics=True`), including: + - 'q_per_partition': Quality metrics for each partition. + - 'std_residuals_partition': Standard deviations of residuals for each partition. + - 'rmse_residuals_partition': RMSE of residuals for each partition. + - 'mean_sigma_post_arc': Mean post-fit sigma values for each arc. + - 'mean_a_priori_sigma_arc': Mean a priori sigma values for each arc. + - 'arc_length': Lengths of the arcs. + - 'mean_sigma_p_i': Mean quality metrics for the first point in each arc. + - 'mean_sigma_p_j': Mean quality metrics for the second point in each arc. + + Notes + ----- + 1. The function uses deterministic assignment for the CR component, setting it to zero for one of the points. + 2. The estimation process includes fallback mechanisms to handle cases where optimal parameters cannot be found. + 3. Requires external utility functions for phase unwrapping, functional model construction, and lsq estimation. + + Raises + ------ + ValueError + If an unknown `vcm_complex_method` is specified. + RuntimeError, ValueError + If parameter estimation fails for an arc during optimization. + """ + # If we want to do some tests on the stochastics + if test_stochastics: + stochastic_results = { + "q_per_partition": [], + "std_residuals_partition": [], + "rmse_residuals_partition": [], + "mean_sigma_post_arc": [], + "mean_a_priori_sigma_arc": [], + "arc_length": [], + "mean_sigma_p_i": [], + "mean_sigma_p_j": [], + } + + # Dictionary to store results + results = { + "unwrap_phases_arc": [], + "sigma_phases_arc": [], + "estimated_phase": [], + "estimated_displ_phase": [], + "estimated_thermal": [], + "estimated_cross_range": [], + "estimated_cross_range_sigma": [], + "estimated_thermal_sigma": [], + "estimated_thermal_phase": [], + "estimated_cross_range_phase": [], + "cr2ph_arcs": [], + "succeeded_arcs": [], + } + + # Counting needed for saving data + p = 0 + + for a in arcs_to_analyse: + pnt_i_idx, pnt_j_idx = a + + print(f"idx pnt i: {pnt_i_idx}") + print(f"idx pnt j: {pnt_j_idx}") + + # Extract information of the two points of the arc + sd_complex_i = sd_complex[pnt_i_idx, :] + slc_quality_i = slc_quality[pnt_i_idx, :] + ampl_i = ampl_ts[pnt_i_idx, :] + bkps_stm_i = bkps_stm[pnt_i_idx, :] + sigma_ampl_sd_i = sigma_ampl_sd[pnt_i_idx, :] + mean_ampl_sd_i = mean_ampl_sd[pnt_i_idx, :] + mad_ampl_sd_i = mad_ampl_sd[pnt_i_idx, :] + median_ampl_sd_i = median_ampl_sd[pnt_i_idx, :] + + sd_complex_j = sd_complex[pnt_j_idx, :] + slc_quality_j = slc_quality[pnt_j_idx, :] + cr2ph_j = cr2ph[pnt_j_idx] + bkps_stm_j = bkps_stm[pnt_j_idx, :] + sigma_ampl_sd_j = sigma_ampl_sd[pnt_j_idx, :] + mean_ampl_sd_j = mean_ampl_sd[pnt_j_idx, :] + mad_ampl_sd_j = mad_ampl_sd[pnt_j_idx, :] + median_ampl_sd_j = median_ampl_sd[pnt_j_idx, :] + + # Step 1. Extract information for the ARC + # Compute the arc length + + arc_length = get_distance( + [x_coordinates[pnt_i_idx], y_coordinates[pnt_i_idx]], + [x_coordinates[pnt_j_idx], y_coordinates[pnt_j_idx]], + mode=coordinate_type, + ) + + # Extract the breakpoints for the arc + bkps_arc_stm = bkps_stm_i + bkps_stm_j + + # Define the indexes of the breakpoints for the arc + bkps = [index for index, value in enumerate(bkps_arc_stm) if value > 0] + bkps.append(len(dates) - 1) + + # The value for the cross range component is cr2ph of point j, since we deterministically set the + # cross-range component of point i to zero + cr2ph_arc = cr2ph_j + + # Step 2. Compute the DD phases for the arc + # point i is the reference point and is subtracted from point j: + dd_arc, Q_yy_diagonal = _compute_dd(sd_complex_i, sd_complex_j, slc_quality_i, slc_quality_j) + + # Compute the variance covariance matrix of the DD based on the NMAD for the arc + Q_yy = np.identity(len(dates)) * Q_yy_diagonal**2 + Q_yy_inv = np.linalg.inv(Q_yy) + + # Step 3. Estimate parameters in the phase domain + # Step is required to get proper intial estimates for the parameter estimation in the complex domain + # Unwrap the phases based on the filtered real and imaginary part + phase_arc_unwrap, _, _, _, _ = _unwrap_phases_filter( + filter_length_complex, dd_arc, np.angle(dd_arc), jump_percentage_2pi + ) + + # Compute the A matrix. Column order (cross_range, temperature, offset, velocity) must match how + # x_hat_arc_ph is indexed positionally below (and elsewhere in this function). + A_arc = md.construct_design_matrix( + ["cross_range", "temperature", "offset", "velocity"], + m2ph, + cross_range=cr2ph_arc, + temperature=temp, + n_epochs=len(Btemporal), + time=Btemporal, + ) + + # Define the observation vector for the arc, which is based on the 'unwrapped' phase based on the filter + y_arc = np.reshape(phase_arc_unwrap, (len(phase_arc_unwrap), 1)) + + # Estimate parameters in the phase domain + x_hat_arc_ph, Q_x_hat_arc_ph = est.blue_q_yy_inv(A_arc, y_arc, Q_yy_inv) + + # Step 4. VCM in the complex domain + # Here we will compute the VCM for the complex domain. + # It is possible to choose between the mean and sigma or mad and median amplitude per partition. + + # Estimate the DD sigma + if vcm_complex_method == "sigma_mean": + sigma_dd = np.abs(mean_ampl_sd_i * mean_ampl_sd_j) * np.sqrt( + (sigma_ampl_sd_i / mean_ampl_sd_i) ** 2 + (sigma_ampl_sd_j / mean_ampl_sd_j) ** 2 + ) + + elif vcm_complex_method == "mad_median": + sigma_dd = np.abs(median_ampl_sd_i * median_ampl_sd_j) * np.sqrt( + (mad_ampl_sd_i * 1.4826 / median_ampl_sd_i) ** 2 + (mad_ampl_sd_j * 1.4826 / median_ampl_sd_j) ** 2 + ) + else: + raise ValueError( + f"You specified an unknown vcm complex method. The method -- {vcm_complex_method} -- does not exist" + ) + + # Compute VCM in the complex domain + sigma_complex = np.append( + sigma_dd, sigma_dd + ) # Real and Imag are stacked together since we use both of the observations + Q_dd_cmplx = np.identity(len(sigma_complex)) + np.fill_diagonal(Q_dd_cmplx, sigma_complex**2) + + # Step 5. Parameter estimation in complex domain + # Complex data preparation for the arc + re_arc = dd_arc.real + im_arc = dd_arc.imag + arc_obs = np.append(re_arc, im_arc) + + # Combine all the independent variables in one independent variable + x_data = bkps, Btemporal, temp, cr2ph_arc + xx_data = Btemporal, temp, cr2ph_arc + + # Create initial value arrays + x0_2_p = np.zeros(3 * len(bkps) + 3) # Create empty array for the bounds + x0_2_p[0 : len(bkps)] = np.ones(len(bkps)) * np.max(re_arc) # The amplitude to be estimated + x0_2_p[len(bkps) + 1] = x_hat_arc_ph[ + 2, 0 + ] # Interception of the dispalcement polynomial. We use estimated values in the phase domain + x0_2_p[len(bkps) + 1 : 2 * len(bkps) + 1] = ( + np.ones(len(bkps)) * x_hat_arc_ph[3, 0] + ) # Value related displacement velocity in the displacement polynomial + x0_2_p[2 * len(bkps) + 1 : 3 * len(bkps) + 1] = np.zeros( + len(bkps) + ) # Value related to the second compont of the dispalcement polynomial + x0_2_p[-2] = x_hat_arc_ph[0, 0] # Cross range + x0_2_p[-1] = x_hat_arc_ph[1, 0] # Thermal expansion + + # Define bounds + ( + amp_lower, + a_lower, + b_lower, + c_lower, + cr_lower, + exp_lower, + amp_upper, + a_upper, + b_upper, + c_upper, + cr_upper, + exp_upper, + ) = bounds + + # define bounds for second order polynomial with partitions + bounds_upper_2_p = np.ones(len(bkps) * 3 + 3) + bounds_upper_2_p[0 : len(bkps)] = amp_upper + bounds_upper_2_p[len(bkps)] = a_upper + bounds_upper_2_p[len(bkps) + 1 : 2 * len(bkps) + 1] = np.ones(len(bkps)) * b_upper + bounds_upper_2_p[2 * len(bkps) + 1 : 3 * len(bkps) + 1] = np.ones(len(bkps)) * c_upper + bounds_upper_2_p[-2] = cr_upper + bounds_upper_2_p[-1] = exp_upper + + bounds_lower_2_p = np.ones(len(bkps) * 3 + 3) + bounds_lower_2_p[0 : len(bkps)] = amp_lower + bounds_lower_2_p[len(bkps)] = a_lower + bounds_lower_2_p[len(bkps) + 1 : 2 * len(bkps) + 1] = np.ones(len(bkps)) * b_lower + bounds_lower_2_p[2 * len(bkps) + 1 : 3 * len(bkps) + 1] = np.ones(len(bkps)) * c_lower + bounds_lower_2_p[-2] = cr_lower + bounds_lower_2_p[-1] = exp_lower + + bounds_2_p = (list(bounds_lower_2_p), list(bounds_upper_2_p)) + + # Estiamte parameters in the complex domain with a 2nd order displacement polynomial + # bounds and partitions + try: + x_hat_2_p_b, pcov_2_p_b = _scipy_fit_partition_2nd_order_bounds( + bkps, xx_data, arc_obs, x0_2_p, bounds_2_p, Q_dd_cmplx, n_max_iter + ) + except (RuntimeError, ValueError): + print(f"Optimal parameters not found. Skipping arc {(pnt_i_idx, pnt_j_idx)}") + + # Fill everything with nans + ts_length = len(ampl_i) + + for key in [ + "unwrap_phases_arc", + "sigma_phases_arc", + "estimated_phase", + "estimated_displ_phase", + "estimated_thermal_phase", + "estimated_cross_range_phase", + "cr2ph_arcs", + ]: + results[key].append(np.full([ts_length], np.nan)) + for key in [ + "estimated_thermal", + "estimated_cross_range", + "estimated_cross_range_sigma", + "estimated_thermal_sigma", + ]: + results[key].append(np.nan) + + results["succeeded_arcs"].append((np.nan, np.nan)) + + if test_stochastics: + for key in [ + "q_per_partition", + "std_residuals_partition", + "rmse_residuals_partition", + "mean_sigma_post_arc", + ]: + stochastic_results[key].append(np.nan) + stochastic_results["mean_a_priori_sigma_arc"].append(np.mean(Q_yy_diagonal)) + stochastic_results["arc_length"].append(arc_length) + stochastic_results["mean_sigma_p_i"].append(np.mean(slc_quality_i)) + stochastic_results["mean_sigma_p_j"].append(np.mean(slc_quality_j)) + + else: + # Estimate the phases: + ( + phase_est_2_p_b, + phase_th_2_p_b, + phase_cross_range_2_p_b, + phase_disp_2_p_b, + _, + _, + ) = _model_arc_partitions_2nd_order_phases(x_data, x_hat_2_p_b) + # Unwrap the observed phases: + phase_unwrap_2_p_b = _unwrap_phases(np.angle(dd_arc), phase_est_2_p_b) + # Estimate 'residual' phase: + phase_res_2_p_b = phase_unwrap_2_p_b - phase_est_2_p_b + + # Add the results for the arc to the dictionary + results["unwrap_phases_arc"].append(phase_unwrap_2_p_b) + results["sigma_phases_arc"].append(Q_yy_diagonal) + results["estimated_phase"].append(phase_est_2_p_b) + results["estimated_displ_phase"].append(phase_disp_2_p_b) + results["estimated_thermal"].append(x_hat_2_p_b[-1] * 1000 / m2ph) + results["estimated_cross_range"].append(x_hat_2_p_b[-2]) + results["estimated_cross_range_sigma"].append(np.sqrt(pcov_2_p_b[-2, -2])) + results["estimated_thermal_sigma"].append(np.sqrt(pcov_2_p_b[-1, -1]) * 1000 / m2ph) + results["estimated_thermal_phase"].append(phase_th_2_p_b) + results["estimated_cross_range_phase"].append(phase_cross_range_2_p_b) + results["cr2ph_arcs"].append(cr2ph_arc) + results["succeeded_arcs"].append((pnt_i_idx, pnt_j_idx)) + # Get the dictionaries in the right shape and format + # results = flatten_arrays_in_dict(results) + + if test_stochastics: + rmse_res_partition, std_res_partition, q_per_part = _compute_residuals_per_partition_stm( + phase_unwrap_2_p_b, phase_est_2_p_b, Q_yy, bkps + ) + stochastic_results["q_per_partition"].append(q_per_part) + stochastic_results["std_residuals_partition"].append(std_res_partition) + stochastic_results["rmse_residuals_partition"].append(rmse_res_partition) + stochastic_results["mean_sigma_post_arc"].append(np.std(phase_res_2_p_b)) + stochastic_results["mean_a_priori_sigma_arc"].append(np.mean(Q_yy_diagonal)) + stochastic_results["arc_length"].append(arc_length) + stochastic_results["mean_sigma_p_i"].append(np.mean(slc_quality_i)) + stochastic_results["mean_sigma_p_j"].append(np.mean(slc_quality_j)) + + # Get the dictionaries in the right shape and format + # stochastic_results = flatten_arrays_in_dict(stochastic_results) + + if print_output: + print( + "Estimated cross_range (phase domain NMAD):", + np.around(x_hat_arc_ph[0, 0], 2), + "+/-", + np.around((np.sqrt(Q_x_hat_arc_ph[0, 0])) / (-1 * m2ph), 2), + ) + print( + "Estimated cross_range (2nd order + partitions and bounds):", + np.around(x_hat_2_p_b[-2], 2), + "+/-", + np.around((np.sqrt(pcov_2_p_b[-2, -2])) / (-1 * m2ph), 2), + ) + print( + "Estimated thermal expansion (phase domain NMAD):", + np.around(x_hat_arc_ph[1, 0] * 1000 / m2ph, 4), + "+/-", + np.around(np.sqrt(Q_x_hat_arc_ph[1, 1]) * 1000 / m2ph, 2), + ) + print( + "Estimated thermal expansion (2nd order + partitions and bounds):", + np.around(x_hat_2_p_b[-1] * 1000 / m2ph, 4), + np.around(np.sqrt(pcov_2_p_b[-1, -1]) * 1000 / m2ph, 2), + ) + print("") + print("") + + p = p + 1 + + # tranform from list to np array + results = {key: np.array(value) for key, value in results.items()} + + if test_stochastics: + return results, stochastic_results + + return results + + +def periodogram( + stm: xr.Dataset, + key_dphase: str, + key_h2ph: str, + key_Btemporal: str, + std_obs: float = 1.0, + std_height: float = 50.0, + std_vel: float = 0.02, + init_height: float = 0.0, + init_vel: float = 0.0, + init_step_height: float = 3.0, + init_step_vel: float = 2e-3, + min_steps: int = 10, +): + """Periodogram algorithm. + + This function performs periodogram unwrapping on arcs. + + It uses a deformation model with two parameters: height and velocity to estimate the unwrapped phase. + + For computation efficiency, the design matrix is constructed only once for all arcs, utilizing the average + height-to-phase conversion factor (h2ph) across all arcs. The effect of using this average is corrected later. + + Parameters + ---------- + stm : xr.Dataset + Input Space-Time Matrix (STM) containing the wrapped phase, height-to-phase conversion factor, and year-time. + key_dphase : str + Key for the wrapped differential phase data variable in the STM. + key_h2ph : str + Key for the height-to-phase conversion factor in the STM. + key_Btemporal : str + Key for the temporal baseline in the STM. + The value should be in decimal years. + std_obs : float, optional + A-priori standard deviation of the observations in rads, by default 1.0. + This value is used to construct the stochastic model (Qyy) of the observations. + std_height : float, optional + A-priori standard deviation of the height in meters, by default 50.0. + This value is used to construct the boundaries of the initial search space for the height parameter. + std_vel : float, optional + A-priori standard deviation of the velocity in meters per year, by default 0.02. + This value is used to construct the boundaries of the initial search space for the velocity parameter. + init_height : float, optional + Initial value for the height parameter in meters, by default 0.0. + init_vel : float, optional + Initial value for the velocity parameter in meters per year, by default 0.0. + init_step_height : float, optional + Initial step size for the height parameter in meters, by default 3.0. + This value sets the resolution of the initial search space for the height parameter. + After every search, the step size will be reduced by a factor of 10. + init_step_vel : float, optional + Initial step size for the velocity parameter in meters per year, by default 2e-3. + This value sets the resolution of the initial search space for the velocity parameter. + After every search, the step size will be reduced by a factor of 10. + min_steps : int, optional + Minimum number of steps in the search space for the height and velocity parameters, by default 10. + If the number of steps in the initial search space is smaller than this value, it will be set to this value. + After the first search, the number of steps will be set to this value. + + Returns + ------- + Tuple[xr.DataArray, xr.DataArray, xr.DataArray, xr.DataArray, xr.DataArray] + Returns the unwrapped phase, ambiguities, estimated height, estimated velocity, and temporal coherence. + - Unwrapped phase: in rads, shape (n_arcs, n_obs), dtype np.float64. + - Ambiguities: unitless, shape (n_arcs, n_obs), dtype np.float64. + - Estimated height: in meters, shape (n_arcs,), dtype np.float64. + - Estimated velocity: in meters per year, shape (n_arcs,), dtype np.float64. + - Temporal coherence: unitless float number, norm of the complex coherence, scalar, dtype np.float64. + """ + # Compute m2ph (meters to phase) conversion factor from wavelength + if "wavelength" not in stm.attrs: + raise ValueError( + "Wavelength is not provided and not found in attributes of STM." + "Please make sure it is provided." + "For example: stm = stm.assign_attrs({'wavelength': wavelength})" + ) + wavelength = stm.attrs["wavelength"] + m2ph = -4 * np.pi / wavelength + + # Make sure year time only contains the time dimension + assert (len(stm[key_Btemporal].dims) == 1) and ("time" in stm[key_Btemporal].dims), ( + "year time should and only should contain the 'time' dimension." + ) + + # Load year time in memory + Btemporal = stm[key_Btemporal].values + + # Set up functional and stochastic model for all arcs + # Here we use the same h2ph (average over all arcs) for all arcs and correct the effect later + # Doing this avoids perform matrix inversion for each arc + h2ph_approx = stm[key_h2ph].mean(dim="space").values # Mean h2ph of all arcs + + # Design matrix B, size n_obs x n_params + # In B, h2ph should also be multiplied by m2ph since it did not when it was created + B = np.stack([h2ph_approx * m2ph, Btemporal * m2ph]).T + + # Stochastic model Qyy, size n_obs x n_obs + # This is the covariance matrix of the observations + n_obs = stm[key_dphase].sizes["time"] # number of observations + Qyy = np.diag(np.repeat(std_obs**2, n_obs)) + + # Normal matrix N and rhs for the least squares solution + N = B.T @ np.linalg.inv(Qyy) @ B # B.T * Qyy^-1 * B , size n_params x n_params + # Solve N * x = B.T * Qyy^-1, then rhs = N^-1 * B.T * Qyy^-1 + rhs = np.linalg.inv(N) @ B.T @ np.linalg.inv(Qyy) + + # check if the time dimension is not chunked, and unchunk it if necessary + if "time" in stm.chunks.keys(): + if len(stm.chunks["time"]) != 1: + stm = stm.chunk({"time": -1}) + + # Build initial search space for height and velocity + n_steps_height = max(round(2 * std_height / init_step_height), min_steps) + n_steps_vel = max(round(2 * std_vel / init_step_vel), min_steps) + init_search_space = _build_periodogram_search_space( + init_height, init_vel, init_step_height, init_step_vel, n_steps_height, n_steps_vel + ) + + # Perform one search for all arcs, and get the best initial estimates for height and velocity per arc + # This is motivated by the fact that the initial search space is the largest, and can be vectorized for all arcs + # First iteration, candidate modeled phases are identical for all arcs + # The residuals phase_residual_all_arcs is a large array with n_arcs x n_obs x n_search + # so use .data to avoid loading it into memory if it is a dask array + dphase_obs = stm[key_dphase].data # n_arcs x n_obs x 1 + + # If the memory size of phase_residual_all_arcs will exceed the threshold + # chunk dphase_obs and init_search_space to enable computation + mem_size_estimation = ( + dphase_obs.shape[0] * dphase_obs.shape[1] * init_search_space.shape[0] * dphase_obs.dtype.itemsize + ) / (1024**2) # in MB + if mem_size_estimation > THRES_TEMP_COH_MEMORY: + dphase_obs, init_search_space = _chunk_for_temp_coh_compute(dphase_obs, init_search_space) + + # Compute modelled phase for all arcs and all search candidates + phs_model = B @ init_search_space.T # n_obs x n_search + + # Expand dimensions and compute the phase residuals for all arcs and all search candidates + dphase_obs = dphase_obs[:, :, None] # n_arcs x n_obs x 1 + phs_model = phs_model[None, :, :] # 1 x n_obs x n_search + phase_residual_all_arcs = dphase_obs - phs_model + + # Find the best initial height and velocity based on the temporal coherence + coh_search_space_all_arcs = ( + np.cos(phase_residual_all_arcs).sum(axis=1) + 1j * np.sin(phase_residual_all_arcs).sum(axis=1) + ) / stm[key_dphase].sizes["time"] # n_arcs x n_search + coh_idx_all_arcs = np.argmax(np.abs(coh_search_space_all_arcs), axis=1) # n_arcs + + # Implicitly compute best coh index if dask array + coh_idx_all_arcs = coh_idx_all_arcs.compute() if isinstance(coh_idx_all_arcs, da.Array) else coh_idx_all_arcs + + # Build xr.DataArray for the initial height and velocity of all arcs + da_init_height_all_arcs = xr.DataArray( + init_search_space[coh_idx_all_arcs, 0], + dims=["space"], + ) + da_init_vel_all_arcs = xr.DataArray( + init_search_space[coh_idx_all_arcs, 1], + dims=["space"], + ) + + # Apply the _periodogram_arc on stm[key_dphase] along "space" dimension + # Set up input core dimensions, which are the dimensions _periodogram_arc will be applied to + # We are broadcasting _periodogram_arc on stm[key_dphase] and stm[key_h2ph] along the space dimension + # The height and velocity are scalars + # Therefore, we are only calling it on the "time" dimension for the first two parameters + # So we have the input_core_dims as [["time"], ["time"], [], []] + input_core_dims = [["time"], ["time"], [], []] + + # There are 5 outputs from _periodogram_arc + # The first two are np arrays with time dimension + # The other three are scalars, so they have no dimensions + output_core_dims = [["time"], ["time"], [], [], []] + + results = xr.apply_ufunc( + _periodogram_arc, + stm[key_dphase], + stm[key_h2ph], + da_init_height_all_arcs, + da_init_vel_all_arcs, + input_core_dims=input_core_dims, + output_core_dims=output_core_dims, + kwargs={ + "h2ph_approx": h2ph_approx, + "B": B, + "Qyy": Qyy, + "N": N, + "rhs": rhs, + "init_step_height": init_step_height, + "init_step_vel": init_step_vel, + "min_steps": min_steps, + }, + vectorize=True, + dask="parallelized", + output_dtypes=[np.float64, np.float64, np.float64, np.float64, np.float64], + ) + + return results + + +def _periodogram_arc( + phs_obs_wrapped: np.ndarray, + h2ph: np.ndarray, + init_height: float, + init_vel: float, + h2ph_approx: np.ndarray, + B: np.ndarray, + Qyy: np.ndarray, + N: np.ndarray, + rhs: np.ndarray, + init_step_height: float, + init_step_vel: float, + min_steps: float, +): + """Periodogram unwrapping for a single arc. + + Parameters + ---------- + phs_obs_wrapped : np.ndarray + Wrapped phase observations in radians, shape (n_obs,). + h2ph : np.ndarray: + Height-to-phase factor of the arc, shape (n_obs,). + init_height : float + Initial value for the height parameter in meters. + init_vel : float + Initial value for the velocity parameter in meters per year. + h2ph_approx : np.ndarray + Approximate height-to-phase factor calculated by spatial average of all h2ph, shape (n_obs,). + B : np.ndarray + Design matrix, size n_obs x n_params, where n_params = 2 (height and velocity). + Qyy : np.ndarray + Stochastic model of the observations, size n_obs x n_obs. + N : np.ndarray + Normal matrix, size n_params x n_params. + rhs : np.ndarray + Right-hand side matrix for the least squares solution, size n_params x n_obs. + init_step_height : float + Initial step size for the height parameter in meters. + init_step_vel : float + Initial step size for the velocity parameter in meters per year. + min_steps : float + Minimum number of steps in the search space for the height and velocity parameters. + + Returns + ------- + Tuple[np.ndarray, np.ndarray, float, float, float] + Returns the unwrapped phase, ambiguities, estimated height, estimated velocity, and temporal coherence. + - Unwrapped phase: in rads, shape (n_obs,), dtype np.float64. + - Ambiguities: unitless, shape (n_obs,), dtype np.float64. + - Estimated height: in meters, scalar, dtype np.float64. + - Estimated velocity: in meters per year, scalar, dtype np.float64. + - Temporal coherence: unitless float number, norm of the complex coherence, scalar, dtype np.float64. + """ + # Assign initial values for the search + step_height = init_step_height + step_vel = init_step_vel + param_height = init_height + param_vel = init_vel + + # Calculate the initial temporal coherence for the initial height and velocity, + # in case the search loop is not entered + phs_model = B @ np.array([param_height, param_vel]) # size n_obs + phase_residual = phs_obs_wrapped[:, None] - phs_model + coh_best = (np.cos(phase_residual).sum() + 1j * np.sin(phase_residual).sum()) / phs_obs_wrapped.shape[0] + + # Search loop + count = 0 + while step_height > STOP_HEIGHT and step_vel > STOP_VEL and count < MAX_COUNT: + # Build search space + search_space = _build_periodogram_search_space( + param_height, param_vel, step_height, step_vel, min_steps, min_steps + ) + + # Calculate the wrapped model phase for all candidates + phs_model = wrap_phase(B @ search_space.T) # size n_obs x n_search + + # Calculate the temporal coherence for all search candidates + # Expand dimension of phs_obs_wrapped to facilitate broadcasting + # No need to repeat phs_obs_wrapped since the minus operation will broadcast to the shape of phs_model + # Sum along axis=0 which is the observation axis + # Reference: van Leijen 2014, Eq. 4.55 + # The following implementation equivalent to: + # np.exp(1j * (np.expand_dims(phs_obs_wrapped, axis=1) - phs_model)).sum(axis=0) / phs_obs_wrapped.shape[0] + phase_residual = phs_obs_wrapped[:, None] - phs_model + coh_search_space = ( + np.cos(phase_residual).sum(axis=0) + 1j * np.sin(phase_residual).sum(axis=0) + ) / phs_obs_wrapped.shape[0] + + # Get the best temporal coherence value and its index + coh_idx = np.argmax(np.abs(coh_search_space)) + coh_best = coh_search_space[coh_idx] + + # Update values needed for search space + # Reduce step size to 1/10 + param_height = search_space[coh_idx, 0] + param_vel = search_space[coh_idx, 1] + step_height /= 10 + step_vel /= 10 + + count += 1 + + # Correct the height parameter for using h2ph_approx + # Method copied from MATLAB DePSI code + factor = np.median(h2ph / h2ph_approx) # correct factor + param_height = param_height / factor + + # Calculate the modelled phase and unwrapped phase + model_est = B @ np.array([param_height, param_vel]) + np.angle(coh_best) # Absolute modelled phase + dphase_new = wrap_phase(phs_obs_wrapped - model_est) # Wrapped modelled phase + ambiguities = np.round((model_est + dphase_new - phs_obs_wrapped) / (2 * np.pi)) # Ambiguities + phs_obs_unwrapped = 2 * np.pi * ambiguities + phs_obs_wrapped # Unwrapped phase + param = rhs @ phs_obs_unwrapped # [height_est, velocity_est] + + return phs_obs_unwrapped, ambiguities, param[0], param[1], np.abs(coh_best) + + +def _chunk_for_temp_coh_compute(phs_obs_wrapped, search_space): + """Chunk observations and search space for temporal coherence computation.""" + if isinstance(phs_obs_wrapped, da.Array): # Existing chunk size for the arc dimension + chunk_arcs = phs_obs_wrapped.chunks[0][0] + else: + # If phs_obs_wrapped is not a dask array, chunk it in the arc dimension, making each chunk about 10 MB + chunk_arcs = max(1, 10 * 1024**2 // (phs_obs_wrapped.shape[1] * phs_obs_wrapped.dtype.itemsize)) + phs_obs_wrapped = da.from_array(phs_obs_wrapped, chunks=(chunk_arcs, phs_obs_wrapped.shape[1])) + # Decide the chunk size for the search space dimension + # making each n_arcs x n_obs x n_search chunk about 100 MB + chunk_searches = max(1, 100 * 1024**2 // (chunk_arcs * phs_obs_wrapped.shape[1] * phs_obs_wrapped.dtype.itemsize)) + search_space = da.from_array(search_space, chunks=(chunk_searches, 2)) + + return phs_obs_wrapped, search_space + + +def _build_periodogram_search_space(init_height, init_vel, step_height, step_vel, n_steps_height, n_steps_vel): + """Construct the periodogram search space for height and velocity parameters. + + For both height and velocity, the candidates are generated around the initial values according to the step size + and the number of steps. On each side of the initial value, N candidates are generated with a step size, where + N is specified by `n_steps_height` and `n_steps_vel`, and the step size is specified by `step_height` and + `step_vel`. + + Then all possible combinations of height and velocity candidates are created to form + the search space. + + Parameters + ---------- + init_height : float + Initial height parameter in meters. + init_vel : float + Initial velocity parameter in meters per year. + step_height : int + Step size of search for height parameter, in meters. + step_vel : int + Step size of search for velocity parameter, in meters per year. + n_steps_height : int + Number of steps for height parameter on each side of the initial value. + n_steps_vel : int + Number of steps for velocity parameter on each side of the initial value. + + Returns + ------- + np.ndarray + Search space for height and velocity parameters, shape (n_candidates_vel * n_candidates_height, 2) + """ + height_candidates = np.arange( + init_height - n_steps_height * step_height, + init_height + n_steps_height * step_height + step_height, + step_height, + ) + + vel_candidates = np.arange( + init_vel - n_steps_vel * step_vel, init_vel + n_steps_vel * step_vel + step_vel, step_vel + ) + + # All possible combinations of height and velocity + search_space = np.array(np.meshgrid(height_candidates, vel_candidates)).T.reshape(-1, 2) + + return search_space diff --git a/constants.py b/constants.py new file mode 100644 index 00000000..14c5abad --- /dev/null +++ b/constants.py @@ -0,0 +1,11 @@ +"""Physical constants used in DePSI.""" + +# Earth radius +# Unit in meters +EARTH_RADIUS = 6378136 + +# Speed of light in vacuum +SPEED_OF_LIGHT = 299792458.0 # m/s + +# Sentinel-1 wavelength +WAVELENGTH_S1 = 0.055465763 # m diff --git a/network.py b/network.py new file mode 100644 index 00000000..cfe12998 --- /dev/null +++ b/network.py @@ -0,0 +1,2054 @@ +"""Module for creating networks from STM points.""" + +import logging +import math +from typing import Literal + +import matplotlib.pyplot as plt +import networkx as nx +import numpy as np +import scipy +import sparse +import xarray as xr +from scipy.spatial import Delaunay, KDTree, distance_matrix +from sklearn.neighbors import BallTree + +from depsi.arc_estimation import arc_estimation_control_network +from depsi.constants import EARTH_RADIUS +from depsi.stats import pretest +from depsi.utils import compute_phase_difference, get_distance + +logger = logging.getLogger(__name__) + + +# Constants for MHT in network integration +ALPHA0 = 0.1 # Significance level for 1-dimensional test +GAMMA0 = 0.5 # Power of the test +# Overall Model Test threshold for accepting the network +# In arc/point rejection phase, if OMT < OMT_THRES, stop rejection iteration +# In ambiguity fixing phase, if OMT < OMT_THRES, stop fixing iteration +# In arc/point rejection phase this is hardly triggered +OMT_THRES = 1e-10 +# Threshold for arc rejection statistics TT1 +# If for all arcs max(TT1) < TT1_THRES, stop rejection iteration +# For most cases this threshold is triggered in rejection phase +TT1_THRES = 1.0 + + +def spatial_integration( + stm_pnts: xr.Dataset, + stm_arcs: xr.Dataset, + key_sdphase: str = "sd_phase", + key_arc_quality: str = "temp_coh", + threshold_arc_quality: float = 0.5, + idx_refpnt: int | None = None, + min_arc_connections: int = 3, + largest_component_ratio: float = 0.8, + parallel: bool = False, + sparse_mode: bool = False, + ensure_network_while_mht: bool = False, + arc_estimation_method: Literal["periodogram"] = "periodogram", + skip_network_adaptation: bool = False, + max_iterations_adaptation: int = None, +) -> tuple[xr.Dataset, xr.Dataset]: + """Spatially integrate the ambiguities of network arcs to points. + + This function estimates the integer ambiguities of the points from arc ambiguities. It assumes a network + has been formed from the points by calling "network.form_network", and arc ambiguities have been estimated + by calling relevant functions in the "depsi.arc_estimation" module. + + The function returns an updated STM of arcs which contains the adjusted arc ambiguities after spatial integration, + as well as an updated STM of points which contains the estimated point ambiguities. + + The following steps are performed: + 1. Validate the network arcs (`stm_arcs`) against the point STM (`stm_pnts`) + 2. Select arcs based on quality threshold and ensure minimum connections for all points + 3. Select a reference point which assumes zero phase (hence zero ambiguity) + 4. Adjust the network by removing arcs/points which potentially cause errors using Multi-Hypothesis Testing (MHT) + 5. Adjust the ambiguities per time epoch to make sure spatial solutions give zero residuals. + 6. Calculate point ambiguities and unwrapped phases w.r.t. the reference point. + + Parameters + ---------- + stm_pnts : xr.Dataset + Space-Time Matrix of points. + stm_arcs : xr.Dataset + Space-Time Matrix of arcs. + The arcs should be formed from stm_pnts, using the "network.form_network" function. This function generates + coordinates "source" and "target" in stm_arcs which refer to the indices in stm_pnts. An "uid" data variable + is also generated by "form_network" for easy indexing. + An arc estimation should have been applied on stm_arcs before calling "spatial_integration". One can use + relevant functions in "depsi.arc_estimation" module for this purpose. Arc estimation adds the variable + "ambiguities" to stm_arcs, which are the estimated arc ambiguities. It also adds quality variables such as + "temp_coh" (ensemble coherence), which are used to select arcs for spatial integration. + key_sdphase : str, optional + Key of the single difference phase variable in stm_pnts, by default "sd_phase" + This phase is used to compute unwrapped phases after ambiguity estimation. + key_arc_quality : str, optional + Key of the arc quality variable in stm_arcs, by default "temp_coh" + threshold_arc_quality : float, optional + Threshold for arc quality, by default 0.5 + idx_refpnt : int | None, optional + Index of the reference point in stm_pnts. If None, the source point of the arc with highest quality is selected + as the reference point. + min_arc_connections : int, optional + Minimum number of connections for arcs, by default 3 + largest_component_ratio : float, optional + Threshold for determining the largest component when multiple components exist in the network, by default 0.8. + When removing arcs/points, it may happen that the network is split into multiple disconnected components. + In this case, only the largest component is kept and the others are discarded. + The largest component should contain at least this fraction of the total points, otherwise an error is raised. + Hence, when an error should be raised at all times, set this value to 1.0 . + parallel : bool, optional + Whether to use parallel processing, by default False + sparse_mode : bool, optional + Whether to use sparse matrix format for large networks, by default False + ensure_network_while_mht : bool, optional + Whether to ensure minimum connections in MHT network adaptation, by default False + arc_estimation_method : Literal["periodogram"], optional + Method used for arc estimation, by default "periodogram". + This constrains the method used for VCM computation. + skip_network_adaptation : bool, optional + Whether to skip network adaptation by MHT, by default False. + When enabling this option, it is recommended to set the threshold_arc_quality to a + high value (e.g. 0.75) to ensure only high-quality arcs are selected for spatial + integration. + max_iterations_adaptation : int, optional + Maximum number of iterations for network adaptation. + If None, the maximum number if iterations will be the number of arcs. + + Returns + ------- + xr.Dataset, xr.Dataset + Updated Space-Time Matrix of arcs and updated Space-Time Matrix of points. + For arcs, the "ambiguities" variable contains the adjusted arc ambiguities after spatial integration. + For points, the "ambiguities" variable contains the estimated point ambiguities, and "unwrapped_phase" + contains the unwrapped phase w.r.t. the reference point. + + References + ---------- + Van Leijen, F.J.. "Persistent scatterer interferometry based on geodetic estimation theory." (2014). + """ + # Check parallelization behavior + if parallel: + raise NotImplementedError("Dask support is not implemented yet for spatial_integration.") + else: + # Compute all data into memory + stm_pnts = stm_pnts.compute() + stm_arcs = stm_arcs.compute() + + # Validate that stm_arcs are formed from stm_pnts + if (stm_arcs["source"].max().values >= stm_pnts.sizes["space"]) or ( + stm_arcs["target"].max().values >= stm_pnts.sizes["space"] + ): + raise ValueError("stm_arcs contains source/target indices that exceed the number of points in stm_pnts.") + if "ambiguities" not in stm_arcs: + raise ValueError("stm_arcs does not contain 'ambiguities' variable. Please estimate arc ambiguities first.") + if key_arc_quality not in stm_arcs: + raise ValueError(f"stm_arcs does not contain '{key_arc_quality}' variable for arc quality assessment.") + + # Check arc estimation method, this constrains VCM computation method + if arc_estimation_method not in ["periodogram"]: + raise NotImplementedError(f"Unknown arc estimation method {arc_estimation_method}.") + + if sparse_mode: + raise NotImplementedError("Sparse mode is not implemented yet for spatial_integration.") + + # If idx_refpnt is specified + # Get radar coordinates of the reference point before any shape change + if idx_refpnt is not None: + azimuth_refpnt = stm_pnts["azimuth"].isel(space=idx_refpnt).values + range_refpnt = stm_pnts["range"].isel(space=idx_refpnt).values + + # Select arcs with quality > threshold_arc_quality + # Then ensure all points have at least min_arc_connections connections + mask = (np.abs(stm_arcs[key_arc_quality]) > threshold_arc_quality).compute() + stm_arcs = stm_arcs.where(mask, drop=True) + stm_arcs, stm_pnts = _ensure_network_min_connections(stm_arcs, stm_pnts, min_arc_connections) + + # Ensure the network is a single connected component after arc selection and point removal + stm_arcs, stm_pnts = _ensure_single_network(stm_arcs, stm_pnts, largest_component_ratio) + + # Select reference point as the source pnt of arcs with highest temp_coh + if idx_refpnt is None: + idx_arc_max_coh = stm_arcs[key_arc_quality].argmax().values + idx_refpnt = stm_arcs["source"].isel(space=idx_arc_max_coh).values + azimuth_refpnt = stm_pnts["azimuth"].isel(space=idx_refpnt).values + range_refpnt = stm_pnts["range"].isel(space=idx_refpnt).values + else: + # Make sure idx_refpnt is still valid after arc selection and point removal + mask_refpnt = (stm_pnts["azimuth"].values == azimuth_refpnt) & (stm_pnts["range"].values == range_refpnt) + if not np.any(mask_refpnt): + raise ValueError( + f"Reference point ({azimuth_refpnt}, {range_refpnt}) removed after arc selection. " + f"Please choose another reference point." + ) + + # Get reference point from radar coordinates + idx_refpnt = np.where(mask_refpnt)[0][0] + + # Adjust the network by removing bad arcs/points using MHT + if skip_network_adaptation: + logger.info("Skipping MHT network adaptation step.") + stm_arcs_adjusted, stm_pnts_adjusted = stm_arcs, stm_pnts + else: + stm_arcs_adjusted, stm_pnts_adjusted = _mht_network_adaptation( + stm_arcs, + stm_pnts, + idx_refpnt, + azimuth_refpnt, + range_refpnt, + ensure_network_while_mht, + sparse_mode, + arc_estimation_method, + max_iterations_adaptation, + largest_component_ratio, + ) + + # Update idx_refpnt after MHT adaptation + idx_refpnt = np.where( + (stm_pnts_adjusted["azimuth"].values == azimuth_refpnt) + & (stm_pnts_adjusted["range"].values == range_refpnt) + )[0][0] + + # Adjust ambiguities to fix unwrapping errors + stm_arcs_output, stm_pnts_output, idx_refpnt = _ambiguity_adaptation( + stm_arcs_adjusted, + stm_pnts_adjusted, + idx_refpnt, + sparse_mode, + arc_estimation_method, + ) + + # Assign idx_refpnt as attribute to stm_pnts_output + stm_pnts_output = stm_pnts_output.assign_attrs({"idx_refpnt": idx_refpnt}) + + # Add unwrapped phase to stm_pnts_output + # Unwrapped phase is w.r.t. the reference point + # Therefore the sd_phase of the reference point is subtracted + unwrapped_phase_pnts = ( + stm_pnts_output[key_sdphase].data + + stm_pnts_output["ambiguities"].data * 2 * np.pi + - np.tile( + stm_pnts_output[key_sdphase].isel(space=idx_refpnt).data, + (stm_pnts_output.sizes["space"], 1), + ) + ) + stm_pnts_output["unwrapped_phase"] = (("space", "time"), unwrapped_phase_pnts) + + return stm_arcs_output, stm_pnts_output + + +def form_network( + stm: xr.Dataset, + key_phase: str, + key_h2ph: str, + key_Btemporal: str, + key_complex: str = "complex", + key_xcrds: str = "lon", + key_ycrds: str = "lat", + network_method: Literal["redundant", "delaunay"] = "redundant", + max_length: float = None, + min_links: int = 16, + num_partitions: int = 8, + dphase_method: Literal["conjmult", "subtract"] = "subtract", +) -> xr.Dataset: + """Generate an STM of arcs from an STM of points. + + Parameters + ---------- + stm : xr.Dataset + Space-Time Matrix of scatterers. + key_phase : str + Key of the phase values in the STM. + This phase will be used to compute the differential arc phase. + key_h2ph : str + Key of the h2ph values in the STM. + The arc h2ph will be computed as the average between source and target. + key_Btemporal : str + Key of the temporal baseline values in the STM. + key_complex : str, optional + Key of the complex values, by default "complex" + key_xcrds : str, optional + Key of the x coordinates for calulating arc length, by default "lon" + key_ycrds : str, optional + Key of the y coordinates for calulating arc length, by default "lat" + network_method : Literal["redundant", "delaunay"], optional + network formation method, by default "redundant" + max_length : float, optional + maximum arc length, by default None + min_links : int, optional + minimum links per point, by default 16 + only effective when network_method is "redundant" + num_partitions : int, optional + number of partitions of searching when forming redundant network, by default 8 + only effective when network_method is "redundant" + dphase_method : Literal["conjmult", "subtract"], optional + method of computing phase difference, by default "subtract" + "subtract" method subtracts the source phase from the target phase (without re-wrapping); + "conjmult" method computes the phase difference by conjugate multiplication: + d_phase = np.angle(complex_target * complex_source.conj()) + + Returns + ------- + xr.Dataset + Space-Time Matrix of arcs, containing the following variables: + - d_phase: the arc phase, which is the difference between source and target points + - Btemp: the temporal baseline, which is the same for all arcs + - h2ph: the arc h2ph, which is the average between source and target points + """ + # Generate the network arcs. + if network_method == "redundant": + if min_links <= 0: + logger.error(f"min_links must be strictly positive (currently: {min_links})") + return + if num_partitions <= 0: + logger.error(f"num_partitions must be strictly positive (currently: {num_partitions})") + return + elif network_method != "delaunay": + raise NotImplementedError(f"Unknown network method {network_method}, known are delaunay and redundant") + + # Collect point coordinates. + indices = [stm[coord] for coord in [key_xcrds, key_ycrds]] + coordinates = np.column_stack(indices) + + arcs = None + + # Create network arcs as list of tuples of point ids. + if network_method == "delaunay": + arcs = _generate_arcs_delaunay(coordinates, max_length) + elif network_method == "redundant": + arcs = _generate_arcs_redundant(coordinates, max_length, min_links, num_partitions) + + # Compute the phase difference. + arcs_unzipped = list(zip(*arcs, strict=False)) + source_idx = list(arcs_unzipped[0]) + target_idx = list(arcs_unzipped[1]) + + if dphase_method not in ["conjmult", "subtract"]: + raise NotImplementedError(f"Unknown dphase_method '{dphase_method}'.") + dict_key_method = {"subtract": key_phase, "conjmult": key_complex} # mapping for selecting the correct key + d_phase = compute_phase_difference( + stm.isel(space=source_idx)[dict_key_method[dphase_method]].data, + stm.isel(space=target_idx)[dict_key_method[dphase_method]].data, + method=dphase_method, + ) + + # Temporal baseline + Btemp = stm[key_Btemporal].data + + # Height to phase factor + h2ph = (stm[key_h2ph].isel(space=source_idx).data + stm[key_h2ph].isel(space=target_idx).data) / 2 + + # Generate a unique identifier of arcs based on source and target for easy indexing + # This is because when updating network, points can be removed and reindexed + # Therefore we cannot use 2d index (source, target) as uid + # NOTE: This encoding is safe for typical networks (<10,000 points, <100,000 arcs). + # For 10,000 points, max UID ≈ 1e9, well within int64 max (2^63-1 ≈ 9.22e18), providing + # a safety margin of >9 billion times. Even for 1M points, safety margin is >900,000x. + scale = 10 ** (math.floor(math.log10(stm.sizes["space"])) + 1) # Scale to ensure no overlap + uid = scale * (np.array(source_idx) + 1) + (np.array(target_idx) + 1) # Plus one to avoid zero uid + uid = uid.astype(np.int64) + + arcs = xr.Dataset( + data_vars={ + "d_phase": (["space", "time"], d_phase), + "Btemp": (["time"], Btemp), + "h2ph": (["space", "time"], h2ph), + }, + coords={"source": (["space"], source_idx), "target": (["space"], target_idx), "uid": (["space"], uid)}, + attrs=stm.attrs, + ) + + return arcs + + +def _mht_network_adaptation( + stm_arcs: xr.Dataset, + stm_pnts: xr.Dataset, + idx_refpnt: int, + azimuth_refpnt: int | float, + range_refpnt: int | float, + ensure_network_while_mht: bool, + sparse_mode: bool, + arc_estimation_method: str, + max_iterations_adaptation: int, + largest_component_ratio: float, +) -> tuple[xr.Dataset, xr.Dataset]: + """Adjust the network by removing bad arcs/points by applying MHT. + + This function implements the Multi-Hypothesis Testing (MHT) approach iteratively to identify and remove + arcs and points to reduce the overall residual in ambiguity estimation. + + Parameters + ---------- + stm_arcs : xr.Dataset + Space-Time Matrix of arcs. + stm_pnts : xr.Dataset + Space-Time Matrix of points. + idx_refpnt : int + Index of the reference point in stm_pnts. + azimuth_refpnt : int | float + Azimuth coordinate of the reference point. + range_refpnt : int | float + Range coordinate of the reference point. + ensure_network_while_mht : bool + Whether to ensure minimum connections in MHT network adaptation. + sparse_mode : bool + Whether to use sparse matrix format for large networks. + arc_estimation_method : str + Method used for arc estimation. + max_iterations_adaptation : int + Maximum number of iterations for network adaptation. + largest_component_ratio : float + Threshold for determining the largest component when multiple components exist in the network. + + Returns + ------- + xr.Dataset, xr.Dataset + Updated Space-Time Matrix of arcs and points. + """ + # Setup functional and stochastic model + A = _network_relation_matrix( + stm_arcs["source"], stm_arcs["target"], stm_pnts.sizes["space"], idx_refpnt, sparse_mode + ) # Network relation matrix A + + if arc_estimation_method == "periodogram": + Qyy_diag = 1 - stm_arcs["temp_coh"].values + else: + raise NotImplementedError(f"arc_estimation_method '{arc_estimation_method}' is not supported.") + invQy = np.diag(1 / Qyy_diag) + + _, echeck, _ = _solve_float_ambiguities(A, stm_arcs["ambiguities"].data, invQy) # Estimate initial residual + OMT = np.diag(echeck.T @ invQy @ echeck).sum() # Test statistics for Overall Model Test + + # Setup test parameters + kb_dict = {} + max_con = np.abs(A).sum(axis=0).max() + for n_con in range(1, max_con + 1): + _, k1, kb, _ = pretest(n_con, ALPHA0, GAMMA0) + kb_dict[n_con] = kb + + # By default, set max_iterations_adaptation to the number of arcs + if max_iterations_adaptation is None: + max_iterations_adaptation = stm_arcs.sizes["space"] + + # Iteratively remove arcs/points until OMT and all arc statistics pass the test + stm_pnts_updated = stm_pnts.copy() + stm_arcs_updated = stm_arcs.copy() + TT1max = TT1_THRES + 1.0 # Initial TT1_max to trigger the while loop + niter = 0 + while (OMT >= OMT_THRES) and (TT1max >= TT1_THRES) and (niter < max_iterations_adaptation): + # The iteration stops when one of the following conditions is met: + # 1) overall model test pass: OMT < OMT_THRES (very rare case) + # 2) all arc test statistics smaller than threshold: max(TT1) < TT1_THRES (most common case) + # 3) maximum number of iterations reached (fail case) + logger.debug(f"MHT iteration {niter}: OMT={OMT:.2e}") + + # Because OMT failed, choose from two Ha: 1) remove an arc; 2) remove a point + # Decision is made based on flag_rm + flag_rm, idx_rm, TT1max, TTqmax = _mht_network_adaptation_reject_one( + A, stm_arcs_updated["ambiguities"].data, Qyy_diag, k1, kb_dict + ) + + if flag_rm == 0: # remove arc + logger.debug(f"MHT iteration {niter}: removing arc index {idx_rm} with TT1={TT1max:.2f}") + stm_arcs_updated = stm_arcs_updated.drop_isel(space=idx_rm) # Remove the arc + elif flag_rm == 1: # remove point + if idx_rm >= idx_refpnt: + idx_rm += 1 # Adjust index due to removed reference point column in A + + logger.debug(f"MHT iteration {niter}: removing point index {idx_rm} with TT1={TT1max:.2f}") + + # Removing points is achieved by removing all arcs connects to the point + # Later the points will be actually removed when ensuring minimum connections + # Arc indices connecting to the point to remove + idx_arcs_selected = np.where( + ((stm_arcs_updated["source"] != idx_rm) & (stm_arcs_updated["target"] != idx_rm)).data + )[0] + # Remove all arcs connects to the point to remove + stm_arcs_updated = stm_arcs_updated.isel(space=idx_arcs_selected) + + if ensure_network_while_mht: + min_connections_to_ensure = 3 # Ensure all points in the network have at least 3 connections + else: + min_connections_to_ensure = 2 # Just ensure all points are connected in the network + + # This makes sure all points can be tested in case of disagreement between arcs + stm_arcs_updated, stm_pnts_updated = _ensure_network_min_connections( + stm_arcs_updated, stm_pnts_updated, min_connections=min_connections_to_ensure + ) + + # Ensure the network is a single connected component after arc/point removal + stm_arcs_updated, stm_pnts_updated = _ensure_single_network( + stm_arcs_updated, stm_pnts_updated, largest_component_ratio + ) + + # Make sure the reference point is still in stm_pnts_updated, by checking its azimuth and range + mask_refpnt = (stm_pnts_updated["azimuth"].values == azimuth_refpnt) & ( + stm_pnts_updated["range"].values == range_refpnt + ) + if not np.any(mask_refpnt): + raise ValueError( + f"Reference point ({azimuth_refpnt}, {range_refpnt}) removed in the MHT process. " + f"Please choose another reference point." + ) + idx_refpnt = np.where(mask_refpnt)[0][0] # Update idx_refpnt + + # Get indices of selected arcs based on uid + if arc_estimation_method == "periodogram": + Qyy_diag = 1 - stm_arcs_updated["temp_coh"].values # VCM diagonal from temporal coherence + invQy = np.diag(1 / Qyy_diag) + + A = _network_relation_matrix( + stm_arcs_updated["source"], + stm_arcs_updated["target"], + stm_pnts_updated.sizes["space"], + idx_refpnt, + sparse_mode, + ) # Update A matrix + _, echeck, _ = _solve_float_ambiguities( + A, stm_arcs_updated["ambiguities"].data, invQy + ) # Estimate residual again + OMT = np.diag(echeck.T @ invQy @ echeck).sum() # Update OMT statistic + + niter += 1 + + if niter >= stm_arcs.sizes["space"]: + raise RuntimeError( + "Maximum number of iterations reached in MHT network adaptation. " + "The network may still contain bad arcs or points." + ) + + return stm_arcs_updated, stm_pnts_updated + + +def _mht_network_adaptation_reject_one( + A: np.ndarray | scipy.sparse.spmatrix, + y: np.ndarray, + Qyy_diag: np.ndarray, + k1: float, + kb_dict: dict, +) -> tuple[int, int, float, float]: + """Remove one point/arc from the network to reduce the residual in ambiguity estimation.""" + # Retrieve shapes + N_arcs, N_epochs = y.shape + N_points = A.shape[1] + + # Inverse of VCM of observations + if Qyy_diag.ndim == 1: # Diagonal VCM + invQy = np.diag(1 / Qyy_diag) + Qyy = np.diag(Qyy_diag) + else: + raise NotImplementedError("Currently only diagonal VCM is supported. Qyy_diag should be a 1d array.") + + # Solve ambiguities as float + _, echeck, invAtQyA = _solve_float_ambiguities(A, y, invQy) + + # Post-priori VCM of residuals + Qecheck = Qyy - (A @ invAtQyA @ A.T) # TODO: check how to handle large Qecheck + + # Test statistics TT1 per arc + Qecheck_diag = np.array(np.diag(Qecheck).flatten()).squeeze() + w = echeck**2 / np.tile(np.abs(Qecheck_diag), (N_epochs, 1)).T + TT1 = np.sum(w, axis=1) / k1**2 + TT1max = max(TT1) + + # Test statistics per point + # Build arc-point connectivity mask and + connected_mask = A != 0 + # Drop one connected arc per point to create the basis (see e.g. verhoef97). + has_connection = connected_mask.any(axis=0) + first_connected_idx = np.argmax(connected_mask, axis=0) + selected_mask = connected_mask.copy() + selected_mask[first_connected_idx[has_connection], np.where(has_connection)[0]] = False + # Tq for point q: sum_i(sum_t(e_i,t^2) / Qe_i) over selected arcs i connected to point q. + e2_sum = np.sum(echeck**2, axis=1) + e2_sum_weighted = e2_sum / np.abs(Qecheck_diag) + Tq_num = selected_mask.T @ e2_sum_weighted + # Find where to calculate TTq based on connectivity + narcs_connected = selected_mask.sum(axis=0).astype(int) + kb_vals = np.array([kb_dict.get(d, np.nan) for d in narcs_connected], dtype=float) + valid = (narcs_connected > 0) & np.isfinite(kb_vals) & (kb_vals != 0) + # Calculate TTq for points with valid kb values + # assign -inf to invalid ones to make sure they won't be selected for removal + TTq = np.full(N_points, -np.inf, dtype=float) + with np.errstate(divide="ignore", invalid="ignore"): + TTq[valid] = Tq_num[valid] / kb_vals[valid] + TTqmax = max(TTq) + + # Decision one removal strategy + if TT1max > TTqmax: + idx_removal = np.argmax(TT1) # index of arc to remove + flag_removal = 0 # remove arc + else: + idx_removal = np.argmax(TTq) # index of point to remove + flag_removal = 1 # remove point + + return flag_removal, idx_removal, TT1max, TTqmax + + +def _ambiguity_adaptation( + stm_arcs: xr.Dataset, + stm_pnts: xr.Dataset, + idx_refpnt: int, + sparse_mode: bool, + arc_estimation_method: str, +) -> tuple[xr.Dataset, xr.Dataset, int]: + """Fix unwrapping errors by adjusting ambiguities per epoch. + + This function iterates over each epoch and adjusts the ambiguities to make sure the spatial + solutions of ambiguities gives zero residuals for each epoch. + + Parameters + ---------- + stm_arcs : xr.Dataset + Space-Time Matrix of arcs. + stm_pnts : xr.Dataset + Space-Time Matrix of points. + idx_refpnt : int + Index of the reference point. + sparse_mode : bool + Whether to use sparse matrix format for large networks. + arc_estimation_method : str + Method used for arc estimation. + + Returns + ------- + (xr.Dataset, xr.Dataset) + Updated Space-Time Matrices of arcs and points. + For arcs, the "ambiguities" variable contains the adjusted arc ambiguities. + For points, the "ambiguities" variable are estimated from the adjusted arc ambiguities. + """ + # Setup functional and stochastic model + A = _network_relation_matrix( + stm_arcs["source"], stm_arcs["target"], stm_pnts.sizes["space"], idx_refpnt, sparse_mode + ) + if arc_estimation_method == "periodogram": + Qyy_diag = 1 - stm_arcs["temp_coh"].values # VCM diagonal from temporal coherence + else: + raise NotImplementedError(f"arc_estimation_method '{arc_estimation_method}' is not supported in form_network.") + invQy = np.diag(1 / Qyy_diag) + + # Initialize adjusted ambiguities storage, shape: (n_points-1, n_epochs) + # Space dimension is n_points-1 because reference point is excluded + acheck = np.zeros((stm_pnts.sizes["space"] - 1, stm_pnts.sizes["time"])) + + # Fix unwrapping ambiguities by looping over epochs + stm_arcs_updated = stm_arcs.copy() + stm_pnts_updated = stm_pnts.copy() + for epoch in range(stm_pnts.sizes["time"]): + logger.debug(f"Adjusting ambiguities for epoch {epoch}") + y = stm_arcs["ambiguities"].isel(time=epoch).data + acheck_ifg, echeck_ifg, _ = _solve_float_ambiguities(A, y, invQy) + OMT = echeck_ifg.T @ invQy @ echeck_ifg + idx_previous_arc_fix = -1 # Avoid fixing the same arc again in the same epoch + + while OMT >= OMT_THRES: # While the OMT fails, fix for this epoch + # Find arc index with largest abs echeck + # When OMT > kOMT, echeck_ifg[idx_max_echeck] is guaranteed to be non-zero + idx_sort = np.argsort(np.abs(echeck_ifg))[::-1] # Indices of echeck sorted by abs value, descending + idx_max_echeck = idx_sort[0] # Index of arc with largest abs echeck + if idx_max_echeck == idx_previous_arc_fix: + # If get same arc as previous fix, take the second largest + idx_max_echeck = idx_sort[1] + + if np.round(abs(echeck_ifg[idx_max_echeck])) >= 1: # If >= 1, minus closest integer + y[idx_max_echeck] -= np.round(echeck_ifg[idx_max_echeck]) + elif echeck_ifg[idx_max_echeck] > 0: # if (0, 1), minus 1 + y[idx_max_echeck] -= 1.0 + elif echeck_ifg[idx_max_echeck] < 0: # if (-1, 0), plus 1 + y[idx_max_echeck] += 1.0 + + idx_previous_arc_fix = idx_max_echeck # record the fixed arc index + + # Recalculate OMT + acheck_ifg, echeck_ifg, _ = _solve_float_ambiguities(A, y, invQy) + OMT = echeck_ifg.T @ invQy @ echeck_ifg + + logger.debug(f"Fixing arc index {idx_max_echeck}, new OMT={OMT:.2e}") + + stm_arcs_updated["ambiguities"][:, epoch] = y # Store adjusted arc ambiguities + acheck[:, epoch] = acheck_ifg # Store adjusted point ambiguities + + # Round acheck to closest integer + acheck = np.round(acheck).astype(np.int16) + + # Assign reference point ambiguities as zero + acheck_full = np.zeros((stm_pnts.sizes["space"], stm_pnts.sizes["time"])).astype(np.int16) + acheck_full[np.arange(stm_pnts.sizes["space"]) != idx_refpnt, :] = acheck + + # Assign acheck_full to stm_pnts_updated + stm_pnts_updated["ambiguities"] = (["space", "time"], acheck_full) + + return stm_arcs_updated, stm_pnts_updated, idx_refpnt + + +def _ensure_network_min_connections( + stm_arcs: xr.Dataset, + stm_pnts: xr.Dataset, + min_connections: int, +) -> tuple[xr.Dataset, xr.Dataset]: + """Ensure that all points in the network have at least min_connections arcs. + + This is achieved by an iterative process of removing points which have less than + min_connections arcs, and removing all arcs connected to these points. + The process is repeated until no more points are removed. + + Parameters + ---------- + stm_arcs : xr.Dataset + Space-Time Matrix of arcs. + stm_pnts : xr.Dataset + Space-Time Matrix of points. + min_connections : int + Minimum number of connections for each point. + + Returns + ------- + xr.Dataset, xr.Dataset + Updated Space-Time Matrix of arcs and points. + """ + # Ensure all points have at least min_connections connections + previous_size = -1 # Initialize with an impossible value to trigger the while loop + # Keep iterating until no more points are removed + while stm_pnts.sizes["space"] != previous_size: + previous_size = stm_pnts.sizes["space"] + # Remove points with < min_connections connections + stm_pnts, stm_arcs = _remove_network_points_min_connections(stm_pnts, stm_arcs, min_connections) + + return stm_arcs, stm_pnts + + +def _ensure_single_network( + stm_arcs: xr.Dataset, stm_pnts: xr.Dataset, largest_component_ratio: float +) -> tuple[xr.Dataset, xr.Dataset]: + """Ensure the network is connected and discard the smaller disconnected sub-network(s). + + This function utilizes the NetworkX library to identify connected components in the network + formed by stm_arcs and stm_pnts. When building the graph, the point indices are used as node identifiers, + and the "source" and "target" coordinates in stm_arcs are used to add edges between the corresponding nodes. + Note that "source" and "target" coordinates in stm_arcs are indices of the points STM stm_pnts, + but not necessarily the same as the "space" coordinate of stm_pnts. + + If there are multiple connected components, only the largest one is kept and the others are discarded. + However, if the largest component is smaller than a certain ratio (largest_component_ratio) of the total + points, an error is raised. + """ + G = nx.Graph() + G.add_nodes_from(np.arange(stm_pnts.sizes["space"])) # Use point indices as node identifiers + G.add_edges_from( + zip( + stm_arcs["source"].values, + stm_arcs["target"].values, + strict=True, + ) + ) + + # Get list of connected components + list_components = [cc for cc in nx.connected_components(G)] + + # If there are multiple connected components, keep only the largest one and discard the others + if len(list_components) > 1: + nodes_largest = max(list_components, key=len) # set of node indices in the largest connected component + + # Check if the largest component is significantly larger than the second largest one + if (len(nodes_largest) / stm_pnts.sizes["space"]) < largest_component_ratio: + raise RuntimeError( + f"The largest connected component contains only {len(nodes_largest)} points, which is less than " + f"{largest_component_ratio * 100:.1f}% of the total {stm_pnts.sizes['space']} points. " + "In this case DePSI cannot automatically decide which component to keep. " + "This may indicate a problem with the network formation. " + "Please check the input data and parameters." + ) + + mask_arcs = xr.DataArray( + np.isin(stm_arcs["source"].values, list(nodes_largest)) + & np.isin(stm_arcs["target"].values, list(nodes_largest)), + dims=["space"], + ) + stm_arcs_output = stm_arcs.where(mask_arcs, drop=True) + stm_pnts_output = stm_pnts.isel(space=list(nodes_largest)) + + # Update the source and target indices in stm_arcs_output to match the new stm_pnts_output + idx_map = {old_idx: new_idx for new_idx, old_idx in enumerate(sorted(nodes_largest))} + stm_arcs_output_updated = stm_arcs_output.copy() + stm_arcs_output_updated["source"] = xr.DataArray( + np.vectorize(idx_map.get)(stm_arcs_output["source"].values), dims="space" + ) + stm_arcs_output_updated["target"] = xr.DataArray( + np.vectorize(idx_map.get)(stm_arcs_output["target"].values), dims="space" + ) + stm_arcs_output = stm_arcs_output_updated + + n_components = len(list_components) + logger.info("Separated components detected in the network!") + logger.info(f"Network has {n_components} connected components.") + logger.info( + f"Keeping only the largest component with {stm_pnts_output.sizes['space']} points " + f"and {stm_arcs_output.sizes['space']} arcs." + ) + logger.info( + f"Discarded {stm_pnts.sizes['space'] - stm_pnts_output.sizes['space']} points " + f"and {stm_arcs.sizes['space'] - stm_arcs_output.sizes['space']} arcs." + ) + else: + stm_arcs_output = stm_arcs + stm_pnts_output = stm_pnts + + return stm_arcs_output, stm_pnts_output + + +def _solve_float_ambiguities(A, y, invQy, sparse_mode: bool = False): + """Solve ambiguities as a float based on Least-Squares.""" + # Solve ambiguities as they are float numbers + # This solves the equation y = Ax in least-squares sense + # With A a sparse matrix + # And stochastic model Qyy taken into account + invQyA = invQy @ A # Avoid repeated computation in vectorized lsmr + invAtQyA = np.linalg.inv(A.T @ invQyA) + + if sparse_mode: + + @np.vectorize(signature="(i)->(j)") + def lsmr(y): + """Least-squares iterative solver for sparse data.""" + x, *_ = scipy.sparse.linalg.lsmr(invQyA, invQy @ y) + return x + + acheck = lsmr(y.T).T # float ambiguity estimation + else: + acheck = invAtQyA @ (A.T @ invQy @ y) + echeck = y - A @ acheck # residuals estimation + + return acheck, echeck, invAtQyA + + +def _remove_network_points_min_connections( + stm: xr.Dataset, arcs: xr.Dataset, min_connections: int +) -> tuple[xr.Dataset, xr.Dataset]: + """Remove points which have less than min_connections arc connections. + + The following steps are performed: + + 1. Remove points from stm which have less than min_connections connections in arcs. + 2. Remove arcs which connect to the removed points. + 3. Update the space indices in points/arcs STM accordingly. + The point indices are always a 0-based continuous array. + + Note that this function does not perform iterative removal to assure that all points have + at least min_connections connections, but only performs one round of removal. + """ + if min_connections < 1: + raise ValueError("min_connections must be at least 1") + + # Load source and target indices from arcs + # these are 1d arrays so should fit in memory + idx_source = arcs["source"].values + idx_target = arcs["target"].values + + # Select STM points that are in arcs + # Only keep points which ids are in arcs, isolated points are removed in idx_selected + idx_selected, counts = np.unique(np.concatenate([idx_source, idx_target]), return_counts=True) + idx_selected = idx_selected[counts >= min_connections] # only keep points with at least min_connections connections + + # If no change, return directly + if len(idx_selected) == stm.sizes["space"]: + return stm, arcs + + # Select points + stm_pnts_updated = stm.isel(space=idx_selected) + + # Select arcs that connect selected points (Some arcs may be dropped together with points) + mask_source = np.isin(idx_source, idx_selected) + mask_target = np.isin(idx_target, idx_selected) + mask_arcs = mask_source & mask_target + arcs = arcs.isel(space=np.where(mask_arcs)[0]) + + # The space size of the STM changes, resulting non-contiguous indices in space dimension + # hence an update in arcs space coordinates is needed + # Here we use a mapping solution, since the maximum number of network points is usually <100k + # Map old indices in arcs to new indices + idx_map = {old_idx: new_idx for new_idx, old_idx in enumerate(idx_selected)} + # apply the mapping to the source and target indices in arcs + arcs_updated = arcs.copy() + arcs_updated["source"] = xr.DataArray(np.vectorize(idx_map.get)(arcs["source"].values), dims="space") + arcs_updated["target"] = xr.DataArray(np.vectorize(idx_map.get)(arcs["target"].values), dims="space") + + return stm_pnts_updated, arcs_updated + + +def _generate_arcs_delaunay(coordinates, max_length): + """Create a network using Delaunay triangulation.""" + # Create network and collect neighbors. + network = Delaunay(coordinates) + neighbors_ptr, neighbors_idx = network.vertex_neighbor_vertices + + # Convert ptr and idx arrays into list of sorted index pairs. + arcs = [] + for s in range(len(neighbors_ptr) - 1): + for t in range(neighbors_ptr[s], neighbors_ptr[s + 1]): + length = get_distance(coordinates[int(s)], coordinates[neighbors_idx[t]], mode="euclidean") + if max_length is None or length <= max_length: + arcs.append(tuple(sorted([int(s), int(neighbors_idx[t])]))) + + # Remove duplicates and make the list canonical. + arcs = sorted(list(set(arcs))) + + return arcs + + +def _generate_arcs_redundant(coordinates, max_length, min_links, num_partitions): + """Create a network with at least min_links arcs per node. + + The redundant network is formed with the following steps: + + 1. Create a KDTree and find all pairs of points within the maximum distance. + 2. Loop through each point and find its neighbors within the maximum distance. + 3. Divide neighbors into partitions based on their direction. + 4. Select the nth nearest neighbors from all partitions, starting from n=1. + 5. Sort the selected neighbors by distance, add them to the arcs list. If min_links is not + exceeded, continue to the n+1th nearest neighbors of all partitions. + 6. Repeat until min_links is reached. + """ + arcs = [] + indices = range(len(coordinates)) + + # Create a KDTree for fast distance queries. + tree = KDTree(coordinates) + if max_length is not None: + pairs = tree.query_pairs(r=max_length, output_type="ndarray") + else: + pairs = tree.query_pairs(r=np.inf, output_type="ndarray") + + # Duplicate pairs with reversed indices to ensure that arcs are undirected. + pairs = np.concatenate((pairs, np.flip(pairs, axis=1)), axis=0) + pairs = pairs[np.argsort(pairs[:, 0])] # Sort pairs by first column (source index). + + for cur_index in indices: + # Get the neighbors of the current node + neighbors = pairs[pairs[:, 0] == cur_index][:, 1].tolist() + + if len(neighbors) == 0: # skip if there are no neighbors + continue + elif len(neighbors) <= min_links: + # If there are not enough neighbors, connect them all. + for idx in neighbors: + arc_to_add = (min(cur_index, idx), max(cur_index, idx)) + arcs.append(arc_to_add) + else: + # Calculate partitions and distances for neighbors + partitions = [ + int(math.floor(num_partitions * (0.5 + math.atan2(coordinate[1], coordinate[0]) / math.tau))) + for coordinate in coordinates[neighbors] - coordinates[cur_index] + ] + distances = [math.dist(coordinates[cur_index], coordinates[idx]) for idx in neighbors] + + # Create sorted array by partition and then distance + sorted_arr = np.array(sorted(list(zip(partitions, distances, neighbors, strict=False)))) + + # Split into partitions + partitions_diff = sorted_arr[1:, 0] - sorted_arr[:-1, 0] + separators = np.where(partitions_diff > 0)[0] + partitions_split = np.split(sorted_arr, separators + 1) + partitions_split = [partition[:min_links] for partition in partitions_split] + + # Collect the neighbor 'hierarchies' + neighbor_hierarchies = [[] for _ in range(min_links)] + count = 0 + for n in range(min_links): + # Break early if we have gathered enough neighbors. + if min_links <= count: + break + for partition in partitions_split: + # Note that we do not break inside this loop, + # because we want the nth nearest neighbors from all partitions. + if n < len(partition) and (max_length is None or partition[n][1] <= max_length): + neighbor_hierarchies[n].append(partition[n]) + count = count + 1 + + # Sort hierarchies per partition by distance to the current node + neighbor_hierarchies = [ + sorted(hierarchy, key=lambda x: x[1]) for hierarchy in neighbor_hierarchies if len(hierarchy) != 0 + ] + + # Add sorted arcs to at least min_links neighbors + cur_arcs = [ + (min(cur_index, int(neighbor[2])), max(cur_index, int(neighbor[2]))) + for hierarchy in neighbor_hierarchies + for neighbor in hierarchy + ] + cur_arcs = cur_arcs[:min_links] + + arcs.extend(cur_arcs) + + # Remove duplicates and make the list canonical. + arcs = sorted(list(set(arcs))) + + return arcs + + +def get_ordered_arcs( + x_ref_search, + y_ref_search, + buffer_radius_ref, + dist_to_quality, + n_max_arcs, + nad_nmad_max, + x_coordinates, + y_coordinates, + slc_quality, + nad_nmad_vals, + coordinate_type, +): + """Get a list with ordered arcs based on pnt quality and a search area. + + Args: + ---- + x_ref_search (float): x-coordinate of the centre of the search area + y_ref_search (float): y-coordinate of the centre of the search area + buffer_radius_ref (float): The buffer (in m) around the centre coordinates where potential arcs are computed + dist_to_quality (float): parameter that relates arc length to additional sigma + n_max_arcs (float): The maximum nr of arcs to be outputed + nad_nmad_max (float): The maximum NAD / NMAD, for the entire time series, for a point to be considered + x_coordinates (xr.DataArray): x-coordinates of all points in the STM + y_coordinates (xr.DataArray): y-coordinates of all points in the STM + slc_quality (xr.DataArray): quality of the estimates per partition + nad_nmad_vals (xr.DataArray): NAD / NMAD of the entire time series per point + coordinate_type: 'Euclidean' (RD) / 'geographic' (lon/lat) + + Returns: + ------- + arcs (list): ordered arcs + arcs_and_quality (list): ordered list with the points and quality + quality_dict (dictionary): dictionary with the arcs and their quality + """ + # Find all points within the buffer around the starting location x, y + idx_pnts_buffer_ref = find_points_within_buffer( + x_coordinates, y_coordinates, x_ref_search, y_ref_search, buffer_radius_ref, coordinate_type + ) + + # Get the a-priori quality of all potential arcs that can be made + arcs_and_quality = _ordered_arcs_all_points( + x_coordinates, + y_coordinates, + slc_quality, + idx_pnts_buffer_ref, + dist_to_quality, + nad_nmad_vals, + nad_max=nad_nmad_max, + ) + + # We will only work with n_max_arcs, otherwise we need to load an extensive dataset everytime + arcs_and_quality = arcs_and_quality[0:n_max_arcs] + + # Only store the arcs, and remove the quality values + arcs = [] + for _, arc in arcs_and_quality: + arcs.append(arc) + + # Create a dictionary with the quality of the arcs + quality_dict = {tuple(sorted(arc[1])): arc[0] for arc in arcs_and_quality} + + return arcs, arcs_and_quality, quality_dict + + +def find_points_within_buffer( + x_coords, + y_coords, + x_pnts, + y_pnts, + buffer_radius, + coordinate_type: Literal["euclidean", "geographic"] = "euclidean", + return_aggregate_point_buffer: bool = True, +): + """Find all points located within a specified buffer radius around a given location. + + This function determines which points in a set of coordinates are located within a defined buffer + distance around one or more specified points. + + The function: + 1. Converts the input x and y coordinates of the search point(s) to arrays if they are not already. + 2. Computes the distance between each point in `x_coords` and `y_coords` and the reference points. + 3. Returns the indices of these points for further processing or analysis. + + Args: + ---- + x_coords (numpy.ndarray): Array of x-coordinates for all points in the dataset. + y_coords (numpy.ndarray): Array of y-coordinates for all points in the dataset. + x_pnts (float or array-like): The x-coordinate or array of x-coordinates of the reference point(s). + y_pnts (float or array-like): The y-coordinate or array of y-coordinates of the reference point(s). + buffer_radius (float): The radius of the buffer zone around the reference points, specified in meters. + coordinate_type (Literal["euclidean", "geographic"]): whether the coordinates provided are Euclidean (such as + RD) or geographic (such as lon / lat) + return_aggregate_point_buffer (bool): if True, a list of indices of the points within the buffer of any of the + points in x_pnts/y_pnts is returned. If False, the list of indices within the buffer of each point in + x_pnts/y_pnts is returned individually. Default True + + Returns: + ------- + indices (numpy.ndarray): Array of indices of the points located within the buffer zone, either the aggregate + buffer zone (`return_aggregate_point_buffer`=True) or per point in `x_pnts`/`y_pnts` + (`return_aggregate_point_buffer`=False) + + Example: + indices = find_points_within_buffer(x_coords, y_coords, x_pnts=10.5, y_pnts=20.3, buffer_radius=5.0) + (with Euclidean coordinates and an aggregate buffer output) + """ + # Make sure x_pnts and y_pnts are arrays + x_pnts = np.atleast_1d(x_pnts) + y_pnts = np.atleast_1d(y_pnts) + assert len(x_pnts.shape) == 1, f"Received more than 1 dimension in x_pnts (shape {x_pnts.shape})!" + assert len(y_pnts.shape) == 1, f"Received more than 1 dimension in y_pnts (shape {y_pnts.shape})!" + + if coordinate_type == "euclidean": + # could possibly also be a KDTree but for consistency inside the function the minkowski metric does Euclidean + tree = BallTree(np.vstack([x_coords, y_coords]).T, metric="minkowski") + search_radius = buffer_radius + search_points = np.vstack([x_pnts, y_pnts]).T + elif coordinate_type == "geographic": + # the geographic tree assumes an Earth radius of 1, so we need to divide the search radius by the radius of + # the Earth + # It also expects input in radians instead of degrees, and first latitude (y), then longitude (x) + tree = BallTree(np.vstack([np.radians(y_coords), np.radians(x_coords)]).T, metric="haversine") + search_radius = buffer_radius / EARTH_RADIUS + search_points = np.vstack([np.radians(y_pnts), np.radians(x_pnts)]).T + else: + raise ValueError(f"Unknown coordinate type {coordinate_type}! Known are euclidean and geographic.") + + res_indices = tree.query_radius(search_points, search_radius) + + if not return_aggregate_point_buffer: + return res_indices + + all_indices = [] + for i in res_indices: # flatten won't work since res_indices is an irregularly shaped numpy array + for j in i: + all_indices.append(j) + indices = np.array(list(sorted(list(set(all_indices))))) + + return indices + + +def _ordered_arcs_all_points(rdx, rdy, slc_quality, idx_pnts_buffer, dist_to_quality, nad_vals, nad_max): + """Generate a sorted list of unique arcs between all points within buffer, ranked by combined quality and distance. + + This function computes arcs between all points in a given buffer based on their spatial distance and + quality metrics. The arcs are ranked according to a combination of the Euclidean distance between + points and the maximum quality value between them. + + The function: + 1. Computes the Euclidean distance matrix for all points within the buffer. + 2. Calculates the arc quality time series for all point pairs based on their SLC quality values. + 3. Extracts the maximum quality for each arc and combines it with the distance between the points. + 4. Uses the lower triangular matrix to avoid duplicate arcs + 5. Sorts the arcs based on the combined distance and quality metric. + + Args: + ---- + rdx (numpy.ndarray): Array of x-coordinates for all points. + rdy (numpy.ndarray): Array of y-coordinates for all points. + slc_quality (numpy.ndarray): Array of SLC quality time series for all points. + idx_pnts_buffer (numpy.ndarray): Indices of the points within the buffer. + dist_to_quality (float): Scaling factor for weighting the distance in combination with the quality. + nad_vals (numpy.ndarray): Array of NAD values of all points. + nad_max (float): The maximum NAD, for the entire time series, for a point to be considered + + Returns: + ------- + arcs_and_quality (list): Sorted list of tuples, each tuple contains the combined quality and distance value + and the indices of the two points forming the arc. + + Example: + ------- + arcs_and_quality = ordered_arcs_all_points(rdx, rdy, slc_quality, idx_pnts_buffer, dist_to_quality) + """ + # Compute NAD values of the points within the buffer + nad_buffer_vals = nad_vals[idx_pnts_buffer] + + # Mask points that have an NAD value above the threshold + mask_nad = nad_buffer_vals < nad_max + idx_pnts_buffer = idx_pnts_buffer[mask_nad] + + # Compute the distance matrix for all points (below the NAD threshold) within the buffer + rdx_buffer = rdx[idx_pnts_buffer] + rdy_buffer = rdy[idx_pnts_buffer] + coords_points = np.vstack((rdx_buffer, rdy_buffer)).T + dist_matrix = distance_matrix(coords_points, coords_points) + + # Compute the quality matrix for all potential arcs within the buffer + slc_quality_buffer = slc_quality[idx_pnts_buffer, :] + slc_quality_i_buffer = slc_quality_buffer[:, np.newaxis] + slc_quality_j_buffer = slc_quality_buffer[np.newaxis, :] + + # Compute the arc quality time series + # quality is based on both sides of the arc + arc_quality_ts = np.sqrt(slc_quality_i_buffer**2 + slc_quality_j_buffer**2) + + # Get the maximum value in the time dimension per arc + # This is considered as the 'worst' quality for the entire time period + arcs_quality_max = np.max(arc_quality_ts, axis=2) + + # Get only the lower triangular matrix + arcs_quality_max_lower = np.tril(arcs_quality_max) + arcs_dist = np.tril(dist_matrix) + + # Add additional sigma because of the arc length + arc_quality_dist_max = arcs_quality_max_lower + arcs_dist * dist_to_quality + + # Order the arcs + arcs_and_quality = [] + + # Loop trough the lower tringular matrix and add values list + rows, cols = arc_quality_dist_max.shape + for i in range(rows): + for j in range(i): # j < i makes sure that we only get the lower triangular matrix + arcs_and_quality.append((arc_quality_dist_max[i, j], (idx_pnts_buffer[i], idx_pnts_buffer[j]))) + + # Sort the list with all the arcs from best to worse + arcs_and_quality.sort() + + return arcs_and_quality + + +def construct_control_network( + arcs, + quality_dict_arcs, + excluded_arcs, + n_top, + n_batch, + deg_threshold, + min_nodes, + min_redundancy, + visualize_network=False, +): + """Construct a control network of arcs based on quality metrics and structural requirements. + + This function iteratively builds a network from a search area of arcs, filtering out excluded arcs + (where an excluded arc is an arc we know we don't want to have) and ensuring that the resulting + network meets specified criteria, such as a minimum number of nodes and redundancy. + The network is constructed by: + 1. Ranking arcs based on quality metrics provided in `quality_dict_arcs`. + 2. Excluding arcs that are in the `excluded_arcs` list. + 3. Iteratively adding arcs in batches and refining the network to remove nodes with low centrality. + 4. Evaluating the network against requirements such as minimum nodes and average redundancy. + + If the requirements are not met with the given arcs, a warning is issued. Optionally, the network + construction process can be visualized. + + Args: + ---- + arcs (list of tuples): List of arcs (pairs of points) within the search area. + quality_dict_arcs (dict): Dictionary mapping arcs (tuples) to a sigma value. + excluded_arcs (list of tuples): List of arcs that should be excluded from the network. + n_top (int): Number of top-ranked arcs to start with. + n_batch (int): Number of additional arcs to add in each iteration. + deg_threshold (float): Degree threshold for removing low-centrality nodes from the network. + min_nodes (int): Minimum number of nodes required in the final network. + min_redundancy (float): Minimum average redundancy (degree) required in the final network. + visualize_network (bolean, optional): If set to True, the network construction process is visualized. + + Returns: + ------- + ref_pnt (int): The reference point used in the final iteration of the network. + arcs_updated_network_sorted (list of tuples): The sorted list of arcs in the final network + based on quality metrics. + ref_pnt_initial (int): The reference point used in the initial network. + arcs_initial_network (list of tuples): The list of arcs in the network before refinement. + + Example: + ------- + ref_pnt, final_arcs, initial_ref_pnt, initial_arcs = construct_control_network( + arcs, quality_dict_arcs, excluded_arcs, + n_top=50, n_batch=10, + deg_threshold=1.5, min_nodes=20, min_redundancy=2.0, + visualize_network=False + ) + """ + # Sort arcs by quality, removing excluded arcs from the ranked arcs variable + excluded_arcs_sorted = [tuple(sorted(arc)) for arc in excluded_arcs] + arcs_without_excluded = [arc for arc in arcs if tuple(sorted(arc)) not in excluded_arcs_sorted] + + # Initialize variables + current_network = None + avg_degree = 0 + iteration = 0 + + more_arcs_to_test = True + network_reqs_not_met = True + + # the following while loop will stop when either the network requirements (# nodes and redundancy level) are met, + # OR when we run out of arcs to text + while more_arcs_to_test and network_reqs_not_met: + # Select the arcs for the current iteration + iteration += 1 + end_idx = n_top + iteration * n_batch + arcs_to_test = arcs_without_excluded[0:end_idx] + + if not arcs_to_test: # Break if no more arcs to add + print("No more arcs to test.") + more_arcs_to_test = False + continue + + # Create a new network using the selected arcs + current_network = nx.Graph() + current_network, _, ref_pnt = _from_arcs_to_graph(arcs_without_excluded[:end_idx], plot=visualize_network) + if iteration == 1: + ref_pnt_initial = ref_pnt + arcs_initial_network = [tuple(sorted(arc)) for arc in current_network.edges()] + + # Remove nodes with degree equal or lower than the threshold + current_network, _, ref_pnt = _remove_low_centrality_nodes( + current_network, deg_threshold=deg_threshold, plot=visualize_network + ) + arcs_updated_network = [tuple(sorted(arc)) for arc in current_network.edges()] + + # Test the network against requirements + avg_degree = len(current_network.edges) / len(current_network.nodes) if len(current_network.nodes) > 0 else 0 + print(f"The average degree is {avg_degree:.2f}") + + if len(current_network.nodes) >= min_nodes and avg_degree >= min_redundancy: + network_reqs_not_met = False + # Stop if requirements are met + + # Final check if the network meets requirements + if len(current_network.nodes) < min_nodes or avg_degree < min_redundancy: + print("Warning: Network could not meet all requirements with the given arcs.") + + # The arcs constructed above are no longer sorted based on their quality + arcs_updated_network_sorted = sorted( + arcs_updated_network, key=lambda arc: quality_dict_arcs.get(tuple(sorted(arc)), float("inf")) + ) + + return ref_pnt, arcs_updated_network_sorted, ref_pnt_initial, arcs_initial_network + + +def _from_arcs_to_graph(arcs, plot=False, save_path="./network.png"): + """Construct a graph from a set of arcs and identifies key properties of the network. + + Args: + ---- + arcs : (list of tuple) + List of arcs (edges) in the graph. Each arc is represented as a tuple + of two nodes, e.g., [(node1, node2), (node2, node3)]. + plot : (boolean) + Default is False, If True, the function visualizes the graph structure, including highlighting + connected components. Default is False (no visualization). + save_path : (string) + Default = "./network.png" + + Returns: + ------- + tuple: A tuple containing: + - network (networkx.Graph): The constructed graph based on the provided arcs. + - degree_centrality (dict): A dictionary where keys are nodes and values are their + degree centrality scores. + - ref_pnt (int): The index of the reference point (point with most connections). + """ + if not arcs: + raise ValueError("The input arcs list is empty. Please provide a valid list of arcs.") + + # Compute the network + network = nx.Graph() + network.add_edges_from(arcs) + + # Compute the degree centrality + # Which says something on how connected one particular node is with other nodes + degree_centrality = nx.degree_centrality(network) + + # Get the reference point (node with the most connections) + ref_pnt = max(degree_centrality, key=degree_centrality.get) + + if plot: + # Visualize the network + pos = nx.spring_layout(network) # Lay-out for the graph + nx.draw( + network, + pos, + with_labels=True, + node_size=500, + node_color="skyblue", + font_size=10, + font_weight="bold", + edge_color="gray", + ) + + connected_components = list(nx.connected_components(network)) + # Highlight the connected components + for component in connected_components: + nx.draw_networkx_nodes(network, pos, nodelist=component, node_color="orange", node_size=700) + + # Highlight the reference point + nx.draw_networkx_nodes(network, pos, nodelist=[ref_pnt], node_color="red", node_size=800) + plt.title(f"Network Visualization - {len(connected_components)} Connected Components") + plt.savefig(save_path, bbox_inches="tight") + + return network, degree_centrality, ref_pnt + + +def _remove_low_centrality_nodes(network, deg_threshold, plot=False, save_path="./network.png"): + """Remove nodes with a low degree of centrality. + + Args: + ---- + network (networkx.Graph): The input graph from which nodes will be removed. + deg_threshold (int): The degree threshold; nodes with a degree equal to or less than this value will be removed. + plot (bolean, optional): If True, the function visualizes the updated graph, including its connected components. + Default is False (no visualization). + save_path : (string) + Default = "./network.png" + + Returns: + ------- + tuple: A tuple containing: + - network (networkx.Graph): The updated graph after removing nodes with the specified degree threshold. + - degree_centrality (dict): A dictionary where keys are nodes and values are their degree centrality scores. + - ref_pnt (int or None): The node with the highest degree centrality in the updated graph. + + Example: + ------- + updated_network, degree_centrality, ref_point = remove_low_centrality_nodes( + network, deg_threshold=1, plot=False + ) + """ + # Identify nodes to remove + nodes_to_remove = [node for node, degree in network.degree() if degree <= deg_threshold] + removed_nodes = set(nodes_to_remove) # For plot highlighting + + # Remove these nodes from the network + network.remove_nodes_from(nodes_to_remove) + + # Handle empty graph case + if not network.nodes: + return network, {}, None + + # Degree centrality + degree_centrality = nx.degree_centrality(network) + + # Get the reference point (node with the highest degree centrality) + ref_pnt = max(degree_centrality, key=degree_centrality.get) + + if plot: + # Visualize the updated network + pos = nx.spring_layout(network) + nx.draw( + network, + pos, + with_labels=True, + node_size=500, + node_color="skyblue", + font_size=10, + font_weight="bold", + edge_color="gray", + ) + + connected_components = list(nx.connected_components(network)) + + # Highlight connected components + for component in connected_components: + nx.draw_networkx_nodes(network, pos, nodelist=component, node_color="orange", node_size=700) + + # Highlight removed nodes in the plot (optional) + if removed_nodes: + # Filter valid nodes that exist in the layout + valid_removed_nodes = [node for node in removed_nodes if node in pos] + nx.draw_networkx_nodes(network, pos, nodelist=valid_removed_nodes, node_color="red", node_size=800) + + nx.draw_networkx_nodes(network, pos, nodelist=[ref_pnt], node_color="red", node_size=800) + + plt.title(f"Network Visualization after Removing Nodes with Degree ≤ {deg_threshold}") + plt.savefig(save_path, bbox_inches="tight") + + return network, degree_centrality, ref_pnt + + +def test_succeeded_arcs_control_network( + succeeded_arcs, quality_dict_arcs, deg_threshold, min_nodes, min_redundancy, visualize_network=False +): + """Evaluate a network constructed from a set of arcs to determine if it meets structural requirements. + + This function tests a network built from a set of arcs to ensure it satisfies minimum structural criteria, + including the number of nodes, redundancy, and centrality. It checks whether any nodes with low degree should + be removed and then evaluates the network again. The arcs are sorted by quality after refinement. + + Parameters + ---------- + succeeded_arcs : list of tuples + A list of arcs (pairs of points) that are used to construct the network. + quality_dict_arcs : dict + A dictionary where the keys are the arcs and the values are their respective quality scores. + deg_threshold : float + The degree threshold for identifying low-centrality nodes. Nodes with degrees smaller will be removed. + min_nodes : int + The minimum number of nodes required for the network to be valid. + min_redundancy : float + The minimum average degree (redundancy) required for the network. + visualize_network : bool, optional + A flag to control network visualization of the network during evaluation. Default is False (no visualization). + + Returns + ------- + network_check : int + A flag indicating whether the network meets the requirements (1 if successful, 0 if not). + arcs_updated_network_sorted : list of tuples + The sorted list of arcs based on their quality after refining the network. + ref_pnt : int + The reference point used in the network. + + Example + ------- + network_check, final_arcs, ref_pnt = test_succeeded_arcs_control_network( + succeeded_arcs, deg_threshold=1.5, min_nodes=20, + min_redundancy=2.0, visualize_network=1 + ) + """ + arcs_updated_network_sorted = [] + network_check = 0 + + # Construct the network based on the input arcs + current_network = nx.Graph() + current_network, _, ref_pnt = _from_arcs_to_graph(succeeded_arcs, plot=visualize_network) + arcs_updated_network = [tuple(sorted(arc)) for arc in current_network.edges()] + + # Check whether there are nodes with degree smaller than one + nodes_to_remove = [node for node, degree in current_network.degree() if degree <= deg_threshold] + + if not nodes_to_remove: + # Test the network against requirements + avg_degree = len(current_network.edges) / len(current_network.nodes) if len(current_network.nodes) > 0 else 0 + print(f"The average degree is {avg_degree:.2f}") + + if len(current_network.nodes) >= min_nodes and avg_degree >= min_redundancy: + network_check = 1 + + # The arcs constructed above are no longer sorted based on their quality + arcs_updated_network_sorted = sorted( + arcs_updated_network, key=lambda arc: quality_dict_arcs.get(tuple(sorted(arc)), float("inf")) + ) + + else: + print("We remove low degree nodes") + # Remove the nodes with low degree and test the network again + current_network, _, ref_pnt = _remove_low_centrality_nodes( + current_network, deg_threshold=deg_threshold, plot=visualize_network + ) + arcs_updated_network = [tuple(sorted(arc)) for arc in current_network.edges()] + + # Test the network against requirements + avg_degree = len(current_network.edges) / len(current_network.nodes) if len(current_network.nodes) > 0 else 0 + print(f"The average degree is {avg_degree:.2f}") + + if len(current_network.nodes) >= min_nodes and avg_degree >= min_redundancy: + network_check = 1 + + # The arcs constructed above are no longer sorted based on their quality + arcs_updated_network_sorted = sorted( + arcs_updated_network, key=lambda arc: quality_dict_arcs.get(tuple(sorted(arc)), float("inf")) + ) + + else: + network_check = 0 + + return network_check, arcs_updated_network_sorted, ref_pnt + + +def ordered_arcs_connection_point_and_control_network( + rdx_connection_point, + rdy_connection_point, + slc_quality_connection_point, + connection_point_idx, + rdx_control, + rdy_control, + slc_quality_control, + control_idx, + dist_to_quality, + coordinate_type: Literal["euclidean", "geometric"], +): + """Generate a sorted array of unique arcs between a 'connection_point' and the control network. + + This function computes arcs between a connection point and control points, evaluating each arc based on + a combination of spatial distance and SLC quality. The arcs are sorted from best to worst, where "best" + is determined by the lowest combined quality and distance value. + + The function: + 1. Computes the Euclidean distance between the connection point and the control points. + 2. Calculates the arc quality time series for each arc. + 3. Combines the distance and maximum quality for each arc to define an overall quality value. + 4. Sorts the arcs by quality values from best (lowest) to worst (highest). + 5. Constructs a sorted array of arcs, ensuring the control point appears first in each arc. + + Args: + ---- + rdx_connection_point (xarray.DataArray): x-coordinate of the connection point. + rdy_connection_point (xarray.DataArray): y-coordinate of the connection point. + slc_quality_connection_point (xarray.DataArray): Array of SLC quality time series for the connection point. + connection_point_idx (int): Index of the connection point. + rdx_control (xarray.DataArray): x-coordinates of the control points. + rdy_control (xarray.DataArray): y-coordinates of the control points. + slc_quality_control (xarray.DataArray): Array of SLC quality time series for the control points. + control_idx (xarray.DataArray): Indices of the control points. + dist_to_quality (float): Conversion factor to scale distance relative to quality. + coordinate_type (Literal["euclidean", "geographic"]): whether the provided coordinates are Euclidean or + geographic + + Returns: + ------- + tuple: + - arcs (numpy.ndarray): A sorted array of arcs, where each row contains a control point + followed by the connection point. + - sorted_quality_values (numpy.ndarray): Sorted quality values corresponding to the arcs. + """ + # Stack the coordinates of the connection point and the control points + coords_connection_point = np.vstack((rdx_connection_point, rdy_connection_point)).T + coords_control = np.vstack((rdx_control, rdy_control)).T + + # Distance matrix between the connection point and the control points + if coordinate_type == "euclidean": + dist_matrix = distance_matrix(coords_connection_point, coords_control).squeeze() + else: + dist_matrix = np.zeros((coords_connection_point.shape[0], coords_control.shape[0])) + for m in range(coords_connection_point.shape[0]): + for n in range(coords_control.shape[0]): + dist_matrix[m, n] = get_distance(coords_connection_point[m], coords_control[n], mode=coordinate_type) + + # # Compute the quality matrix for connection point (pnt i) and all control points (point j) + slc_quality_j = slc_quality_control.values # Quality values of the control points + slc_quality_i = np.expand_dims(slc_quality_connection_point.values, axis=0) # Make sure the dimensions match + slc_quality_i = np.repeat(slc_quality_i, repeats=slc_quality_j.shape[0], axis=0) + + # Compute arc quality time series (which is a function of the slc_quality of point i and point j) + arc_quality_ts = np.sqrt(slc_quality_i**2 + slc_quality_j**2) + arc_quality_max = np.max(arc_quality_ts, axis=1) # Compute the maximum value + + # Combine distance and quality + arc_quality_dist_max = arc_quality_max + dist_matrix * dist_to_quality + + # Sort the arcs and compute the indices of the control points (since point i, the connection_point, is in all arcs) + sorted_control = np.argsort(arc_quality_dist_max) + sorted_control_idx = control_idx.values[sorted_control] # sort the indices of the control points as well + sorted_quality_values = np.sort(arc_quality_dist_max) + + # Compute the arcs between the connection_point and the control points. + # Make sure that the control_points comes first + arcs = np.zeros((len(sorted_control_idx), 2), dtype=int) + arcs[:, 0] = sorted_control_idx # control network points + arcs[:, 1] = connection_point_idx # connection_point + + return arcs, sorted_quality_values + + +def construct_control_network_test_arcs( + x_ref_search, + y_ref_search, + buffer_radius_ref, + dist_to_quality, + N_max_arcs, + N_top, + N_batch, + deg_threshold, + min_nodes, + min_redundancy, + nad_nmad_max, + visualize_network, + sigma_post_over_sigma_prior, + nr_max_iter_control, + bounds, + m2ph, + years, + dates, + temperature, + sd_complex, + slc_quality, + cr2ph, + ampl_ts, + bkps_stm, + mean_ampl_sd, + sigma_ampl_sd, + mad_ampl_sd, + median_ampl_sd, + x_coordinates, + y_coordinates, + nad_nmad, + coordinate_type: Literal["euclidean", "geographic"] = "euclidean", +): + """Construct the control network while testing the arcs inside. + + Parameters + ---------- + x_ref_search: + X coordinate of central point to search for reference point + y_ref_search: + Y coordinate of central point to search for reference point + buffer_radius_ref: + Size of search window for reference point + dist_to_quality: + relates arc length to additional sigma + N_max_arcs: + Maximum number of arcs + N_top: + Number of top-ranked arcs to start with. + N_batch: + Number of additional arcs to add in each iteration. + deg_threshold: + Degree threshold for removing low-centrality nodes from the network. + min_nodes: + Minimum number of nodes required in the final network. + min_redundancy: + Minimum average redundancy (degree) required in the final network. + nad_nmad_max: + Maximum NAD / NMAD for a point to be considered + visualize_network: + Boolean whether or not to visualize the network + sigma_post_over_sigma_prior: + Upper limit on how much the aposteriori sigma of an arc is allowed to differ from the apriori sigma + nr_max_iter_control: + Maximum number of iterations in the network testing + bounds: list of tuples + Bounds for parameter estimation in the format (lower_bounds, upper_bounds). + m2ph: float + Meters to phase conversion factor + years: np.ndarray + Array of decimal years corresponding to the time series epochs. + dates: np.ndarray + Array of date indices or timestamps corresponding to the time series. + temperature: np.ndarray + Array of temperature values for thermal expansion modeling. + sd_complex: np.ndarray + Single-difference complex + slc_quality: np.ndarray + Complex-valued standard deviations of the signal for all points. + cr2ph: np.ndarray + crossrange-to-phase for each point + ampl_ts: np.ndarray + amplitude timeseries + bkps_stm: np.ndarray + Breakpoints stm + mean_ampl_sd: np.ndarray + Single difference mean amplitude + sigma_ampl_sd: np.ndarray + Single difference mean amplitude standard deviation + mad_ampl_sd: np.ndarray + Median absolute deviation of the single difference amplitude + median_ampl_sd: np.ndarray + Median single difference amplitude + x_coordinates: np.ndarray + x coordinates (RD or longitude) + y_coordinates: np.ndarray + y coordinates (RD or latitude) + nad_nmad: np.ndarray + NAD or NMAD to be used in the point selection upon which `nad_nmad_max` is imposed. + coordinate_type: Literal["euclidean", "geographic"] = "euclidean" + Whether the provided coordinates are Euclidean (such as RD) or geographic (such as lon/lat) + + Returns + ------- + dict + Results of the accepted network + list + Reference point + list + Results of the accepted arcs + """ + # At the start, no arcs are tested yet, so we don't have failed_arcs, and noisy_arcs + failed_arcs = [] + noisy_arcs = [] + + # Get an ordered list of all potential arcs with a buffer area + arcs_search_area, _, quality_dict_arcs = get_ordered_arcs( + x_ref_search, + y_ref_search, + buffer_radius_ref, + dist_to_quality, + N_max_arcs, + nad_nmad_max, + x_coordinates, + y_coordinates, + slc_quality, + nad_nmad, + coordinate_type, + ) + + network_meets_requirements = False + + while not network_meets_requirements: + # Create a first network based on the ranked quality and leaving out the failed_arcs and noisy_arcs + ref_pnt, arcs_updated_network, ref_pnt_initial, arcs_initial_network = construct_control_network( + arcs_search_area, + quality_dict_arcs, + failed_arcs, + N_top, + N_batch, + deg_threshold, + min_nodes, + min_redundancy, + False, + ) + + print("Computing the solutions for the arcs") + # Test whether solutions for all arcs can be found (sometimes it happens that because of the complex + # functions no solutions can be found) + results_initial_control_network_v1 = arc_estimation_control_network( + arcs_updated_network, + bounds, + m2ph, + nr_max_iter_control, + years, + dates, + temperature, + sd_complex, + slc_quality, + cr2ph, + ampl_ts, + bkps_stm, + mean_ampl_sd, + sigma_ampl_sd, + mad_ampl_sd, + median_ampl_sd, + x_coordinates, + y_coordinates, + coordinate_type, + ) + + # Save the failed arcs to an array (such that they are not taken into account any more) + succeeded_arcs = [ + arc + for arc, succeeded in zip( + arcs_updated_network, results_initial_control_network_v1["succeeded_arcs"], strict=True + ) + if not np.isnan(succeeded).any() + ] + + failed_arcs_temp = [ + arc + for arc, succeeded in zip( + arcs_updated_network, results_initial_control_network_v1["succeeded_arcs"], strict=True + ) + if np.isnan(succeeded).any() or not succeeded.any() + ] + failed_arcs = list(set(failed_arcs).union(failed_arcs_temp)) + print(f"Failed arcs {failed_arcs}") + + # Remove the failed arcs from the dictionary with all the results (as the estimated parameters etc) + results_initial_control_network_v2 = {} # Make a new dictionary where we will not save + # the results of the failed_arcs + + # Find the indices where the arcs are saved where no solution was found. + # These arcs have nan values + valid_indices = ~np.isnan(results_initial_control_network_v1["succeeded_arcs"]).any(axis=1) + + # Filter all variables in the dictionary and only copy the values for the arcs where we found a solution + for key, value in results_initial_control_network_v1.items(): + if ( + isinstance(value, np.ndarray) + and value.shape[0] == results_initial_control_network_v1["succeeded_arcs"].shape[0] + ): + results_initial_control_network_v2[key] = value[valid_indices] + else: + results_initial_control_network_v2[key] = value # Keep values that are not row-based unchanged + + print("Check if the network with the solved arcs still meets our requirements") + + # Since we have 'failed_arcs', where no solution was found, the new network need to be tested + # It can happen that we have isolated points, + # or that the average degree of the network is not high enough anymore + network_check, arcs_updated_network, ref_pnt = test_succeeded_arcs_control_network( + succeeded_arcs, quality_dict_arcs, deg_threshold, min_nodes, min_redundancy, visualize_network + ) + + if network_check == 0: + print("Network fails requirements, starting again") + + if network_check == 1: + print("Network meets requirements") + # After the last test, the isolated arcs are removed (so they are still in + # 'succeeded_arcs' and in the dictionary) + # And these arcs need to be removed from the dictionary + missing_indices = [ + idx for idx, arc in enumerate(succeeded_arcs) if arc not in arcs_updated_network + ] # missing indices are the arcs that were isolated and removed. But they are still in the + # dictionary so there they need to be removed as well + + results_control_network_temp = {} + for key, value in results_initial_control_network_v2.items(): + if isinstance(value, np.ndarray): + if value.ndim in {1, 2}: # Both 1D and 2D arrays here + results_control_network_temp[key] = np.delete(value, missing_indices, axis=0) + else: + results_control_network_temp[key] = value + else: + results_control_network_temp[key] = value + + # Now we have a network that fullfills requirements but there might be noisy arcs + # We compute solutions for all arcs and computed RMSE + # In the next part, we will remove any arcs that are too noisy for the requirements, and then test if + # the network still fulfills the requirements + print("Calculate whether there are arcs where the solution that we found is noisy") + + est_displ_phase = ( + results_control_network_temp["unwrap_phases_arc"] + - results_control_network_temp["estimated_cross_range_phase"] + - results_control_network_temp["estimated_thermal_phase"] + ) + displ_phase = results_control_network_temp["estimated_displ_phase"] + + residues_per_arc = est_displ_phase - displ_phase + sigma_post_arc = np.std(residues_per_arc, axis=1) + mean_sigma_prior_arc = np.mean(results_control_network_temp["sigma_phases_arc"], axis=1) + + idx_bad_arcs = np.where(sigma_post_arc >= sigma_post_over_sigma_prior * mean_sigma_prior_arc)[0] + idx_good_arcs = np.where(sigma_post_arc < sigma_post_over_sigma_prior * mean_sigma_prior_arc)[0] + + bad_arcs = results_control_network_temp["succeeded_arcs"][idx_bad_arcs] + good_arcs = results_control_network_temp["succeeded_arcs"][idx_good_arcs].astype(int) + good_arcs = [tuple(row) for row in good_arcs] # Change the output to a list + + print(f"The value for sigma_post_over_prior is {sigma_post_over_sigma_prior}") + + print("we removed arcs") + + print("The bad arcs are") + print(idx_bad_arcs) + print("The good arcs are") + print(idx_good_arcs) + + # Add the noisy arcs to the list with failed_arcs + failed_arcs = list(set(failed_arcs).union([tuple(row.astype(int)) for row in bad_arcs])) + noisy_arcs = list(set(noisy_arcs).union([tuple(row.astype(int)) for row in bad_arcs])) + + # Check whether the network without the noisy arcs still meets requirements + print("Check whether the network stil meets the requirements, even after the removal of bad arcs") + network_check_good, arcs_updated_network, ref_pnt = test_succeeded_arcs_control_network( + good_arcs, quality_dict_arcs, deg_threshold, min_nodes, min_redundancy, visualize_network=False + ) + + if network_check_good == 0: + print("Network does not meet requirements, start over") + + if network_check_good == 1: + print("We are happy! The network consisting of the good arcs fulfills the requirements.") + + # If there are noisy arcs, they need to be removed from the final network + # Convert noisy_arcs to set for quicker lookup + noisy_arcs_set = {tuple(map(float, arc)) for arc in noisy_arcs} + + # Get the current succeeded_arcs + succeeded_arcs = results_control_network_temp["succeeded_arcs"] + + # Deterimine which arcs to keep (so NOT in noisy_arcs_set) + valid_indices = np.array([tuple(row) not in noisy_arcs_set for row in succeeded_arcs]) + + # New dict with only valid rows + results_control_network = {} + + for key, value in results_control_network_temp.items(): + if isinstance(value, np.ndarray) and value.shape[0] == succeeded_arcs.shape[0]: + results_control_network[key] = value[valid_indices] + else: + results_control_network[key] = value # Leave unrelated variables unchanged + + network_meets_requirements = True # Stop the loop + + if not network_meets_requirements: + # If the network doesn't meet the requirements, start over + print("Network does not meet the requirements. Recomputing the network with updated failed_arcs...") + + return results_control_network, ref_pnt, arcs_updated_network + + +def _network_relation_matrix(idx_source, idx_target, n_points, idx_refpnt, sparse_mode: bool = False): + """Create the network relation matrix A as a sparse matrix. + + A network relation matrix has shape (n_arcs, n_points - 1). + Each row corresponds to an arc, and each column corresponds to a point, excluding the reference point. + For each arc, the column corresponding to the source point has a value of -1, and the column corresponding + to the target point has a value of +1. + All other entries are zero. + + The reference point column removal refers to Eq.4.11 of Kampes (2006). + + Parameters + ---------- + idx_source : list or np.ndarray + List of source point indices for each arc. + idx_target : list or np.ndarray + List of target point indices for each arc. + n_points : int + Total number of points in the network. + idx_refpnt : int + Index of the reference point to be excluded from the matrix. This index assumes 0-based indexing of the points. + sparse_mode : bool + Whether to return the matrix in sparse format. If False, returns a dense numpy array. + + References + ---------- + Kampes, Bert M. Radar interferometry: persistent scatterer technique. Dordrecht: Springer Netherlands, 2006. + DOI: 10.1007/978-1-4020-4723-7 + """ + n_arcs = len(idx_source) + A_start = sparse.COO( + (np.arange(n_arcs), idx_source), + np.full_like(np.arange(n_arcs), -1, dtype=np.int8), + shape=(n_arcs, n_points), + ) + A_end = sparse.COO( + (np.arange(n_arcs), idx_target), + np.full_like(np.arange(n_arcs), 1, dtype=np.int8), + shape=(n_arcs, n_points), + ) + A = A_start + A_end + + # Convert to Compressed Sparse Row (CSR) matrix for efficient arithmetic and matrix vector operations + A = A.tocsr() + + # Remove reference point column + A = scipy.sparse.hstack([A[:, :idx_refpnt], A[:, idx_refpnt + 1 :]]) + + if not sparse_mode: + A = np.array(A.todense()) + + return A + + +def _independent_arcs(arcs: np.ndarray) -> np.ndarray: + """Select independent arcs from a list of arcs. + + An arc is independent if its starting and ending points do not exist in any other arc's + starting or ending points. + + Parameters + ---------- + arcs : np.ndarray + A 2D array of shape (n_points, 2) where each row represents indices of the starting and ending points + of an arc. + + Returns + ------- + np.ndarray + A 2D array of independent arcs, where each row represents indices of the starting and ending points + of an arc. + """ + # Select arcs with unique starting points + _, unique_idx_start = np.unique(arcs[:, 0], return_index=True) + arcs = arcs[unique_idx_start, :] + + # Select arcs with unique ending points + _, unique_idx_end = np.unique(arcs[:, 1], return_index=True) + arcs = arcs[unique_idx_end, :] + + # After previous two steps, no arcs will share starting or ending points. + # However, there starting points may be the ending points of other arcs, and vice versa. + # To ensure independency, we loop through the rest arcs and add arc one by one + # In each interation, remove arcs that + # 1) start with the ending point of this arc, or + # 2) end with the starting point of this arc + arcs_selected = np.empty((0, 2), dtype=int) + while arcs.shape[0] > 0: + arc_current = arcs[0, :] + arcs_selected = np.append(arcs_selected, [arc_current], axis=0) + # Remove arcs which contain the starting point or ending point of the current arc + idx_remove = np.where((arcs[:, 1] == arc_current[0]) | (arcs[:, 0] == arc_current[1]))[0] + # add the index of the current arc to idx_remove + idx_remove = np.append(idx_remove, 0) + arcs = np.delete(arcs, idx_remove, axis=0) + + return arcs_selected diff --git a/stochastic.py b/stochastic.py new file mode 100644 index 00000000..90fac63a --- /dev/null +++ b/stochastic.py @@ -0,0 +1,160 @@ +"""stochastic model related functions.""" + +import numpy as np +import xarray as xr + +from depsi.arc_estimation import periodogram +from depsi.network import _independent_arcs, form_network +from depsi.utils import get_m2ph + +SIGMA_MOTHER_ATMO = 15.0 # std of mother atmosphere when its included in the stochastic model +SIGMA_OTHERS = 20.0 # std of other interferograms when master +SIGMA_OVERALL = 30 # std when mother atmosphere is in functional model instead of stochastic model +SIGMA_MINIMUM = 10 # minimum value for variance components, used to avoid negative values, in degrees + + +def vce_temporal( + stm: xr.Dataset, + key_phase: str, + key_Btemporal: str, + key_h2ph: str, + key_x="lon", + key_y="lat", + max_length: float = 0.01, # TODO: change distance from degree to meters + include_mother_atmo: bool = False, +) -> np.ndarray: + """Estimate variance components per epoch. + + This estimation is performed on an Space-Time Matrix (STM) of points. + Independent arcs are formed from the STM and unwrapped as redundancies of the estimation. + + Parameters + ---------- + stm : xr.Dataset + Input space-time matrix (STM). + key_phase : str + key for the phase data variable in the STM. + key_Btemporal : str + key for the Btemp data variable in the STM. + key_h2ph : str + key for the h2ph data variable in the STM. + key_x : str, optional + x coordinate for network formation, by default "lon" + key_y : str, optional + y coordinate for network formation, by default "lat" + max_length : float, optional + maximum length of the arcs, in degrees, by default 0.01 + include_mother_atmo : bool, optional + whether to include mother atmosphere in the stochastic model. + when False, it is assumed that the mother atmosphere is included in the functional model, + by default False + + Returns + ------- + np.ndarray + Estimated variance components per epoch, in radians squared. + This is an array of shape (Nifgs + 1,) where Nifgs is the number of interferograms. + The extra element is for the mother atmosphere. + The first element is the variance component for the mother atmosphere. + If `include_mother_atmo` is False, the first element is 0.0. + """ + # Generate a delaunay network + arcs = form_network( + stm, key_phase=key_phase, key_h2ph=key_h2ph, key_Btemporal=key_Btemporal, network_method="delaunay" + ) + + # Select independent arcs + arcs_source_target = _independent_arcs(np.stack([arcs["source"].data, arcs["target"].data], axis=1)) + arcs_source_target_set = set(map(tuple, arcs_source_target)) + pairs = np.stack([arcs["source"].data, arcs["target"].data], axis=1) + mask = np.array([tuple(x) in arcs_source_target_set for x in pairs]) + arcs = arcs.isel(space=mask) + + # Unwrap the arcs, arcs stm has standard data vars 'd_phase', 'h2ph', 'Btemp' + phase_unwrapped, _, _, _, _ = periodogram(arcs, key_dphase="d_phase", key_h2ph="h2ph", key_Btemporal="Btemp") + + # Intiate variance components + Nifgs = stm.sizes["time"] + if include_mother_atmo: + # include mother atmosphere in the stochastic model + Qy1, Qy = _q_with_mother_atmo(Nifgs) + else: + # estimate variance components in the functional model + Qy1, Qy = _q_no_mother_atmo(Nifgs) + + Qyinv = np.linalg.inv(Qy) + + # Compute Pao and QP + Btemp = stm[key_Btemporal].values + h2ph_approx = stm[key_h2ph].mean(dim="space").values # Mean h2ph of all arcs + m2ph = get_m2ph() # Convert meters to phase + B = np.stack([h2ph_approx * m2ph, Btemp * m2ph]).T # Design matrix of the functional model + Pao = np.eye(Nifgs) - B @ np.linalg.inv(B.T @ Qyinv @ B) @ B.T @ Qyinv + QP = Qyinv @ Pao + + # Compute QPQy1QP and N, optimized using einsum + # The following code is equivalent to this nested loop: + # Nsig = Qy1.shape[2] # Number of sigmas, i.e. the number of components to estimate + # Narcs_vce = phase_unwrapped.shape[0] # Number of arcs for VCE + # QPQy1QP = np.full((Nifgs, Nifgs, Nsig), np.nan) + # N = np.full((Nsig, Nsig), np.nan) + # for k in range(Nsig): + # QPQy1QP[:, :, k] = QP @ Qy1[:, :, k] @ QP + # for j in range(Nsig): + # N[k, j] = np.trace(QPQy1QP[:, :, k] @ Qy1[:, :, j]) + QPQy1QP = np.einsum("ij,jlk,lm ->imk", QP, Qy1, QP, optimize=True) + N = np.einsum("abk,baj->kj", QPQy1QP, Qy1, optimize=True) + + Ninv = np.linalg.inv(N) + + # Estimate variance components from all independent arcs + # The following code is equivalent to this nested loop: + # sig2 = np.full((Nsig, Narcs_vce), np.nan) + # l = np.full((Nsig, 1), np.nan) + # for v in range(Narcs_vce): + # y = phase_unwrapped[v, :].reshape(-1, 1) + # for k in range(Nsig): + # l[k, 0] = (y.T @ QPQy1QP[:, :, k] @ y).squeeze() + # sig2[:, v] = (Ninv @ l).flatten() + l_vec = np.einsum("ij,jmk,mi ->ki", phase_unwrapped, QPQy1QP, phase_unwrapped.T, optimize=True) # l vector + sig2_all_arcs = Ninv @ l_vec + sig2_est = np.mean(sig2_all_arcs, axis=1) + + # Apply threshold to avoid small and negative values + threshold = (np.pi * SIGMA_MINIMUM / 180) ** 2 + sig2_est[sig2_est < threshold] = threshold + + # if atmosphere is not included in the stochastic model, add a zero at the beginning + if not include_mother_atmo: + sig2_est = np.insert(sig2_est, 0, 0.0) + + return sig2_est + + +def _q_with_mother_atmo(Nifgs: int) -> tuple: + """Build Qy1 and Qy matrices with mother atmosphere.""" + sig0_mother = (np.pi * SIGMA_MOTHER_ATMO / 180) ** 2 + sig0_others = (np.pi * SIGMA_OTHERS / 180) ** 2 + + # Build "Design matrix" for variance components estimation + Qy1 = np.zeros((Nifgs, Nifgs, Nifgs + 1), dtype=np.int16) # Build Qy1 matrix, has 0 or 2 + Qy1[:, :, 0] = 2 # Mother epoch, all 2 + for v in range(0, Nifgs): + Qy1[v, v, v + 1] = 2 # Other epochs, on location set 2 + Qy = np.full((Nifgs, Nifgs), 2 * sig0_mother) # Build Qy matrix has 2 * sig0_mother in background + for v in range(Nifgs): + Qy[v, v] += 2 * sig0_others # Per epoch add 2 * sig0_others + + return Qy1, Qy + + +def _q_no_mother_atmo(Nifgs: int) -> tuple: + """Build Qy1 and Qy matrices without mother atmosphere.""" + sig0 = (np.pi * SIGMA_OVERALL / 180) ** 2 + + # Build "Design matrix" for variance components estimation + Qy1 = np.zeros((Nifgs, Nifgs, Nifgs), dtype=np.int16) # Build Qy1 matrix, has 0 or 2 + for v in range(Nifgs): + Qy1[v, v, v] = 2 # Other epochs, on location set 2 + Qy = np.diag(np.full(Nifgs, 2 * sig0)) # Build Qy matrix has 2 * sig0 in background + return Qy1, Qy diff --git a/test_network.py b/test_network.py new file mode 100644 index 00000000..a1188a54 --- /dev/null +++ b/test_network.py @@ -0,0 +1,531 @@ +"""test_network.py""" + +import numpy as np +import pytest +import xarray as xr + +from depsi.network import ( + _ensure_network_min_connections, + _ensure_single_network, + _independent_arcs, + _network_relation_matrix, + _remove_network_points_min_connections, + form_network, + spatial_integration, +) + + +@pytest.fixture +def stm_random(): + """Fixture to create a random STM dataset.""" + rng = np.random.default_rng(42) + Npoints = 12 # Number of points + Ntimes = 31 # Number of epochs + # Coordinates and time + lat = rng.uniform(51.14, 51.15, Npoints) + lon = rng.uniform(6.9, 7.0, Npoints) + time = np.arange(Ntimes) + # Data + complex = rng.uniform(-1, 1, (Npoints, Ntimes)) + 1j * rng.uniform(-1, 1, (Npoints, Ntimes)) + phase = np.angle(complex) + h2ph = rng.uniform(1e3, 1e4, (Npoints, Ntimes)) + # Create the xarray Dataset + stm = xr.Dataset( + data_vars={ + "phase": (("space", "time"), phase), + "h2ph": (("space", "time"), h2ph), + "complex": (("space", "time"), complex), + }, + coords={ + "space": ("space", np.arange(Npoints)), + "time": ("time", time), + "lat": ("space", lat), + "lon": ("space", lon), + }, + ) + + return stm + + +@pytest.fixture +def stm_random_grid(): + """STM points forming a 10x10 grid""" + N_time = 50 + grid_shape = 10 + x_grid, y_grid = np.meshgrid(np.arange(0, 100, grid_shape), np.arange(0, 100, grid_shape)) + N_points = x_grid.flatten().shape[0] + + stm = xr.Dataset( + coords={ + "space": (["space"], np.arange(N_points)), + "time": (["time"], np.arange(N_time)), + "x": (["space"], x_grid.flatten()), + "y": (["space"], y_grid.flatten()), + }, + data_vars={ + "phase": (["space", "time"], np.random.uniform(0, 1, (N_points, N_time))), + "h2ph": (["space", "time"], np.random.uniform(0, 1, (N_points, N_time))), + "ambiguity": (["space", "time"], np.random.choice([-1, 0, 1], (N_points, N_time), p=[0.02, 0.96, 0.02])), + }, + ) + + return stm + + +@pytest.fixture +def arcs_random(stm_random): + """Fixture of fully connected arcs from stm_random.""" + # Fully connected arcs + # Defaul method is redundant + # No max_length, so all points are connected + arcs = form_network(stm_random, key_phase="phase", key_h2ph="h2ph", key_Btemporal="time") + + # Most arcs have quality 0.9 + # Except the last two have quality 0.0 + # And the first five have quality 0.99 + temp_coh = np.zeros((arcs.sizes["space"],)) + temp_coh[:-2] = 0.9 + temp_coh[:5] = 0.99 + arcs["temp_coh"] = (("space"), temp_coh) + + return arcs + + +def _build_network_components(component_sizes: list[int]) -> tuple[xr.Dataset, xr.Dataset]: + """Build coordinate only point/arcs STMs from connected-component sizes.""" + n_points = int(sum(component_sizes)) + stm_pnts = xr.Dataset(coords={"space": ("space", np.arange(n_points))}) + + source = [] + target = [] + offset = 0 + for size in component_sizes: + # Build each component as a simple chain graph. + for idx in range(offset, offset + size - 1): + source.append(idx) + target.append(idx + 1) + offset += size + + stm_arcs = xr.Dataset( + coords={ + "source": ("space", np.array(source, dtype=int)), + "target": ("space", np.array(target, dtype=int)), + } + ) + + return stm_arcs, stm_pnts + + +class TestNetworkFormation: + def test_form_network_simulated_grid(self, stm_random_grid): + arcs = form_network( + stm_random_grid, + key_phase="phase", + key_h2ph="h2ph", + key_Btemporal="time", + key_xcrds="x", + key_ycrds="y", + max_length=25, + min_links=8, + num_partitions=8, + ) + + source = arcs["source"].values + target = arcs["target"].values + + assert arcs.sizes["space"] == 428 # nr arcs should be 428 with a 10x10 grid setting + assert np.all(np.diff(source) >= 0) # check if source is mono-increasing + assert np.all(source < target) # check if all sources < targets + assert ( + np.unique(np.column_stack((source, target)), axis=0).shape[0] == source.shape[0] + ) # check if all (source, target) pairs are unique + assert np.unique(arcs["uid"].values).shape[0] == arcs.sizes["space"] + + def test_stm_to_arcs_subtract(self, stm_random): + # Generate arcs of a Delaunay network with subtracted phase differences. + stm_arcs = form_network( + stm_random, + key_phase="phase", + key_h2ph="h2ph", + key_Btemporal="time", + network_method="delaunay", + max_length=0.05, + dphase_method="subtract", + ) + + assert all( + [all([-2 * np.pi <= phase <= 2 * np.pi for phase in phases]) for phases in stm_arcs["d_phase"].values] + ) + + def test_stm_to_arcs_conjmult(self, stm_random): + # Generate arcs of a Delaunay network with conjugate multiplication phase differences. + stm_arcs = form_network( + stm_random, + key_phase="phase", + key_h2ph="h2ph", + key_Btemporal="time", + network_method="delaunay", + max_length=0.05, + dphase_method="conjmult", + ) + + assert all([all([-np.pi <= phase <= np.pi for phase in phases]) for phases in stm_arcs["d_phase"].values]) + + def test_stm_to_arcs_fail(self, stm_random): + # Test incorrect method fail. + with pytest.raises(NotImplementedError): + form_network( + stm_random, + key_phase="phase", + key_h2ph="h2ph", + key_Btemporal="time", + network_method="unknown", + dphase_method="subtract", + ) + with pytest.raises(NotImplementedError): + form_network( + stm_random, + key_phase="phase", + key_h2ph="h2ph", + key_Btemporal="time", + network_method="delaunay", + dphase_method="unknown", + ) + + +class TestNetworkEnsure: + @pytest.mark.parametrize("thres, min_n_connections", [(0.5, 2), (0.5, 1)]) + def test_select_arcs_discard_two(self, arcs_random, stm_random, thres, min_n_connections): + """Should only discard two arcs, with temp_coh < 0.5.""" + # Select arcs based on temp_coh threshold. + mask = np.abs(arcs_random["temp_coh"]) > thres # mask as DataArray + arcs_selected = arcs_random.where(mask, drop=True) + arcs_results, _ = _ensure_network_min_connections(arcs_selected, stm_random, min_connections=min_n_connections) + + # Threshold is 0.5, so only the last two arcs are discarded + # The min_n_connections should not affect the selection + assert arcs_results.sizes["space"] == arcs_random.sizes["space"] - 2 + + def test__remove_network_points_min_connections_nconnection_zero(self, stm_random, arcs_random): + """Raise error when min_connections <1.""" + with pytest.raises(ValueError): + _remove_network_points_min_connections(stm_random, arcs_random, min_connections=-1) + with pytest.raises(ValueError): + _remove_network_points_min_connections(stm_random, arcs_random, min_connections=0) + + def test__remove_network_points_min_connections_keep_all_pnts(self, stm_random, arcs_random): + """No STM points removed since no arc is discarded.""" + stm_updated, arcs_updated = _remove_network_points_min_connections(stm_random, arcs_random, min_connections=1) + + assert stm_updated.sizes["space"] == stm_random.sizes["space"] + assert arcs_updated.sizes["space"] == arcs_random.sizes["space"] + + def test__remove_network_points_min_connections_discard_one(self, stm_random, arcs_random): + """Remove one STM point.""" + # remove arcs with source or target == 1 + arcs = arcs_random.copy(deep=True) + arcs = arcs.where((arcs["source"] != 1) & (arcs["target"] != 1), drop=True) + + stm_updated, arcs_updated = _remove_network_points_min_connections(stm_random, arcs, min_connections=1) + + # Should remove the point with index 1 + assert stm_updated.sizes["space"] == stm_random.sizes["space"] - 1 + + @pytest.mark.parametrize("component_sizes", [[12]]) + def test_ensure_single_network_no_separated_part(self, component_sizes): + """Keep the network unchanged when there is only one connected component.""" + stm_arcs, stm_pnts = _build_network_components(component_sizes) + + stm_arcs_out, stm_pnts_out = _ensure_single_network(stm_arcs, stm_pnts, largest_component_ratio=0.8) + + assert stm_pnts_out.sizes["space"] == stm_pnts.sizes["space"] + assert stm_arcs_out.sizes["space"] == stm_arcs.sizes["space"] + assert np.array_equal(stm_arcs_out["source"].values, stm_arcs["source"].values) + assert np.array_equal(stm_arcs_out["target"].values, stm_arcs["target"].values) + + @pytest.mark.parametrize( + ["component_sizes", "largest_component_ratio"], + [([15, 1, 1, 1], 0.8), ([8, 6], 0.5), ([9, 1, 1], 0.8), ([5, 3, 2], 0.4)], + ) + def test_ensure_single_network_keep_largest_significant(self, component_sizes, largest_component_ratio): + """Keep only the largest component when it is significant enough.""" + stm_arcs, stm_pnts = _build_network_components(component_sizes) + largest_size = max(component_sizes) + + stm_arcs_out, stm_pnts_out = _ensure_single_network(stm_arcs, stm_pnts, largest_component_ratio) + + assert stm_pnts_out.sizes["space"] == largest_size + assert stm_arcs_out.sizes["space"] == largest_size - 1 + assert np.all(stm_arcs_out["source"].values >= 0) + assert np.all(stm_arcs_out["target"].values >= 0) + assert np.all(stm_arcs_out["source"].values < largest_size) + assert np.all(stm_arcs_out["target"].values < largest_size) + + @pytest.mark.parametrize( + "component_sizes", + [ + [6, 4], + [8, 7, 1], + [10, 9, 1, 1], + ], + ) + def test_ensure_single_network_raise_when_largest_not_significant(self, component_sizes): + """Raise when the largest component is not clearly dominant.""" + largest_component_ratio = 0.8 + stm_arcs, stm_pnts = _build_network_components(component_sizes) + + with pytest.raises(RuntimeError): + _ensure_single_network(stm_arcs, stm_pnts, largest_component_ratio) + + +class TestNetworkUnwrap: + @pytest.mark.parametrize( + ["id_ref", "idx_err_space", "idx_err_time", "error_values", "skip_network_adaptation"], + [ + (3, [], [], [], False), # No error + (3, [2, 11], [7, 13], [-1, 1], False), # Two errors in arc ambiguities + (3, [2, 11], [7, 13], [-1, 1], True), # Two errors, skip network adjustment, should still be corrected + (9, [0, 4, 8], [5, 10, 15], [1, -100, 1], False), # Three errors, one large, but should be corrected + ], + ) + def test_spatial_integration(self, id_ref, idx_err_space, idx_err_time, error_values, skip_network_adaptation): + """Test spatial unwrapping based on arc ambiguities. + + Build points with true value of ambiguities. + Construct arcs with arc ambiguities derived from true ambiguities. + Add tiny errors to arc ambiguities at certain space/time indices. + + Then perform spatial unwrapping with a specified reference point. + + The spatial unwrapping should be able to solve the point ambiguities correctly. + The solved ambiguities should w.r.t. the reference point. + """ + # Set up test parameters + rng = np.random.default_rng(42) + Npoints = 17 # Number of points + Ntimes = 29 # Number of epochs + time = np.arange(Ntimes) + complex = rng.uniform(-1, 1, (Npoints, Ntimes)) + 1j * rng.uniform(-1, 1, (Npoints, Ntimes)) + phase = np.angle(complex) + h2ph = rng.uniform(1e3, 1e4, (Npoints, Ntimes)) + + # Create the points + stm_pnts = xr.Dataset( + data_vars={ + "phase": (("space", "time"), phase), + "h2ph": (("space", "time"), h2ph), + "complex": (("space", "time"), complex), + "ambiguities_true": ( + ("space", "time"), + np.round(rng.normal(0, 0.5, (Npoints, Ntimes))).astype(int).clip(-1, 1), + ), + }, + coords={ + "space": ("space", np.arange(Npoints)), + "time": ("time", time), + "azimuth": ("space", np.round(rng.normal(0, 10, (Npoints))).astype(int)), + "range": ("space", np.round(rng.normal(0, 10, (Npoints))).astype(int)), + }, + attrs={"wavelength": 0.056}, # Wavelength in meters + ) + + # Construct arcs based on true ambiguities + # All arcs by default have 0.99 temp_coh + stm_arcs = form_network( + stm_pnts, + key_xcrds="azimuth", + key_ycrds="range", + key_phase="phase", + key_h2ph="h2ph", + key_Btemporal="time", + network_method="redundant", + max_length=30, + ) + temp_coh = np.zeros((stm_arcs.sizes["space"],)) + 0.99 + stm_arcs["temp_coh"] = (("space"), temp_coh) + + # Compute arc ambiguities from true point ambiguities + ambigs = ( + stm_pnts["ambiguities_true"].values[stm_arcs["target"].values, :] + - stm_pnts["ambiguities_true"].values[stm_arcs["source"].values, :] + ) + # Introduce some errors in ambiguities + ambigs_errors = np.zeros_like(ambigs) + for idx_s, idx_t, err in zip(idx_err_space, idx_err_time, error_values, strict=False): + ambigs_errors[idx_s, idx_t] += err + stm_arcs["ambiguities"] = (("space", "time"), ambigs + ambigs_errors) + + stm_arcs_output, stm_pnts_output = spatial_integration( + stm_pnts, stm_arcs, idx_refpnt=id_ref, key_sdphase="phase", skip_network_adaptation=skip_network_adaptation + ) + + # Verify output dimensions, no points should be rejected + assert stm_pnts_output.sizes["space"] == stm_pnts.sizes["space"] + + # Check that the solved ambiguities match the true ambiguities w.r.t. the reference point + assert np.allclose( + stm_pnts_output["ambiguities"].values + - stm_pnts["ambiguities_true"].values + + np.tile(stm_pnts["ambiguities_true"].isel(space=id_ref).values, (stm_pnts.sizes["space"], 1)), + 0, + ) + + # Check that the unwrapped phase is correct + ref_phase = stm_pnts["phase"].isel(space=id_ref).values + relative_ambiguities = stm_pnts["ambiguities_true"].values - np.tile( + stm_pnts["ambiguities_true"].isel(space=id_ref).values, (stm_pnts.sizes["space"], 1) + ) # true ambiguities relative to reference point + unwrapped_phase_expected = ( + stm_pnts["phase"].values + + relative_ambiguities * 2 * np.pi + - np.tile(ref_phase, (stm_pnts.sizes["space"], 1)) + ) + assert np.allclose(stm_pnts_output["unwrapped_phase"].values, unwrapped_phase_expected) + + # Check that the reference point index remains the same + assert stm_pnts_output.attrs["idx_refpnt"] == id_ref + + @pytest.mark.parametrize("idx_ref", [0, 5, 10, 16]) + def test_spatial_integration_ref_pnt_removed(self, idx_ref): + """Raise error when reference point is removed""" + # Set up test parameters + rng = np.random.default_rng(42) + Npoints = 17 # Number of points + Ntimes = 29 # Number of epochs + time = np.arange(Ntimes) + complex = rng.uniform(-1, 1, (Npoints, Ntimes)) + 1j * rng.uniform(-1, 1, (Npoints, Ntimes)) + phase = np.angle(complex) + h2ph = rng.uniform(1e3, 1e4, (Npoints, Ntimes)) + + # Create the points + stm_pnts = xr.Dataset( + data_vars={ + "phase": (("space", "time"), phase), + "h2ph": (("space", "time"), h2ph), + "complex": (("space", "time"), complex), + "ambiguities_true": ( + ("space", "time"), + np.round(rng.normal(0, 0.5, (Npoints, Ntimes))).astype(int).clip(-1, 1), + ), + }, + coords={ + "space": ("space", np.arange(Npoints)), + "time": ("time", time), + "azimuth": ("space", np.round(rng.normal(0, 10, (Npoints))).astype(int)), + "range": ("space", np.round(rng.normal(0, 10, (Npoints))).astype(int)), + }, + attrs={"wavelength": 0.056}, + ) + + # Construct arcs based on true ambiguities + # All arcs by default have 0.99 temp_coh + stm_arcs = form_network( + stm_pnts, + key_xcrds="azimuth", + key_ycrds="range", + key_phase="phase", + key_h2ph="h2ph", + key_Btemporal="time", + network_method="redundant", + max_length=30, + ) + temp_coh = np.zeros((stm_arcs.sizes["space"],)) + 0.99 + stm_arcs["temp_coh"] = (("space"), temp_coh) + + # Set temp_coh of all arcs connected to reference point to 0.01 + mask_ref_arcs = (stm_arcs["source"] == idx_ref) | (stm_arcs["target"] == idx_ref) + stm_arcs["temp_coh"] = stm_arcs["temp_coh"].where(~mask_ref_arcs, other=0.01) + + # Compute arc ambiguities from true point ambiguities + ambigs = ( + stm_pnts["ambiguities_true"].values[stm_arcs["target"].values, :] + - stm_pnts["ambiguities_true"].values[stm_arcs["source"].values, :] + ) + stm_arcs["ambiguities"] = (("space", "time"), ambigs) + + with pytest.raises(ValueError): + spatial_integration(stm_pnts, stm_arcs, idx_refpnt=idx_ref, key_sdphase="phase") + + @pytest.mark.parametrize( + ["idx_source", "idx_target", "n_points", "idx_refpnt"], + [ + (np.array([0, 1, 2]), np.array([1, 2, 3]), 4, 0), # 4 points, 3 arcs + (np.array([0, 1, 2]), np.array([1, 2, 3]), 7, 0), # 7 points, 3 arcs + (np.array([1, 1, 2, 2]), np.array([0, 2, 1, 3]), 4, 2), # 4 points, 4 arcs, unsorted + (np.array([0, 0, 0, 1, 1, 2, 2]), np.array([1, 2, 3, 3, 4, 3, 4]), 5, 3), # 5 points, 6 arcs + ], + ) + def test_init_network_relation_matrix( + self, + idx_source, + idx_target, + n_points, + idx_refpnt, + ): + A = _network_relation_matrix(idx_source, idx_target, n_points, idx_refpnt) + + # Create expected matrix in a for loop + A_exp = np.zeros((idx_source.shape[0], n_points), dtype=int) + for i, (src, tgt) in enumerate(zip(idx_source, idx_target, strict=False)): + A_exp[i, src] = -1 + A_exp[i, tgt] = 1 + A_exp = np.delete(A_exp, idx_refpnt, axis=1) # Remove reference point column + + assert A.shape == A_exp.shape + assert np.all(A == A_exp) + + @pytest.mark.parametrize( + ["idx_source", "idx_target", "n_points", "idx_refpnt"], + [ + (np.array([0, 1, 2]), np.array([1, 2, 3]), 4, 0), # 4 points, 3 arcs + (np.array([0, 1, 2]), np.array([1, 2, 3]), 7, 0), # 7 points, 3 arcs + (np.array([1, 1, 2, 2]), np.array([0, 2, 1, 3]), 4, 2), # 4 points, 4 arcs, unsorted + (np.array([0, 0, 0, 1, 1, 2, 2]), np.array([1, 2, 3, 3, 4, 3, 4]), 5, 3), # 5 points, 6 arcs + ], + ) + def test_init_network_relation_matrix_sparse( + self, + idx_source, + idx_target, + n_points, + idx_refpnt, + ): + A = _network_relation_matrix(idx_source, idx_target, n_points, idx_refpnt, sparse_mode=True) + + # Create expected matrix in a for loop + A_exp = np.zeros((idx_source.shape[0], n_points), dtype=int) + for i, (src, tgt) in enumerate(zip(idx_source, idx_target, strict=False)): + A_exp[i, src] = -1 + A_exp[i, tgt] = 1 + A_exp = np.delete(A_exp, idx_refpnt, axis=1) # Remove reference point column + + assert A.shape == A_exp.shape + assert np.all(A.todense() == A_exp) + + +class TestArcsUtils: + @pytest.mark.timeout(10) # Each should finish in 10 seconds + @pytest.mark.parametrize( + "npoints, narcs", + [ + (103, 1000), + (1923, 10000), + (12, 30), + ], + ) + def test_independent_arcs(self, npoints, narcs): + # Simulate random arcs + rng = np.random.default_rng(42) + arcs = rng.integers(0, npoints, size=(narcs, 2)) + # remove arcs which has the same start and end point + arcs = arcs[arcs[:, 0] != arcs[:, 1]] + # Remove duplicate arcs + arcs = np.unique(np.sort(arcs, axis=1), axis=0) + + # Test that the arcs are independent. + independent_arcs = _independent_arcs(arcs) + + # A point index should only appear once + # either as a start or end point of an arc. + all_idx = independent_arcs.flatten() + assert all_idx.shape == np.unique(all_idx).shape diff --git a/test_stochastic.py b/test_stochastic.py new file mode 100644 index 00000000..036b6b9d --- /dev/null +++ b/test_stochastic.py @@ -0,0 +1,81 @@ +import numpy as np +import pytest +import xarray as xr + +from depsi.stochastic import _q_no_mother_atmo, _q_with_mother_atmo, vce_temporal + + +def simulated_stm(n_ifg, n_points): + """function to simulate arcs stm for testing.""" + rng = np.random.default_rng(31) + + # Simulate coordinates + lat = np.linspace(51.14, 51.15, n_points) + lon = rng.uniform(6.9, 7.0, n_points) + + # Simulate Btemp + TIME_STEP = 15 # Time step in days + years = np.linspace(0, (n_ifg - 1) * TIME_STEP / 365.25, n_ifg) # years from 0 to n_ifg-1 + + # Simulated phase values, linear+ noise + phase = np.tile(np.linspace(-np.pi, np.pi, n_ifg), (n_points, 1)) + rng.normal(0, 0.1, (n_points, n_ifg)) + + # Simulated h2ph values + h2ph = rng.random((n_points, n_ifg)) * 1e-3 # fixed h2ph values for each arc + + arcs = xr.Dataset( + data_vars={ + "lon": (("space",), lon), + "lat": (("space",), lat), + "phase": (("space", "time"), phase), + "h2ph": (("space", "time"), h2ph), + "Btemporal": (("time",), years), + } + ) + + arcs.attrs["wavelength"] = 0.056 # example wavelength in meters + + return arcs + + +@pytest.mark.parametrize( + ["n_ifg", "n_points", "include_mother_atmo"], + [ + (12, 41, False), + (19, 107, False), + (42, 127, True), + ], +) +def test_vce_temporal(n_ifg, n_points, include_mother_atmo): + arcs = simulated_stm(n_ifg, n_points) + sigma2 = vce_temporal( + arcs, key_phase="phase", key_Btemporal="Btemporal", key_h2ph="h2ph", include_mother_atmo=include_mother_atmo + ) + + assert sigma2.shape[0] == n_ifg + 1 + if include_mother_atmo: + assert sigma2[0] != 0 + else: + assert sigma2[0] == 0 + + +@pytest.mark.parametrize("Nifgs", [4, 11, 23]) +def test_q_no_mother_atmo(Nifgs): + """Test the _q_with_mother_atmo function.""" + Qy1, Qy = _q_no_mother_atmo(Nifgs) + + assert Qy1.shape == (Nifgs, Nifgs, Nifgs) + assert Qy.shape == (Nifgs, Nifgs) + assert np.all(np.isin(np.unique(Qy1.flatten()), [0, 2])) + assert np.all(Qy == np.diag(np.diag(Qy))) # check Qy is a diagonal matrix + + +@pytest.mark.parametrize("Nifgs", [4, 11, 23]) +def test_q_with_mother_atmo(Nifgs): + """Test the _q_with_mother_atmo function.""" + Qy1, Qy = _q_with_mother_atmo(Nifgs) + + assert Qy1.shape == (Nifgs, Nifgs, Nifgs + 1) + assert Qy.shape == (Nifgs, Nifgs) + assert np.all(np.isin(np.unique(Qy1.flatten()), [0, 2])) + assert np.all(np.isin(np.unique(Qy1[:, :, 0].flatten()), [2])) # check first slice is all 2 diff --git a/utils.py b/utils.py new file mode 100644 index 00000000..912f0028 --- /dev/null +++ b/utils.py @@ -0,0 +1,841 @@ +import math +import os +from typing import Literal + +import asf_search as asf +import dask.array as da +import pandas as pd +import pyproj + +try: + from datetime import UTC, datetime, timedelta +except ImportError: # UTC can only be imported from Python 3.11 onwards + import warnings + from datetime import datetime, timedelta, timezone + + UTC = timezone.utc + warnings.warn( + """ + DePSI uses datetime.UTC, which is only supported from Python 3.11 onwards. + For older Python versions, datetime.timezone.utc is used. + This might be deprecated in newer DePSI versions. + """, + DeprecationWarning, + stacklevel=1, # necessary to start the call stack here. + ) + +import logging + +import geopandas +import numpy as np +import pytz +import xarray as xr + +from depsi.constants import EARTH_RADIUS, WAVELENGTH_S1 + +logger = logging.getLogger(__name__) + + +def wrap_phase(phs_abs): + """Wrap the absolute phase to the range [-pi, pi). + + Parameters + ---------- + phs_abs : array_like or float + The absolute phase. + + Returns + ------- + ndarray or float + The wrapped phase in the range [-pi, pi). + """ + phs_wrapped = np.remainder(phs_abs + np.pi, 2 * np.pi) - np.pi + + return phs_wrapped + + +def _orbit_fit(orbit, verbose=0, der=True): + """Return a orbit_fit dict. + + Modified from the "orbitFit" function: + https://github.com/Pbaz98/Caroline-Radar-Coding-Toolbox/blob/main/gecoris/geoUtils.py#L325 + + Satellite state vector interpolation using Chebyshev polynomials of + 7th order (according to DLR recommendations). Function returns Chebyshev + polynomial coefficients. Use these to evaluate orbit state at given time + via function 'orbitVal'. + + input: snappy 'orbit' object (as read by 'read_metadata' function) + + CHANGE LOG + - 30/6/2023: Modified to adapt the input to a np.array Nx4 (N number of timesamples) + - 22/09/23: add the flag for derivative or not + """ + # parse masterorb: + t = orbit[:, 0] + x = orbit[:, 1] + y = orbit[:, 2] + z = orbit[:, 3] + + # interpolate orbits using Chebyshev polynomials of 7th order: + t0 = (min(t) + max(t)) / 2 + px = t - t0 # time argument px (centered around mid interval) + cx = np.polynomial.chebyshev.chebfit(px, x, 7) # position + cy = np.polynomial.chebyshev.chebfit(px, y, 7) + cz = np.polynomial.chebyshev.chebfit(px, z, 7) + + if der: + cvx = np.polynomial.chebyshev.chebder(cx) # velocity + cvy = np.polynomial.chebyshev.chebder(cy) + cvz = np.polynomial.chebyshev.chebder(cz) + else: + x_vel = orbit[:, 4] + y_vel = orbit[:, 5] + z_vel = orbit[:, 6] + + cvx = np.polynomial.chebyshev.chebfit(px, x_vel, 7) # velocity + cvy = np.polynomial.chebyshev.chebfit(px, y_vel, 7) + cvz = np.polynomial.chebyshev.chebfit(px, z_vel, 7) + + cax = np.polynomial.chebyshev.chebder(cvx) # acceleration + cay = np.polynomial.chebyshev.chebder(cvy) + caz = np.polynomial.chebyshev.chebder(cvz) + + if verbose: + # position fit residuals: + x_res = np.polynomial.chebyshev.chebval(px, cx) - x + y_res = np.polynomial.chebyshev.chebval(px, cy) - y + z_res = np.polynomial.chebyshev.chebval(px, cz) - z + x_std = np.std(x_res) + y_std = np.std(y_res) + z_std = np.std(z_res) + print(f"Orbit fit position residuals: X {x_std:.4f} m, Y {y_std:.4f} m, Z {z_std:.4f} m. ") + # velocity residuals: + vx_res = np.polynomial.chebyshev.chebval(px, np.polynomial.chebyshev.chebder(cx)) - x_vel + vy_res = np.polynomial.chebyshev.chebval(px, np.polynomial.chebyshev.chebder(cy)) - y_vel + vz_res = np.polynomial.chebyshev.chebval(px, np.polynomial.chebyshev.chebder(cz)) - z_vel + vx_std = np.std(vx_res) + vy_std = np.std(vy_res) + vz_std = np.std(vz_res) + print(f"Orbit fit velocity residuals: vX {vx_std:.4f} m/s, vY {vy_std:.4f} m/s, vZ {vz_std:.4f} m/s. ") + + orbit_fit = dict() + orbit_fit["t0"] = t0 + orbit_fit["cx"] = cx + orbit_fit["cy"] = cy + orbit_fit["cz"] = cz + orbit_fit["cvx"] = cvx + orbit_fit["cvy"] = cvy + orbit_fit["cvz"] = cvz + orbit_fit["cax"] = cax + orbit_fit["cay"] = cay + orbit_fit["caz"] = caz + + return orbit_fit + + +def get_distance( + source: list | tuple | np.ndarray, + target: list | tuple | np.ndarray, + mode: Literal["euclidean", "geographic"] = "euclidean", +): + """Calculate the distance between two points. + + The Euclidean mode calculates distance on a 2D XY-plane. The Geographic mode approximates the Earth as a sphere + with radius 6378136 meter (the polar radius). On long north-south oriented arcs, distance errors of up to 0.3% are + possible. + + Parameters + ---------- + source: list | tuple | np.ndarray + The source point, formatted as (x, y) / (lon, lat) + target: list | tuple | np.ndarray + The target point, formatted as (x, y) / (lon, lat) + mode: Literal["euclidean", "geographic"], default "euclidean" + Whether the source and target points are given in (x, y) (units meters) or (lon, lat) (units degrees) + + Returns + ------- + The distance between the two points in meters. + """ + if mode == "euclidean": + return math.dist(source, target) + elif mode == "geographic": + # this is the Haversine formula + lat1 = source[1] + lat2 = target[1] + dphi = np.radians(lat1 - lat2) + dlambda = np.radians(source[0] - target[0]) + dist = ( + 2 + * EARTH_RADIUS + * np.arcsin( + np.sqrt( + (1 - np.cos(dphi) + np.cos(np.radians(lat1)) * np.cos(np.radians(lat2)) * (1 - np.cos(dlambda))) / 2 + ) + ) + ) + return dist + raise ValueError(f"Unknown mode {mode}, only know euclidean and geographic!") + + +def npdatetime64_to_datetime(date: np.datetime64, tz_aware: bool = True) -> datetime: + """Convert a numpy datetime64 object to a python datetime object. + + Parses the np.datetime64 object into a datetime object. + + Parameters + ---------- + date : np.datetime64 + the date to be converted + tz_aware: bool, default True + whether the returned datetime object should be timezone-aware or not + + Returns + ------- + datetime.datetime + The same date converted to a datetime object + """ + timestamp = (date - np.datetime64("1970-01-01T00:00:00")) / np.timedelta64(1, "s") + dt_obj = datetime.fromtimestamp(timestamp, UTC) + if not tz_aware: + microseconds = timedelta(microseconds=dt_obj.microsecond) + dt_obj = datetime.strptime(dt_obj.strftime("%Y%m%d:%H%M%S"), "%Y%m%d:%H%M%S") + microseconds + return dt_obj + + +def _get_aoi_shapefile_bounding_box(aoi_filename: str) -> tuple: + """Read a .shp shapefile and return the bounding box. + + The shapefile is read and the area of interest is retrieved. The bounding box is then computed, and a tuple of the + coordinates of the bounding box is returned. + + Parameters + ---------- + aoi_filename: str | None + full path to the AoI shapefile, expects .shp format. + + Returns + ------- + tuple + tuple of two lists, the first containing the longitude extent, the second the latitude extent of the + bounding box. + + Raises + ------ + AssertionError + Raised when: + - the aoi_filename does not exist + - the aoi_filename does not end in .shp + + ValueError + Raised when: + - the provided shapefile contains zero polygons, or more than one polygon + - the provided shapefile contains an invalid polygon + """ + assert os.path.exists(aoi_filename), f"The file {aoi_filename} does not exist!" + assert aoi_filename.split(".")[-1] == "shp", f"The provided file {aoi_filename} is not of .shp type!" + + # open the file, and iterate through the geometry + shape = geopandas.read_file(aoi_filename) + # calculate the coordinates of the bounding box of the provided AoI + bounding_box = shape.total_bounds + # format as longitude extent (in x), latitude extent (in y) + bounding_box_formatted = ([bounding_box[0], bounding_box[2]], [bounding_box[1], bounding_box[3]]) + + return bounding_box_formatted + + +def crop_slc_spacetime( + slcs: xr.Dataset, + aoi_filename: str | None = None, + start_date: datetime | str | None = None, + end_date: datetime | str | int | None = None, +) -> xr.Dataset: + """Crop an SLC stack in both space and time. + + To crop in space an AoI shapefile is processed, and the SLC stack is cropped to the bounding box. + To crop in time, a start date is provided, and two options for the end date are available: + - datetime | str - this directly provides the end date + - int - this provides a number of SLCs intended to be in the crop. The end date is set automatically. + + If the aoi_filename is not provided and left to None, only a crop in time is performed. + If the start_date and end_date are not provided and left to None, only a crop in space is performed. + + Parameters + ---------- + slcs : xr.Dataset + the SLC stack to be cropped. Requires at least the following coordinates or variables: + In case of a crop in time: + - time -> the dates of the images + In case of a crop in space: + - lat -> the latitude of the pixels + - lon -> the longitude of the pixels + aoi_filename: str | None + full path to the AoI shapefile, expects .shp format. Set to None if no crop in space is requested. + start_date : datetime | str | None + the start date of the crop, in one of three formats: + - datetime object + - str object, formatted as YYYYMMDD + - None, no cropping in time requested + end_date : datetime | str | int | None + the end date of the crop, in one of four formats: + - datetime object + - str object, formatted as YYYYMMDD + - int object, which is interpreted as the number of images intended in the crop (including the start date). If + more images are requested than exist since the start date, all images from start_date until the last image + are provided. + - None, no cropping in time requested + + Returns + ------- + xr.Dataset + The cropped dataset + + Raises + ------ + AssertionError + Raised when: + - a start_date or end_date is provided in string format, but not in YYYYMMDD format + - a start_date is provided, but the end_date is set to None + - an end_date is provided, but the start_date is set to None + - the aoi_filename does not exist + - the aoi_filename does not end in .shp + - one of the required coordinates or variables is not available in slcs + + ValueError + Raised when: + - start_date is not of type datetime | str | None + - end_date is not of type datetime | str | int | None + """ + # Check the input + + if aoi_filename is not None: + assert os.path.exists(aoi_filename), f"The file {aoi_filename} does not exist!" + assert aoi_filename.split(".")[-1] == "shp", f"The provided file {aoi_filename} is not of .shp type!" + for axis in ["lat", "lon"]: + assert axis in slcs.keys(), f"Expected axis {axis} in SLCs but it is not present!" + + # convert the input to a timezone-aware datetime object + if isinstance(start_date, str): + assert len(start_date) == 8, f"Unknown start_date format {start_date}, expected YYYYMMDD!" + format_start_date = datetime( + eval(start_date[:4]), eval(start_date[4:6].lstrip("0")), eval(start_date[6:].lstrip("0")), tzinfo=pytz.UTC + ) + elif isinstance(start_date, datetime): + format_start_date = datetime(start_date.year, start_date.month, start_date.day, tzinfo=pytz.UTC) + elif start_date is None: + assert end_date is None, f"Start date is None while end date is {end_date} (not None!)" + format_start_date = None + else: + raise ValueError(f'Expected start_date of type "str" | "datetime" | None, got {type(start_date)}!') + + if isinstance(end_date, str): + assert len(end_date) == 8, f"Unknown end_date format {end_date}, expected YYYYMMDD!" + format_end_date = datetime( + eval(end_date[:4]), eval(end_date[4:6].lstrip("0")), eval(end_date[6:].lstrip("0")), tzinfo=pytz.UTC + ) + elif isinstance(end_date, datetime): + format_end_date = datetime(end_date.year, end_date.month, end_date.day, tzinfo=pytz.UTC) + elif isinstance(end_date, int): + fmt_dates = [npdatetime64_to_datetime(date) for date in slcs["time"].values] + valid_dates = [date for date in fmt_dates if date >= format_start_date] + end_idx = fmt_dates.index(valid_dates[0]) + end_date - 1 + end_idx = min(end_idx, len(fmt_dates) - 1) + format_end_date = fmt_dates[end_idx] + elif end_date is None: + assert start_date is None, f"Start date is None while end date is {end_date} (not None!)" + format_end_date = None + else: + raise ValueError(f'Expected end_date of type "str" | "datetime" | "int" | None, got {type(end_date)}!') + + # TIME CROP + if format_start_date is not None and format_end_date is not None: + # first the last assertion + assert "time" in slcs.keys(), "Expected axis 'time' in SLCs but it is not present!" + + fmt_dates_raw = [npdatetime64_to_datetime(date) for date in slcs["time"].values] + fmt_dates = np.array([datetime(date.year, date.month, date.day, tzinfo=pytz.UTC) for date in fmt_dates_raw]) + time_mask = (format_start_date <= fmt_dates) & (fmt_dates <= format_end_date) + slcs = slcs.sel(time=slcs["time"].values[time_mask]) + + # SPACE CROP + if aoi_filename is not None: + bounding_box = _get_aoi_shapefile_bounding_box(aoi_filename) + space_mask = ( + (slcs["lat"] >= min(bounding_box[1])) + & (slcs["lat"] <= max(bounding_box[1])) + & (slcs["lon"] >= min(bounding_box[0])) + & (slcs["lon"] <= max(bounding_box[0])) + ) + + comp_space_mask = space_mask.compute() + az_sum = comp_space_mask.sum(dim="azimuth") + rg_sum = comp_space_mask.sum(dim="range") + + # first and last non zero + min_range, max_range = ( + az_sum.where(az_sum > 0, drop=True)["range"].min().values, + az_sum.where(az_sum > 0, drop=True)["range"].max().values, + ) + min_azimuth, max_azimuth = ( + rg_sum.where(rg_sum > 0, drop=True)["azimuth"].min().values, + rg_sum.where(rg_sum > 0, drop=True)["azimuth"].max().values, + ) + # data at original locations not nan + slcs = slcs.sel(azimuth=range(min_azimuth, max_azimuth), range=range(min_range, max_range)) + + return slcs + + +def project_stm_coordinates(stm: xr.Dataset, projection: str = "RD") -> xr.Dataset: + """Project the latitude and longitude of a space-time matrix to another reference frame. + + The latitude and longitude layers are transformed into the desired projection, default Rijksdriehoek or RD. + + Parameters + ---------- + stm: xr.Dataset + Space-time matrix with the layers `lat` (latitude) and `lon` (longitude in WGS84 (EPSG:4326), and coordinate + `space` + projection: str, optional + Projection to which the latitude and longitude coordinates should be transformed. "RD" defaults to "EPSG:28992". + Default "RD" + + Returns + ------- + xr.Dataset + Space-time matrix with the added layers `projection_x` and `projection_y`, where projection is the requested + parameter `projection` in lower case. + + Raises + ------ + AssertionError + When layers "lon" or "lat" do not exist in `stm`. + """ + assert "lon" in stm.keys(), "Expected a space-time matrix with longitude layer named lon but it is not there!" + assert "lat" in stm.keys(), "Expected a space-time matrix with latitude layer named lat but it is not there!" + if projection == "RD": + projection_formatted = "EPSG:28992" + elif projection[:5] == "EPSG:": + projection_formatted = projection + else: + raise ValueError(f"Invalid projection provided! Expected 'RD' or 'EPSG:###' but got {projection}!") + + wgs84 = pyproj.Transformer.from_crs("EPSG:4326", projection_formatted, always_xy=True).transform + # Convert Lat and Lon to coordinates + proj_x, proj_y = wgs84(stm["lon"], stm["lat"]) + + # Add coordinates to the dataset + stm = stm.assign({f"{projection.lower()}_x": (["space"], proj_x)}) + stm = stm.assign({f"{projection.lower()}_y": (["space"], proj_y)}) + + return stm + + +def add_stm_time_deltas(stm: xr.Dataset) -> xr.Dataset: + """Add the time differences since the first image to a space-time matrix. + + Parameters + ---------- + stm: xr.Dataset + the space-time matrix with an axis "time" + + Returns + ------- + xr.Dataset + the space-time matrix with two new variables: + - `days_since_first_img`, the number of days since the first epoch in the STM + - `years_since_first_img`, the number of years since the first epoch in the STM, assuming 365.2425 days per year + + """ + assert "time" in stm.keys(), "Expected STM to have time axis but it's not there!" + # Add extra time coordinate variables for time intervals since first image + days = np.array( + [ + (npdatetime64_to_datetime(date) - npdatetime64_to_datetime(stm["time"].values[0])).days + for date in stm["time"].values + ] + ) + stm = stm.assign({"days_since_first_img": (["time"], days)}) + stm = stm.assign({"years_since_first_img": (["time"], days / 365.2425)}) + + return stm + + +def stm_compute_single_time_differences( + stm: xr.Dataset, single_difference_mother: str | datetime = "auto" +) -> xr.Dataset: + """Compute the single differences of an STM in time with respect to a given mother image. + + This computes the single difference complex value, phase, unnormalized amplitude, and h2ph values with respect + to the provided single difference mother. The mother image is the first image acquired on or after the provided + date (if a datetime object or str object is provided), or the mother image of the input dataset (if 'auto' mode + is selected). + + Parameters + ---------- + stm: xr.Dataset + the space-time matrix with an axis "time" and "space", and variables `h2ph` and `complex` + single_difference_mother: datetime | str + the date to be used as the mother image for the single difference computations, in one of three formats: + - 'auto' : will detect the mother image in the input SLC dataset, and use that epoch. + - datetime object + - str object, formatted as YYYYMMDD + + Returns + ------- + xr.Dataset + the space-time matrix with four new variables: + - sd_h2ph (space, time): single difference height to phase conversion with respect to single_difference_mother + - sd_complex (space, time): single difference complex phasor with respect to single_difference_mother + - sd_amplitude_unnormalized (space, time): single difference complex phasor amplitude to + single_difference_mother, not normalized + - sd_phase (space, time): single difference phase with respect to single_difference_mother + + Raises + ------ + ValueError + Raised when: + - single_difference_mother is of an unsupported format + - the date provided to single_difference_mother is not in the input stack date range + """ + # Identify the mother image + if isinstance(single_difference_mother, datetime): + format_mother_date = datetime( + single_difference_mother.year, single_difference_mother.month, single_difference_mother.day, tzinfo=pytz.UTC + ) + mother_index = [ + idx for idx, date in enumerate(stm["time"].values) if format_mother_date <= npdatetime64_to_datetime(date) + ] # select all images beyond the mother date + elif isinstance(single_difference_mother, str): + if single_difference_mother == "auto": + mother_index = np.where(abs(stm["h2ph"]).sum(axis=0).values == 0)[0] + elif len(single_difference_mother) == 8: + format_mother_date = datetime( + eval(single_difference_mother[:4]), + eval(single_difference_mother[4:6].lstrip("0")), + eval(single_difference_mother[6:].lstrip("0")), + tzinfo=pytz.UTC, + ) + mother_index = [ + idx + for idx, date in enumerate(stm["time"].values) + if format_mother_date <= npdatetime64_to_datetime(date) + ] # select all images beyond the mother date + else: + raise ValueError(f'Cannot parse {single_difference_mother}, not of type "auto" or "YYYYMMDD"!') + else: + raise ValueError(f"Unknown format {type(single_difference_mother)} for single_difference_mother!") + if len(mother_index) == 0: + raise ValueError( + f"Cannot find provided mother date {single_difference_mother}, " + "please provide a date that is within the range of the stack! Possible dates: " + f"{stm.time.values[0]}--{stm.time.values[-1]}" + ) + sd_mother_index = mother_index[0] # 0 in case more than 1 image is detected + # In that case we take the first image that was detected, as this is expected + sd_mother = npdatetime64_to_datetime(stm["time"].values[sd_mother_index]) + + # Format the single difference mother, and save it to the STM + stm.attrs["ps_sd_mother"] = sd_mother.strftime("%Y%m%d") + + # calculate the h2ph single difference (= daughter - mother) + sd_h2ph = stm["h2ph"] - stm["h2ph"][:, sd_mother_index] + stm = stm.assign({"sd_h2ph": (["space", "time"], sd_h2ph.data)}) + + # calculate the complex single difference, the amplitude, and the phase + mother_comp = stm["complex"][:, sd_mother_index].conj() + sd_complex_transposed = stm["complex"].transpose() * mother_comp + sd_complex = sd_complex_transposed.transpose() + sd_phase = da.angle(sd_complex) + sd_amplitude_unnormalized = da.abs(sd_complex) + stm = stm.assign({"sd_complex": (["space", "time"], sd_complex.data)}) + stm = stm.assign({"sd_amplitude_unnormalized": (["space", "time"], sd_amplitude_unnormalized.data)}) + stm = stm.assign({"sd_phase": (["space", "time"], sd_phase.data)}) + + return stm + + +def identify_s1_orbits_in_aoi(lon: list | np.ndarray, lat: list | np.ndarray) -> tuple[list[str], dict]: + """Identify the Sentinel-1 orbit numbers and directions crossing a AoI. + + Parameters + ---------- + lon: list | np.ndarray + List of all the longitudes of all the points of interest in the AoI + lat: list | np.ndarray + List of all the latitudes of all the points of interest in the AoI + + Returns + ------- + list + The orbits overlapping with the AoI + dict + The footprints of the overlapping SLCs per track + """ + bbox = [[np.min(lon), np.max(lon)], [np.min(lat), np.max(lat)]] + wkt = ( + f"POLYGON((" + f"{bbox[0][0]} {bbox[1][0]}, " + f"{bbox[0][1]} {bbox[1][0]}, " + f"{bbox[0][1]} {bbox[1][1]}, " + f"{bbox[0][0]} {bbox[1][1]}, " + f"{bbox[0][0]} {bbox[1][0]}))" + ) + slcs = None + counter = 0 + while slcs is None: + try: + slcs = asf.geo_search( + intersectsWith=wkt, + platform=asf.PLATFORM.SENTINEL1, + beamMode="IW", + processingLevel="SLC", + start="one month ago", + end="now", + ) + except (asf.exceptions.ASFSearch5xxError, asf.exceptions.ASFSearchError, TimeoutError): + counter += 1 + print(f"ASF encountered an internal error. Retrying... (#{counter})") + + orbits = [ + f"s1_{slc.properties['flightDirection'].lower().replace('e', '')[:3]}_t{slc.properties['pathNumber']:0>3d}" + for slc in slcs + ] + filtered_orbits = list(sorted(list(set(orbits)))) + + extents = [slc.geojson()["geometry"]["coordinates"][0] for slc in slcs] + footprints = {} + for orbit in filtered_orbits: + footprints[orbit] = [] + + for extent in range(len(extents)): + footprints[orbits[extent]].append(extents[extent]) + + return filtered_orbits, footprints + + +def generate_pnt_uids(stm: xr.Dataset, ensure_unique: bool = True, overwrite: bool = False) -> xr.Dataset: + """Generate unique identifiers based on radar coordinates and assign them to the STM. + + The unique identifiers are assigned as a new coordinate "pnt_uid" in the STM. + + Parameters + ---------- + stm: xr.Dataset + The space-time matrix with coordinate "azimuth" and "range". + ensure_unique: bool, optional + Whether to ensure that the generated unique identifiers are unique. Default is True. + When True, numpy.unique is used to check for uniqueness and raise an error if duplicates are found. + For very large STMs, this can be computationally expensive. Consider setting to False if the radar + coordinates are known to be unique. + overwrite: bool, optional + Whether to overwrite existing "pnt_uid" coordinate in the STM. Default is False. + If False and "pnt_uid" already exists in coordinates or data variables, a warning + is logged and the STM is returned unchanged. + If True and "pnt_uid" exists in data variables, it is dropped before generating new identifiers. + + Returns + ------- + xr.Dataset + The space-time matrix with an added unique identifier coordinate "pnt_uid" + """ + # Copy the input STM to avoid modifying it directly + stm_output = stm.copy() + + # Check if pnt_uid already exists + if "pnt_uid" in stm.coords and not overwrite: + warning_msg = ( + "No pnt_uid has been generated. " + "STM already contains 'pnt_uid' coordinate. " + "Set 'overwrite=True' to regenerate unique identifiers." + ) + logger.warning(warning_msg) + return stm_output + + # Check if pnt_uid is in data variables + if "pnt_uid" in stm.data_vars: + if not overwrite: + warning_msg = ( + "No pnt_uid has been generated. " + "STM already contains 'pnt_uid' data variable. " + "Setting 'overwrite=True' will drop this data variable " + "and regenerate unique identifiers as coordinates." + ) + logger.warning(warning_msg) + return stm_output + else: + stm_output = stm_output.drop_vars("pnt_uid") + + # Check input: + # stm should have coordinates "azimuth" and "range" + # they should only have space dimension + # there should be no nan values in these coordinates + for key_dim in ["azimuth", "range"]: + assert key_dim in stm.coords, f"Expected STM to have coordinate '{key_dim}'." + assert stm[key_dim].dims == ("space",), f"Coordinate '{key_dim}' should have and only have 'space' dimension." + assert not np.any(np.isnan(stm[key_dim].values)), f"Coordinate '{key_dim}' contains NaN values." + + # Generate unique identifiers + # This is done by pandas hashing the azimuth and range coordinates together + # Index is set to False to avoid including the index in the hash + # reset coords to avoid involving other coords in the hash + df = stm.reset_coords()[["azimuth", "range"]].to_dataframe() + uid = pd.util.hash_pandas_object(df, index=False).values + + # Ensure uniqueness if requested + if ensure_unique: + unique_uids = np.unique(uid) + if unique_uids.shape[0] != uid.shape[0]: + logger.error("Duplicate unique identifiers detected in STM!") + raise ValueError("Generated unique identifiers are not unique. Check radar coordinates for duplicates.") + + # Assign unique identifiers to the STM + stm_output = stm_output.assign_coords({"pnt_uid": (["space"], uid)}) + + return stm_output + + +def compute_phase_difference( + value_source: np.ndarray, + value_target: np.ndarray, + method: Literal["subtract", "conjmult"], +) -> np.ndarray: + """Calculate the phase difference between two STMs. + + When method is "subtract", the expected input values are phases, and the phase difference is calculated + by simple subtraction. + When method is "conjmult", the expected input values are complex values, and the phase difference is + calculated by conjugate multiplication of the complex values, then taking the angle of the result. + + Parameters + ---------- + value_source: np.ndarray + The values of the source STM. + value_target: np.ndarray + The values of the target STM. + method: Literal["subtract", "conjmult"] + The method to calculate the phase difference. Can be either "subtract" or "conjmult". + + Returns + ------- + np.ndarray + Array containing the phase differences between ``value_target`` and ``value_source``, + computed according to the selected method. + """ + if method == "subtract": + d_phase = value_target - value_source + elif method == "conjmult": + # check that the input values are complex + if not np.iscomplexobj(value_source) or not np.iscomplexobj(value_target): + raise ValueError("Input values must be complex when using 'conjmult' method.") + d_phase = np.angle(value_target * value_source.conj()) + else: + raise NotImplementedError(f"Unknown difference method {method}, known are subtract and conjmult") + + return d_phase + + +def convert_geographic_coords_to_euclidean( + lon: np.ndarray | list, + lat: np.ndarray | list, + target_crs: str = "EPSG:28992", +) -> tuple[np.ndarray, np.ndarray]: + """Convert geographic coordinates (lon, lat) to projected Euclidean coordinates (x, y). + + Parameters + ---------- + lon: np.ndarray | list + Longitudes in degrees. + lat: np.ndarray | list + Latitudes in degrees. + target_crs: str, optional + Target coordinate reference system in EPSG format. Default is + "EPSG:28992" (Amersfoort / RD New). + + Returns + ------- + tuple[np.ndarray, np.ndarray] + Tuple containing: + - x: Projected x coordinates in meters. + - y: Projected y coordinates in meters. + """ + transformer = pyproj.Transformer.from_crs("EPSG:4326", target_crs, always_xy=True) + x, y = transformer.transform(lon, lat) + return np.array(x), np.array(y) + + +def concatenate_stms( + stm_list: list[xr.Dataset], +) -> xr.Dataset: + """Concatenate multiple STMs along the space dimension. + + This function concatenates a list of xarray Datasets representing space-time matrices (STMs) + along the space dimension. + It ensures that all STMs have the required dimensions and coordinates, and handles time-only variables appropriately + to prevent broadcasting issues during concatenation. + If the same time-only variable exists in multiple STMs, it is assumed to be identical across those STMs. Otherwise, + an error will be raised during the concatenation process. + If a time-only variable exists in only a subset of the STMs, it will be included in the final concatenated STM, + assuming this data variable applies to all points in the concatenated STM. + + Parameters + ---------- + stm_list : list[xr.Dataset] + List of xarray Datasets representing STMs to concatenate. + + Returns + ------- + xr.Dataset + Concatenated STM along the space dimension. + """ + # Check stm_list is a non-empty list + assert isinstance(stm_list, list) and len(stm_list) > 0, "stm_list must be a non-empty list of xarray Datasets." + + # All STMs should have space and time dimensions + for idx, stm in enumerate(stm_list): + assert isinstance(stm, xr.Dataset), f"STM at index {idx} is not a Dataset." + assert "space" in stm.dims, f"STM at index {idx} does not have 'space' dimension." + assert "time" in stm.dims, f"STM at index {idx} does not have 'time' dimension." + + # All STMs should have the same size for the time dimension + reference_time_size = stm_list[0].sizes["time"] + for idx, stm in enumerate(stm_list[1:], start=1): + current_time_size = stm.sizes["time"] + assert current_time_size == reference_time_size, ( + f"STM at index {idx} has time dimension size {current_time_size}, expected {reference_time_size}." + ) + + # All coordinates of all STMs should be 1D, only space or time dimension + for idx, stm in enumerate(stm_list): + for coord in stm.coords: + assert len(stm[coord].dims) == 1, ( + f"Coordinate '{coord}' in STM at index {idx} is not 1D (dims {stm[coord].dims})." + ) + + # Identify and temporarily convert time-only variables to coords to prevent broadcasting during concat + time_only_vars = set() + for ds in stm_list: + time_only_vars.update(v for v in ds.data_vars if ds[v].dims == ("time",)) + + # Set identified time-only variables as coordinates in each dataset where they appear + for idx, ds in enumerate(stm_list): + for var in time_only_vars: + if var in ds.data_vars: + ds = ds.set_coords(var) + stm_list[idx] = ds + + # Concat in space dimension with all (space, time) data variables, filling NaNs for missing vars + stm_dens_pnts_output = xr.concat(stm_list, dim="space", data_vars="all") + + # Reset coordinates back to data variables; only reset those that are still coordinates + coords_to_reset = [v for v in time_only_vars if v in stm_dens_pnts_output.coords] + stm_dens_pnts_output = stm_dens_pnts_output.reset_coords(names=coords_to_reset, drop=False) + + return stm_dens_pnts_output + + +def get_m2ph(wavelength: float = WAVELENGTH_S1): + """Get the conversion factor from meters to phase.""" + return -4 * np.pi / wavelength From a2ad8cea22e77a40e6ebfdc3557e29835641c771 Mon Sep 17 00:00:00 2001 From: Ou Ku Date: Mon, 7 Sep 2026 11:30:07 +0200 Subject: [PATCH 2/2] re-org files --- arc_estimation.py | 1782 -------------- constants.py | 11 - depsi/arc_estimation.py | 2 +- depsi/constants.py | 3 + depsi/network.py | 45 + stochastic.py => depsi/stochastic.py | 0 depsi/utils.py | 7 +- network.py | 2054 ----------------- test_network.py | 531 ----- tests/test_network.py | 29 + .../test_stochastic.py | 2 +- utils.py | 841 ------- 12 files changed, 85 insertions(+), 5222 deletions(-) delete mode 100644 arc_estimation.py delete mode 100644 constants.py rename stochastic.py => depsi/stochastic.py (100%) delete mode 100644 network.py delete mode 100644 test_network.py rename test_stochastic.py => tests/test_stochastic.py (97%) delete mode 100644 utils.py diff --git a/arc_estimation.py b/arc_estimation.py deleted file mode 100644 index 5a4d9ad8..00000000 --- a/arc_estimation.py +++ /dev/null @@ -1,1782 +0,0 @@ -"""arc estimation algorithms.""" - -from typing import Literal - -import dask.array as da -import numpy as np -import xarray as xr -from scipy.optimize import curve_fit - -import depsi.model_definition as md -import depsi.stats as est -from depsi.utils import get_distance, wrap_phase - -# Constants -STOP_HEIGHT = 1e-4 # Stop search step for height [m] -STOP_VEL = 1e-7 # Stop search step for velocity [m/y] -MAX_COUNT = 10 # Maximum number of search iterations -THRES_TEMP_COH_MEMORY = 500 # unit MB. Threshold to use dask array for temporal coherence calculation in periodogram. - - -def _compute_dd(sd_complex_i, sd_complex_j, sd_quality_i, sd_quality_j): - """Compute the Double-Difference (DD) phase observation given complex time series of points i and j. - - Point i serves as the reference point and is subtracted from point j: - - `sd_complex_conj_i = sd_complex_i.conj()` - - `dd_arc = sd_complex_j * sd_complex_conj_i` - - The function also calculates the diagonal of the Variance-Covariance Matrix (VCM) of the DD phase - (`Q_yy_diagonal_sigma`), which represents the standard deviation (sigma) of the DD phase quality. - - Parameters - ---------- - sd_complex_i : np.ndarray - Complex time series of the reference point i. Shape (n, ). - sd_complex_j : np.ndarray - Complex time series of point j. Shape (n, ). - sd_quality_i : np.ndarray - Quality values (sigma) of the single-difference (SD) phase for point i. Shape (n, ). - sd_quality_j : np.ndarray - Quality values (sigma) of the single-difference (SD) phase for point j. Shape (n, ). - - Returns - ------- - tuple - - dd_arc : np.ndarray - Double-difference (DD) phase observations. Shape (n, ). - - Q_yy_diagonal_sigma : np.ndarray - Diagonal values of the Variance-Covariance Matrix (VCM) of the DD phases. Shape (n, ). - - Example - ------- - >>> dd_arc, Q_yy_diagonal_sigma = _compute_dd(sd_complex_i, sd_complex_j, - sd_quality_i, sd_quality_j) - """ - sd_complex_conj_i = sd_complex_i.conj() # Compute the complex conjugate for the reference point i - dd_arc = sd_complex_j * sd_complex_conj_i - - # Compute the diagonal of the VCM of the dd phases - Q_yy_diagonal_sigma = np.sqrt((sd_quality_i) ** 2 + (sd_quality_j) ** 2) - - return dd_arc, Q_yy_diagonal_sigma - - -def _unwrap_phases_filter(filter_length, arc_dd, phase_arc, jump): - """Filter and unwrap double-difference (DD) arc phase observations in the complex domain. - - This function filters the real and imaginary components of the DD arc observations, computes the filtered phase - (angle), and unwraps the phase based on the filtered function. It simplifies the detection of 2π jumps and applies - corrections to produce an unwrapped phase time series. - - Parameters - ---------- - filter_length : int - Length of the moving average filter used to smooth the real and imaginary components of the arc observation. - arc_dd : np.ndarray - Double-difference arc observations in the complex domain. Shape (n, ). - phase_arc : np.ndarray - Wrapped phase observations for the arc. Shape (n, ). - jump : float - Threshold as a fraction of 2π used to detect and correct phase jumps during unwrapping. - - Returns - ------- - tuple - - phase_arc_unwrap : np.ndarray - Unwrapped phase observations for the arc. Shape (n, ). - - pi_diff : np.ndarray - Integer multiple of 2π differences between the unwrapped filtered phase and the original phase. Shape (n, ). - - filtered_phase_wrap : np.ndarray - Filtered and wrapped phase observations. Shape (n, ). - - filter_real : np.ndarray - Filtered real component of the arc observations. Shape (n, ). - - filter_imag : np.ndarray - Filtered imaginary component of the arc observations. Shape (n, ). - - Notes - ----- - - The function applies a moving average filter to smooth the real and imaginary parts of the input arc observations. - - The filtered phase is used to detect 2π phase jumps, and a correction is applied to unwrap the phase. - - Corrections are based on the difference between the original phase and the unwrapped filtered phase. - """ - ## Filter in Re and Im domain to detect ambiguity levels easily - filter_real = np.convolve(arc_dd.real, np.ones(filter_length) / filter_length, mode="same") - filter_imag = np.convolve(arc_dd.imag, np.ones(filter_length) / filter_length, mode="same") - complex_filtered = filter_real + filter_imag * 1j - filtered_phase_wrap = np.angle(complex_filtered) - - # Detect 2pi jumps in the filtered double difference and calculate shift of time series - abs_diff = np.zeros(len(filtered_phase_wrap) - 1) - for k in range(len(filtered_phase_wrap) - 1): - abs_diff[k] = filtered_phase_wrap[k + 1] - filtered_phase_wrap[k] - - shift = np.zeros(len(arc_dd)) - for k in range(len(abs_diff) - 1): - if abs_diff[k] < -jump * 2 * np.pi or abs_diff[k] > jump * 2 * np.pi: - if abs_diff[k] < -jump * 2 * np.pi: - shift[k + 1 :] = shift[k + 1 :] + 2 * np.pi - if abs_diff[k] > jump * 2 * np.pi: - shift[k + 1 :] = shift[k + 1 :] - 2 * np.pi - else: - shift[k + 1 : 0] = 0 - - # Unwrap the filtered phase function - filtered_phase_unwrap = filtered_phase_wrap + shift - - # Unwrap the DD phase observation for the arc based on the unwraped filtered function - # Correct for integer pi values differences between the filtered unwraped function and the DD observations - pi_diff = np.around((phase_arc - filtered_phase_unwrap) / (2 * np.pi), 0) - phase_arc_unwrap = phase_arc - 2 * np.pi * pi_diff - - return phase_arc_unwrap, pi_diff, filtered_phase_wrap, filter_real, filter_imag - - -def _scipy_fit_partition_2nd_order_bounds(breakpoints, x_data, arc_obs, initial_guess, bounds, vcm, n_max_iter): - """Estimate the parameters for an arc using a partitioned second-order polynomial fit. - - This function splits the time series of the arc observations into multiple partitions at the specified breakpoints. - For each partition, a 2nd order polynomial is fit, and the function ensures that the time series is continuous - at the breakpoints. The fitting process uses the `curve_fit` function with specified bounds and variance-covariance - matrix (vcm). - - Parameters - ---------- - breakpoints : list of int - Indices where the time series is divided into partitions. - x_data : np.ndarray - The input data for the model, typically including time-related variables. Shape (m, n). - arc_obs : np.ndarray - The observed arc, including both real and imaginary parts. Shape (m,). - initial_guess : np.ndarray - Initial guess for the unknown parameters. Shape (n,). - bounds : tuple of (lower_bounds, upper_bounds) - The bounds for the parameters during fitting. Each bound is an array of length n. - vcm : np.ndarray - The variance-covariance matrix of the observations. Shape (m, m). - n_max_iter : np.ndarray - The maximum nr of iterations for non-linear lsq per arc - - Returns - ------- - estimated_params : np.ndarray - The estimated parameters after fitting. Shape (n,). - pcov : np.ndarray - The covariance matrix of the estimated parameters. Shape (n, n). - - """ - - def _model_arc_2nd_order(x_data, *model_params, bkps=breakpoints): - """Model for an arc using a second-order polynomial for each partition in the time series. - - The arc time series is divided into partitions at the specified breakpoints, and for each partition, a 2nd - polynomial is fit. The model accounts for amplitude variations, displacement, cross range, and temperature. - - This function needs to be defined inside _scipy_fit_partition_2nd_order_bounds since the model - uses breakpoints. - - Parameters - ---------- - x_data : np.ndarray - The input data for the model, including time (t), temperature (T), and CR (cr2ph). Shape (m, n). - model_params : list of float - The parameters for the model. These include amplitude, displacement model parameters, and cross range - and temperature. The exact number and order of parameters depend on the number of breakpoints. - bkps : list of int, optional - Breakpoints at which the time series is divided into partitions. Default is breakpoints. - - Returns - ------- - np.ndarray - The modeled arc, which includes both the real and imaginary parts of the arc observations. - Shape (2 * m,). - """ - t, temp, cr2ph = x_data - - # Define the amplitude per partition - aa = model_params[0 : len(bkps)] - - # Define the parameters for the displacement model (third order polynomial) - intercept = model_params[len(bkps)] - p1 = model_params[len(bkps) + 1 : 2 * len(bkps) + 1] - p2 = model_params[2 * len(bkps) + 1 : 3 * len(bkps) + 1] - - # Define parameters for the cross range and temperature - height = model_params[-2] - expansion = model_params[-1] - - # Define the displacement phase and ampltiudes (they vary per partition) - displ = np.zeros(len(t)) - ampl = np.zeros(len(t)) - - # c is the start of a new partition - c = 0 - for i in range(len(bkps)): - # Define till what index the function should go (which is the end of the partition) - idx = int(bkps[i]) + 1 - - # Define the displacement values - displ[c:idx] = intercept - (p1[i] * t[c] + p2[i] * t[c] ** 2) + (p1[i] * t[c:idx] + p2[i] * t[c:idx] ** 2) - ampl[c:idx] = aa[i] - - # Define the 'intercept' of the new partition (that is the end of the next partition) - intercept = displ[idx - 1] - - # c is the starting point of a new partition - c = idx - 1 - - # Define the Real complex observation - real_part = ampl * np.cos(height * cr2ph + expansion * temp + displ) - imag_part = ampl * np.sin(height * cr2ph + expansion * temp + displ) - return np.append(real_part, imag_part) - - estimated_params, pcov, infodict, _, _ = curve_fit( - f=_model_arc_2nd_order, - xdata=x_data, - ydata=arc_obs, - p0=initial_guess, - bounds=bounds, - sigma=vcm, - absolute_sigma=True, - full_output=True, - max_nfev=n_max_iter, - ) - - return estimated_params, pcov - - -def _scipy_fit_partition_2nd_order_bounds_derivative( - breakpoints, x_data, arc_obs, initial_guess, bounds, vcm, n_max_iter -): - """Estimate the parameters for an arc using a partitioned second-order polynomial fit. - - This function splits the time series of the arc observations into multiple partitions at the specified breakpoints. - For each partition, a 2nd order polynomial is fit, and the function ensures that the time series is continuous - at the breakpoints. The fitting process uses the `curve_fit` function with specified bounds and variance-covariance - matrix (vcm). - - Parameters - ---------- - breakpoints : list of int - Indices where the time series is divided into partitions. - x_data : np.ndarray - The input data for the model, typically including time-related variables. Shape (m, n). - arc_obs : np.ndarray - The observed arc, including both real and imaginary parts. Shape (m,). - initial_guess : np.ndarray - Initial guess for the unknown parameters. Shape (n,). - bounds : tuple of (lower_bounds, upper_bounds) - The bounds for the parameters during fitting. Each bound is an array of length n. - vcm : np.ndarray - The variance-covariance matrix of the observations. Shape (m, m). - n_max_iter : np.ndarray - The maximum nr of iterations for non-linear lsq per arc - - Returns - ------- - estimated_params : np.ndarray - The estimated parameters after fitting. Shape (n,). - pcov : np.ndarray - The covariance matrix of the estimated parameters. Shape (n, n). - - """ - - def _model_arc_2nd_order_derivative(x_data, *model_params, bkps=breakpoints): - """Model for an arc using a second-order polynomial for each partition in the time series. - - The arc time series is divided into partitions at the specified breakpoints, and for each partition, a 2nd - polynomial is fit. The model accounts for amplitude variations, displacement, cross range, and temperature. - - This function needs to be defined inside _scipy_fit_partition_2nd_order_bounds since the model - uses breakpoints. - - Parameters - ---------- - x_data : np.ndarray - The input data for the model, including time (`t`), temperature (`T`), and CR (`cr2ph`). Shape (m, n). - model_params : list of float - The parameters for the model. These include amplitude, displacement model parameters, and cross range - and temperature. The exact number and order of parameters depend on the number of breakpoints. - bkps : list of int, optional - Breakpoints at which the time series is divided into partitions. Default is `breakpoints`. - - Returns - ------- - np.ndarray - The modeled arc, which includes both the real and imaginary parts of the arc observations. - Shape (2 * m,). - """ - t, temp, cr2ph = x_data - - nr_bkps = len(bkps) - - # Define the amplitude per partition - aa = model_params[0:nr_bkps] - - # Define the parameters for the displacement model (third order polynomial) - intercept = model_params[nr_bkps] - p1 = np.array(model_params[nr_bkps + 1 : 2 * nr_bkps + 1]) - p2 = np.array(model_params[2 * nr_bkps + 1 : 3 * nr_bkps + 1]) - - # Define parameters for the cross range and temperature - height = model_params[-2] - expansion = model_params[-1] - - # Define the displacement phase and ampltiudes (they vary per partition) - displ = np.zeros(len(t)) - ampl = np.zeros(len(t)) - - # c is the start of a new partition - c = 0 - prev_slope = 0 - - for i in range(nr_bkps): - # Define till what index the function should go (which is the end of the partition) - idx = int(bkps[i]) + 1 - if i > 0: - p1[i] = prev_slope # Zorg dat de eerste afgeleide overeenkomt met de vorige - intercept = displ[c] - - # Define the displacement values - displ[c:idx] = intercept - (p1[i] * t[c] + p2[i] * t[c] ** 2) + (p1[i] * t[c:idx] + p2[i] * t[c:idx] ** 2) - ampl[c:idx] = aa[i] - - # Define the 'intercept' of the new partition (that is the end of the next partition) - intercept = displ[idx - 1] - - prev_slope = p1[i] + 2 * p2[i] * t[idx - 1] - - # c is the starting point of a new partition - c = idx - 1 - - # Define the Real complex observation - real_part = ampl * np.cos(height * cr2ph + expansion * temp + displ) - imag_part = ampl * np.sin(height * cr2ph + expansion * temp + displ) - return np.append(real_part, imag_part) - - estimated_params, pcov, infodict, _, _ = curve_fit( - f=_model_arc_2nd_order_derivative, - xdata=x_data, - ydata=arc_obs, - p0=initial_guess, - bounds=bounds, - sigma=vcm, - absolute_sigma=True, - full_output=True, - max_nfev=n_max_iter, - ) - - return estimated_params, pcov - - -def _model_arc_partitions_2nd_order_phases(x_data, model_params): - """Calculate the forward model of the total, displacement, cross range, and thermal phase for an arc. - - The function divides the time series into different partitions. For each partition, a 2nd order polynomial is used. - - Parameters - ---------- - x_data : tuple - Contains the following elements: - - bkps (list of int): Breakpoints that divide the time series into partitions. - - t (np.ndarray): Time data for the arc. - - temp (np.ndarray): Temperature data for the arc. - - cr2ph (np.ndarray): Cross-range data for the arc. - model_params : list of float - Model parameters for the arc: - - Amplitudes per partition (A). - - Displacement model parameters (intercept, p1, p2). - - Parameters for cross-range (H) and temperature (expansion). - - Returns - ------- - tuple - Contains the following elements: - - phase_total (np.ndarray): The total phase (displacement + cross range + thermal). - - phase_thermal (np.ndarray): The thermal phase. - - phase_cross_range (np.ndarray): The cross-range phase. - - phase_displacement (np.ndarray): The displacement phase. - - real_part (np.ndarray): The real part of the arc observation. - - imag_part (np.ndarray): The imaginary part of the arc observation. - """ - bkps, t, temp, cr2ph = x_data - - # Define the amplitude per partition - aa = model_params[0 : len(bkps)] - - # Define the parameters for the displacement model (third order polynomial) - intercept = model_params[len(bkps)] - p1 = model_params[len(bkps) + 1 : 2 * len(bkps) + 1] - p2 = model_params[2 * len(bkps) + 1 : 3 * len(bkps) + 1] - - # Define parameters for the cross range and temperature - height = model_params[-2] - expansion = model_params[-1] - - # Define the displacement phase and ampltiudes (they vary per partition) - displ = np.zeros(len(t)) - ampl = np.zeros(len(t)) - - # c is the start of a new partition - c = 0 - for i in range(len(bkps)): - # Define till what index the function should go (which is the end of the partition) - idx = int(bkps[i]) + 1 - - # Define the displacement values - displ[c:idx] = intercept - (p1[i] * t[c] + p2[i] * t[c] ** 2) + (p1[i] * t[c:idx] + p2[i] * t[c:idx] ** 2) - ampl[c:idx] = aa[i] - - # Define the 'intercept' of the new partition (that is the end of the next partition) - intercept = displ[idx - 1] - - # c is the starting point of a new partition - c = idx - 1 - - # Define the Real complex observation - real_part = ampl * np.cos(height * cr2ph + expansion * temp + displ) - imag_part = ampl * np.sin(height * cr2ph + expansion * temp + displ) - - # Define the phases - phase_thermal = expansion * temp - phase_cross_range = height * cr2ph - phase_displacement = displ - phase_total = phase_thermal + phase_cross_range + phase_displacement - - return phase_total, phase_thermal, phase_cross_range, phase_displacement, real_part, imag_part - - -def _unwrap_phases(observed_phase, estimated_phase): - """Unwrap the observed phases based on modeled/estimated phase. - - This function corrects phase jumps by adjusting the observed phase values with respect to the estimated phase. - - Parameters - ---------- - observed_phase : np.ndarray - The observed phase values (typically in radians). - estimated_phase : np.ndarray - The estimated or modeled phase values based on the model parameters. - - Returns - ------- - np.ndarray - The unwrapped phase values, which are corrected to account for phase wrapping. - """ - pi_diff = np.around((estimated_phase - observed_phase) / (2 * np.pi), 0) - phase_unwrap = observed_phase + 2 * np.pi * pi_diff - - return phase_unwrap - - -def _compute_residuals_per_partition_stm(y_arc, y_est, Q_dd, bkps): - """Compute the rmse and std of the residuals per partition. - - This function can be used to compare values with a predefined Q matrix. - - Parameters - ---------- - y_arc : np.ndarray - The observed (real) arc values (should be in a flattened array). - y_est : np.ndarray - The estimated arc values (should be in a flattened array). - Q_dd : np.ndarray - The variance-covariance matrix (Q_dd) of the residuals. - bkps : list or np.ndarray - Breakpoints indicating where the arc is divided into different partitions. - - Returns - ------- - tuple - - rmse_partition (list): List of RMSE values computed per partition. - - std_est_partition (list): List of standard deviations of the residuals per partition. - - q_per_partition (list): List of standard deviations from the Q_dd matrix for each partition. - """ - start = 0 - rmse_partition = [] - std_est_partition = [] - q_per_partition = [] - - y_arc = y_arc.flatten() - y_est = y_est.flatten() - - # Loop over the partitions to comput residues per partition - # and add the values to lists - for i in range(len(bkps)): - n = bkps[i] - start - - rmse_s = np.sqrt((np.sum((y_arc[start : bkps[i]] - y_est[start : bkps[i]]) ** 2)) / n) - std_s = np.sqrt(np.var(y_arc[start : bkps[i]] - y_est[start : bkps[i]])) - q_per_s = np.sqrt(Q_dd[start, start]) # the apriori defined quality per partition - - rmse_partition.append(rmse_s) - std_est_partition.append(std_s) - q_per_partition.append(q_per_s) - - start = bkps[i] - - return rmse_partition, std_est_partition, q_per_partition - - -def _flatten_arrays_in_dict(dictionary): - """Flatten arrays in a dictionary. - - Function is required and used in arc_estimation_functions - - Args: - dictionary (dict): dictionary with arrays - - Returns: - dict: dictionary with flattend arrays - """ - dictionary = {key: np.array(value) for key, value in dictionary.items()} - - for key, value in dictionary.items(): - if isinstance(value, np.ndarray): - if value.ndim > 1: # Only flatten the array if the dimension is larger than 1 - dictionary[key] = value.ravel() # get 1D - return dictionary - - -def arc_estimation_xarray_input( - stm_pnt_i, - stm_pnt_j, - bounds, - m2ph, - n_max_iter, - partition_quality_label: str, - x_crd_label: str = "rd_x", - y_crd_label: str = "rd_y", - coordinate_type: Literal["euclidean", "geographic"] = "euclidean", - filter_length_complex=30, - jump_percentage_2pi=0.85, - vcm_complex_method="mad_median", - test_stochastics=False, - print_output=False, -): - """Estimate parameters for the arc defined between the connection point j and control point i. - - This function performs a series of computations for and arc, - including variance-covariance matrix computation, double-difference phase estimation, and parameter fitting - in both the phase and complex domains. The results of these calculations are stored in structured dictionaries. - - Parameters - ---------- - stm_pnt_i : Xarray.Dataset - Input space time matrix for the reference point i - stm_pnt_j : Xarray.Dataset - Input space time matrix for connection point j - bounds : tuple of lists - Bounds for parameter estimation in the format (lower_bounds, upper_bounds). - m2ph : float - Conversion factor from meters to phase. - n_max_iter : np.ndarray - The maximum nr of iterations for non-linear lsq per arc - partition_quality_label: str - Layer name in the STM of the SLC quality - x_crd_label: str, default "rd_x" - Label of the x-coordinate in the STMs (for geographic, this is 'lon') - y_crd_label: str, default "rd_y" - Label of the y-coordinate in the STMs (for geographic, this is 'lat') - coordinate_type: Literal["euclidean", "geographic"], default "euclidean" - Whether to compute distances in Euclidean space (for RD) or geographic distance (for lat/lon) - filter_length_complex : int, optional - Length of the filter for phase unwrapping (default: 30). - jump_percentage_2pi : float, optional - Threshold for unwrapping phase jumps in terms of 2π (default: 0.85). - vcm_complex_method : str, optional - Method for variance-covariance matrix estimation in the complex domain. - Options are "sigma_mean" or "mad_median" (default: "mad_median"). - test_stochastics : bool, optional - Flag for performing stochastic testing (default: False). - print_output : bool, optional - Flag for enabling or disabling print statements (default: False). - - Returns - ------- - results : dict - Dictionary containing results for each arc, including: - - 'unwrap_phases_arc': Unwrapped phases for each arc. - - 'sigma_phases_arc': Phase variances for each arc. - - 'estimated_phase': Estimated phases for each arc. - - 'estimated_displ_phase': Displacement-related phases for each arc. - - 'estimated_thermal': Estimated thermal expansion coefficients. - - 'estimated_cross_range': Estimated cross-range components. - - 'estimated_cross_range_sigma': Uncertainties of cross-range estimates. - - 'estimated_thermal_sigma': Uncertainties of thermal estimates. - - 'estimated_thermal_phase': Thermal-related phases for each arc. - - 'estimated_cross_range_phase': Cross-range related phases for each arc. - - 'cr2ph_arcs': Cross-range-to-phase conversion factors for each arc. - - 'succeeded_arcs': List of arcs where parameter estimation succeeded. - - stochastic_results : dict, optional - Dictionary containing stochastic testing results (if `test_stochastics=True`), including: - - 'q_per_partition': Quality metrics for each partition. - - 'std_residuals_partition': Standard deviations of residuals for each partition. - - 'rmse_residuals_partition': RMSE of residuals for each partition. - - 'mean_sigma_post_arc': Mean post-fit sigma values for each arc. - - 'mean_a_priori_sigma_arc': Mean a priori sigma values for each arc. - - 'arc_length': Lengths of the arcs. - - 'mean_sigma_p_i': Mean quality metrics for the first point in each arc. - - 'mean_sigma_p_j': Mean quality metrics for the second point in each arc. - - Notes - ----- - 1. The function uses deterministic assignment for the CR component, setting it to zero for one of the points. - 2. The estimation process includes fallback mechanisms to handle cases where optimal parameters cannot be found. - 3. Requires external utility functions for phase unwrapping, functional model construction, and lsq estimation. - - Raises - ------ - ValueError - If an unknown `vcm_complex_method` is specified. - RuntimeError, ValueError - If parameter estimation fails for an arc during optimization. - """ - # If we want to do some tests on the stochastics - if test_stochastics: - stochastic_results = { - "q_per_partition": [], - "std_residuals_partition": [], - "rmse_residuals_partition": [], - "mean_sigma_post_arc": [], - "mean_a_priori_sigma_arc": [], - "arc_length": [], - "mean_sigma_p_i": [], - "mean_sigma_p_j": [], - } - - # Dictionary to store results for one arc - results = { - "unwrap_phases_arc": [], - "sigma_phases_arc": [], - "estimated_phase": [], - "estimated_displ_phase": [], - "estimated_thermal": [], - "estimated_cross_range": [], - "estimated_cross_range_sigma": [], - "estimated_thermal_sigma": [], - "estimated_thermal_phase": [], - "estimated_cross_range_phase": [], - "cr2ph_arcs": [], - "succeeded_arcs": [], - } - - print(f"idx pnt i: {int(stm_pnt_i['space'].values)}") - print(f"idx pnt j: {int(stm_pnt_j['space'].values)}") - - dates = stm_pnt_i["time"].values - Btemporal = stm_pnt_i["years_since_first_img"].values - temp = stm_pnt_i["temperature"].values - - # Extract information of the two points of the arc - pnt_i_idx = int(stm_pnt_i["space"].values) - sd_complex_i = stm_pnt_i["sd_complex"].values - slc_quality_i = stm_pnt_i[partition_quality_label].values - bkps_stm_i = stm_pnt_i["breakpoints"].values - sigma_ampl_sd_i = stm_pnt_i["partition_sd_amplitude_sigma"].values - mean_ampl_sd_i = stm_pnt_i["partition_sd_amplitude_mean"].values - mad_ampl_sd_i = stm_pnt_i["partition_sd_mad"].values - median_ampl_sd_i = stm_pnt_i["partition_sd_amplitude_median"].values - - pnt_j_idx = int(stm_pnt_j["space"].values) - sd_complex_j = stm_pnt_j["sd_complex"].values - slc_quality_j = stm_pnt_j[partition_quality_label].values - cr2ph_j = stm_pnt_j["sd_cr2ph"].values - bkps_stm_j = stm_pnt_j["breakpoints"].values - sigma_ampl_sd_j = stm_pnt_j["partition_sd_amplitude_sigma"].values - mean_ampl_sd_j = stm_pnt_j["partition_sd_amplitude_mean"].values - mad_ampl_sd_j = stm_pnt_j["partition_sd_mad"].values - median_ampl_sd_j = stm_pnt_j["partition_sd_amplitude_median"].values - - # Step 1: extract information on the arc - # Compute the arc length - arc_length = get_distance( - [stm_pnt_i[x_crd_label], stm_pnt_i[y_crd_label]], - [stm_pnt_j[x_crd_label], stm_pnt_j[y_crd_label]], - mode=coordinate_type, - ) - - # Combine breakpoints to have breakpoints per arc - bkps_arc_stm = bkps_stm_i + bkps_stm_j - # Define the indexes of the breakpoints for the arc - bkps = [index for index, value in enumerate(bkps_arc_stm) if value > 0] - bkps.append(len(dates) - 1) - - # Compute the cr2ph for the arc, equals to point j since we determinsitcally set the value for point i to zero - cr2ph_arc = cr2ph_j - - # Step 2. Compute the DD phases for the arc - # point i is the reference point and is subtracted from point j: - # Note that the output of Qyy_diagonal are actually sigmas and NO variances. Therefore we need to square the values - dd_arc, Qyy_diagonal = _compute_dd(sd_complex_i, sd_complex_j, slc_quality_i, slc_quality_j) - - # Compute the variance covariance matrix of the DD based on the NMAD for the arc - Qyy = np.identity(len(dates)) * Qyy_diagonal**2 - Qyy_inv = np.linalg.inv(Qyy) - - # Step 3. Estimate parameters in the phase domain - # This step is required to get proper intial estimates for the parameter estimation in the complex domain - # Unwrap the phases based on the filtered real and imaginary part - phase_arc_unwrap, _, _, _, _ = _unwrap_phases_filter( - filter_length_complex, dd_arc, np.angle(dd_arc), jump_percentage_2pi - ) - - # Compute the A matrix. Column order (cross_range, temperature, offset, velocity) must match how - # x_hat_arc_ph is indexed positionally below (and elsewhere in this function). - A_arc = md.construct_design_matrix( - ["cross_range", "temperature", "offset", "velocity"], - m2ph, - cross_range=cr2ph_arc, - temperature=temp, - n_epochs=len(Btemporal), - time=Btemporal, - ) - - # Define the observation vector for the arc, which is based on the 'unwrapped' phase based on the filter - y_arc = np.reshape(phase_arc_unwrap, (len(phase_arc_unwrap), 1)) - - # Estimate parameters in the phase domain - x_hat_arc_ph, Qx_hat_arc_ph = est.blue_q_yy_inv(A_arc, y_arc, Qyy_inv) - - # Step 4. Create VCM in the complex domain - # Here we will compute the VCM for the complex domain. - # It is possible to choose between the mean and sigma or mad and median amplitude per partition. - - # Estimate sigma of the DD phases - if vcm_complex_method == "sigma_mean": - sigma_dd = np.abs(mean_ampl_sd_i * mean_ampl_sd_j) * np.sqrt( - (sigma_ampl_sd_i / mean_ampl_sd_i) ** 2 + (sigma_ampl_sd_j / mean_ampl_sd_j) ** 2 - ) - - elif vcm_complex_method == "mad_median": - sigma_dd = np.abs(median_ampl_sd_i * median_ampl_sd_j) * np.sqrt( - (mad_ampl_sd_i * 1.4826 / median_ampl_sd_i) ** 2 + (mad_ampl_sd_j * 1.4826 / median_ampl_sd_j) ** 2 - ) - else: - raise ValueError( - f"You specified an unknown vcm complex method. The method -- {vcm_complex_method} -- does not exist" - ) - - # Compute VCM in the complex domain - sigma_complex = np.append( - sigma_dd, sigma_dd - ) # Real and Imag are stacked together since we use both of the observations - Q_dd_cmplx = np.identity(len(sigma_complex)) - np.fill_diagonal(Q_dd_cmplx, sigma_complex**2) - - # Step 5. Parameter estimation in the complex domain - # Complex data preparation for the arc - re_arc = dd_arc.real - im_arc = dd_arc.imag - arc_obs = np.append(re_arc, im_arc) - - # Combine all the independent variables in one independent variable - x_data = (bkps, Btemporal, temp, cr2ph_arc) # used in phase estimation - X_data = (Btemporal, temp, cr2ph_arc) # used in curve fit - - # Create arrays with initial values - x0_2_p = np.zeros(3 * len(bkps) + 3) # Create empty array for the bounds - x0_2_p[0 : len(bkps)] = np.ones(len(bkps)) * np.max(re_arc) # The amplitude to be estimated - x0_2_p[len(bkps) + 1] = x_hat_arc_ph[2, 0] # Interception of the dispalcement polynomial - x0_2_p[len(bkps) + 1 : 2 * len(bkps) + 1] = ( - np.ones(len(bkps)) * x_hat_arc_ph[3, 0] - ) # Value related displacement velocity - x0_2_p[2 * len(bkps) + 1 : 3 * len(bkps) + 1] = np.zeros( - len(bkps) - ) # Value related to the second compont of polynomial - x0_2_p[-2] = x_hat_arc_ph[0, 0] # Cross range - x0_2_p[-1] = x_hat_arc_ph[1, 0] # Thermal expansion - - # Define the bounds - ( - A_lower, - a_lower, - b_lower, - c_lower, - CR_lower, - exp_lower, - A_upper, - a_upper, - b_upper, - c_upper, - CR_upper, - exp_upper, - ) = bounds - - # define bounds for second order polynomial with partitions - bounds_upper_2_p = np.ones(len(bkps) * 3 + 3) - bounds_upper_2_p[0 : len(bkps)] = A_upper - bounds_upper_2_p[len(bkps)] = a_upper - bounds_upper_2_p[len(bkps) + 1 : 2 * len(bkps) + 1] = np.ones(len(bkps)) * b_upper - bounds_upper_2_p[2 * len(bkps) + 1 : 3 * len(bkps) + 1] = np.ones(len(bkps)) * c_upper - bounds_upper_2_p[-2] = CR_upper - bounds_upper_2_p[-1] = exp_upper - - bounds_lower_2_p = np.ones(len(bkps) * 3 + 3) - bounds_lower_2_p[0 : len(bkps)] = A_lower - bounds_lower_2_p[len(bkps)] = a_lower - bounds_lower_2_p[len(bkps) + 1 : 2 * len(bkps) + 1] = np.ones(len(bkps)) * b_lower - bounds_lower_2_p[2 * len(bkps) + 1 : 3 * len(bkps) + 1] = np.ones(len(bkps)) * c_lower - bounds_lower_2_p[-2] = CR_lower - bounds_lower_2_p[-1] = exp_lower - - bounds_2_p = (list(bounds_lower_2_p), list(bounds_upper_2_p)) - - # Curvefit with 2nd order displacement polynomial, partitions, and bounds - # Estimate the unknown parameters: - try: - x_hat_2_p_b, pcov_2_p_b = _scipy_fit_partition_2nd_order_bounds( - bkps, - X_data, - arc_obs, - x0_2_p, - bounds_2_p, - Q_dd_cmplx, - n_max_iter, - ) - - except (RuntimeError, ValueError) as e: - print(f"Optimal parameters not found. Skipping arc {(pnt_i_idx, pnt_j_idx)}") - print(f"Encountered error: {e}") - - # Fill everything with nans - ts_length = len(Btemporal) - - for key in [ - "unwrap_phases_arc", - "sigma_phases_arc", - "estimated_phase", - "estimated_displ_phase", - "estimated_thermal_phase", - "estimated_cross_range_phase", - "cr2ph_arcs", - ]: - results[key].append(np.full([ts_length], np.nan)) - for key in [ - "estimated_thermal", - "estimated_cross_range", - "estimated_cross_range_sigma", - "estimated_thermal_sigma", - ]: - results[key].append(np.nan) - - results["succeeded_arcs"].append((np.nan, np.nan)) - - if test_stochastics: - for key in [ - "q_per_partition", - "std_residuals_partition", - "rmse_residuals_partition", - "mean_sigma_post_arc", - ]: - stochastic_results[key].append(np.nan) - stochastic_results["mean_a_priori_sigma_arc"].append(np.mean(Qyy_diagonal)) - stochastic_results["arc_length"].append(arc_length) - stochastic_results["mean_sigma_p_i"].append(np.mean(slc_quality_i)) - stochastic_results["mean_sigma_p_j"].append(np.mean(slc_quality_j)) - - else: - # Estimate the phases: - phase_est_2_p_b, phase_th_2_p_b, phase_cross_range_2_p_b, phase_disp_2_p_b, _, _ = ( - _model_arc_partitions_2nd_order_phases(x_data, x_hat_2_p_b) - ) - # Unwrap the observed phases: - phase_unwrap_2_p_b = _unwrap_phases(np.angle(dd_arc), phase_est_2_p_b) - # Estimate 'residual' phase: - phase_res_2_p_b = phase_unwrap_2_p_b - phase_est_2_p_b - - # Add the results for the arc to the dictionary - results["unwrap_phases_arc"].append(phase_unwrap_2_p_b) - results["sigma_phases_arc"].append(Qyy_diagonal) - results["estimated_phase"].append(phase_est_2_p_b) - results["estimated_displ_phase"].append(phase_disp_2_p_b) - results["estimated_thermal"].append(x_hat_2_p_b[-1] * 1000 / m2ph) - results["estimated_cross_range"].append(x_hat_2_p_b[-2]) - results["estimated_cross_range_sigma"].append(np.sqrt(pcov_2_p_b[-2, -2])) - results["estimated_thermal_sigma"].append(np.sqrt(pcov_2_p_b[-1, -1]) * 1000 / m2ph) - results["estimated_thermal_phase"].append(phase_th_2_p_b) - results["estimated_cross_range_phase"].append(phase_cross_range_2_p_b) - results["cr2ph_arcs"].append(cr2ph_arc) - results["succeeded_arcs"].append((pnt_i_idx, pnt_j_idx)) - # Get the dictionaries in the right shape and format - results = _flatten_arrays_in_dict(results) - - if test_stochastics: - rmse_res_partition, std_res_partition, q_per_part = _compute_residuals_per_partition_stm( - phase_unwrap_2_p_b, phase_est_2_p_b, Qyy, bkps - ) - stochastic_results["q_per_partition"].append(q_per_part) - stochastic_results["std_residuals_partition"].append(std_res_partition) - stochastic_results["rmse_residuals_partition"].append(rmse_res_partition) - stochastic_results["mean_sigma_post_arc"].append(np.std(phase_res_2_p_b)) - stochastic_results["mean_a_priori_sigma_arc"].append(np.mean(Qyy_diagonal)) - stochastic_results["arc_length"].append(arc_length) - stochastic_results["mean_sigma_p_i"].append(np.mean(slc_quality_i)) - stochastic_results["mean_sigma_p_j"].append(np.mean(slc_quality_j)) - - # Get the dictionaries in the right shape and format - stochastic_results = _flatten_arrays_in_dict(stochastic_results) - - # Step 6. Printing - - if print_output: - print( - "Estimated cross_range (phase domain NMAD):", - np.around(x_hat_arc_ph[0, 0], 2), - "+/-", - np.around((np.sqrt(Qx_hat_arc_ph[0, 0])) / (-1 * m2ph), 2), - ) - print( - "Estimated cross_range (2nd order + partitions and bounds):", - np.around(x_hat_2_p_b[-2], 2), - "+/-", - np.around((np.sqrt(pcov_2_p_b[-2, -2])) / (-1 * m2ph), 2), - ) - print( - "Estimated thermal expansion (phase domain NMAD):", - np.around(x_hat_arc_ph[1, 0] * 1000 / m2ph, 4), - "+/-", - np.around(np.sqrt(Qx_hat_arc_ph[1, 1]) * 1000 / m2ph, 2), - ) - print( - "Estimated thermal expansion (2nd order + partitions and bounds):", - np.around(x_hat_2_p_b[-1] * 1000 / m2ph, 4), - np.around(np.sqrt(pcov_2_p_b[-1, -1]) * 1000 / m2ph, 2), - ) - print("") - print("") - - if test_stochastics: - return results, stochastic_results - - else: - return results - - -def arc_estimation_control_network( - arcs_to_analyse, - bounds, - m2ph, - n_max_iter, - Btemporal, - dates, - temp, - sd_complex, - slc_quality, - cr2ph, - ampl_ts, - bkps_stm, - mean_ampl_sd, - sigma_ampl_sd, - mad_ampl_sd, - median_ampl_sd, - x_coordinates, - y_coordinates, - coordinate_type: Literal["euclidean", "geographic"] = "euclidean", - filter_length_complex=30, - jump_percentage_2pi=0.85, - vcm_complex_method="mad_median", - test_stochastics=False, - print_output=False, -): - """Estimate parameters for arcs in a control network based on input time series and geodetic measurements. - - This function performs a series of computations for each arc in the control network, - including variance-covariance matrix computation, double-difference phase estimation, and parameter fitting - in both the phase and complex domains. The results of these calculations are stored in structured dictionaries. - - Parameters - ---------- - arcs_to_analyse : list of tuples - List of arcs, where each arc is defined as a tuple (i, j) representing indices of two points. - bounds : tuple of lists - Bounds for parameter estimation in the format (lower_bounds, upper_bounds). - m2ph : float - Conversion factor from meters to phase. - n_max_iter : np.ndarray - The maximum nr of iterations for non-linear lsq per arc - Btemporal : numpy.ndarray - Array of decimal years corresponding to the time series epochs. - dates : numpy.ndarray - Array of date indices or timestamps corresponding to the time series. - temp : numpy.ndarray - Array of temperature values for thermal expansion modeling. - sd_complex : numpy.ndarray - Complex-valued standard deviations of the signal for all points. - slc_quality : numpy.ndarray - Quality metric for single-look complex (SLC) data. - cr2ph : numpy.ndarray - Cross-range to phase conversion factors for the points. - ampl_ts : numpy.ndarray - Amplitude time series for each point. - bkps_stm : numpy.ndarray - Breakpoints for state transition modeling. - mean_ampl_sd : numpy.ndarray - Mean amplitudes for each point, used in variance modeling. - sigma_ampl_sd : numpy.ndarray - Standard deviations of amplitudes for each point. - mad_ampl_sd : numpy.ndarray - Median absolute deviations (MAD) of amplitudes. - median_ampl_sd : numpy.ndarray - Median amplitudes for each point. - x_coordinates : numpy.ndarray - X-coordinates of the points in the control network. - y_coordinates : numpy.ndarray - Y-coordinates of the points in the control network. - coordinate_type: Literal["euclidean", "geographic"], default "euclidean" - Whether the given coordinates are in Euclidean space (such as RD) or in geographic space (such as lon/lat) - filter_length_complex : int, optional - Length of the filter for phase unwrapping (default: 30). - jump_percentage_2pi : float, optional - Threshold for unwrapping phase jumps in terms of 2π (default: 0.85). - vcm_complex_method : str, optional - Method for variance-covariance matrix estimation in the complex domain. - Options are "sigma_mean" or "mad_median" (default: "mad_median"). - test_stochastics : bool, optional - Flag for performing stochastic testing (default: False). - print_output : bool, optional - Flag for enabling or disabling print statements (default: False). - - Returns - ------- - results : dict - Dictionary containing results for each arc, including: - - 'unwrap_phases_arc': Unwrapped phases for each arc. - - 'sigma_phases_arc': Phase variances for each arc. - - 'estimated_phase': Estimated phases for each arc. - - 'estimated_displ_phase': Displacement-related phases for each arc. - - 'estimated_thermal': Estimated thermal expansion coefficients. - - 'estimated_cross_range': Estimated cross-range components. - - 'estimated_cross_range_sigma': Uncertainties of cross-range estimates. - - 'estimated_thermal_sigma': Uncertainties of thermal estimates. - - 'estimated_thermal_phase': Thermal-related phases for each arc. - - 'estimated_cross_range_phase': Cross-range related phases for each arc. - - 'cr2ph_arcs': Cross-range-to-phase conversion factors for each arc. - - 'succeeded_arcs': List of arcs where parameter estimation succeeded. - - stochastic_results : dict, optional - Dictionary containing stochastic testing results (if `test_stochastics=True`), including: - - 'q_per_partition': Quality metrics for each partition. - - 'std_residuals_partition': Standard deviations of residuals for each partition. - - 'rmse_residuals_partition': RMSE of residuals for each partition. - - 'mean_sigma_post_arc': Mean post-fit sigma values for each arc. - - 'mean_a_priori_sigma_arc': Mean a priori sigma values for each arc. - - 'arc_length': Lengths of the arcs. - - 'mean_sigma_p_i': Mean quality metrics for the first point in each arc. - - 'mean_sigma_p_j': Mean quality metrics for the second point in each arc. - - Notes - ----- - 1. The function uses deterministic assignment for the CR component, setting it to zero for one of the points. - 2. The estimation process includes fallback mechanisms to handle cases where optimal parameters cannot be found. - 3. Requires external utility functions for phase unwrapping, functional model construction, and lsq estimation. - - Raises - ------ - ValueError - If an unknown `vcm_complex_method` is specified. - RuntimeError, ValueError - If parameter estimation fails for an arc during optimization. - """ - # If we want to do some tests on the stochastics - if test_stochastics: - stochastic_results = { - "q_per_partition": [], - "std_residuals_partition": [], - "rmse_residuals_partition": [], - "mean_sigma_post_arc": [], - "mean_a_priori_sigma_arc": [], - "arc_length": [], - "mean_sigma_p_i": [], - "mean_sigma_p_j": [], - } - - # Dictionary to store results - results = { - "unwrap_phases_arc": [], - "sigma_phases_arc": [], - "estimated_phase": [], - "estimated_displ_phase": [], - "estimated_thermal": [], - "estimated_cross_range": [], - "estimated_cross_range_sigma": [], - "estimated_thermal_sigma": [], - "estimated_thermal_phase": [], - "estimated_cross_range_phase": [], - "cr2ph_arcs": [], - "succeeded_arcs": [], - } - - # Counting needed for saving data - p = 0 - - for a in arcs_to_analyse: - pnt_i_idx, pnt_j_idx = a - - print(f"idx pnt i: {pnt_i_idx}") - print(f"idx pnt j: {pnt_j_idx}") - - # Extract information of the two points of the arc - sd_complex_i = sd_complex[pnt_i_idx, :] - slc_quality_i = slc_quality[pnt_i_idx, :] - ampl_i = ampl_ts[pnt_i_idx, :] - bkps_stm_i = bkps_stm[pnt_i_idx, :] - sigma_ampl_sd_i = sigma_ampl_sd[pnt_i_idx, :] - mean_ampl_sd_i = mean_ampl_sd[pnt_i_idx, :] - mad_ampl_sd_i = mad_ampl_sd[pnt_i_idx, :] - median_ampl_sd_i = median_ampl_sd[pnt_i_idx, :] - - sd_complex_j = sd_complex[pnt_j_idx, :] - slc_quality_j = slc_quality[pnt_j_idx, :] - cr2ph_j = cr2ph[pnt_j_idx] - bkps_stm_j = bkps_stm[pnt_j_idx, :] - sigma_ampl_sd_j = sigma_ampl_sd[pnt_j_idx, :] - mean_ampl_sd_j = mean_ampl_sd[pnt_j_idx, :] - mad_ampl_sd_j = mad_ampl_sd[pnt_j_idx, :] - median_ampl_sd_j = median_ampl_sd[pnt_j_idx, :] - - # Step 1. Extract information for the ARC - # Compute the arc length - - arc_length = get_distance( - [x_coordinates[pnt_i_idx], y_coordinates[pnt_i_idx]], - [x_coordinates[pnt_j_idx], y_coordinates[pnt_j_idx]], - mode=coordinate_type, - ) - - # Extract the breakpoints for the arc - bkps_arc_stm = bkps_stm_i + bkps_stm_j - - # Define the indexes of the breakpoints for the arc - bkps = [index for index, value in enumerate(bkps_arc_stm) if value > 0] - bkps.append(len(dates) - 1) - - # The value for the cross range component is cr2ph of point j, since we deterministically set the - # cross-range component of point i to zero - cr2ph_arc = cr2ph_j - - # Step 2. Compute the DD phases for the arc - # point i is the reference point and is subtracted from point j: - dd_arc, Q_yy_diagonal = _compute_dd(sd_complex_i, sd_complex_j, slc_quality_i, slc_quality_j) - - # Compute the variance covariance matrix of the DD based on the NMAD for the arc - Q_yy = np.identity(len(dates)) * Q_yy_diagonal**2 - Q_yy_inv = np.linalg.inv(Q_yy) - - # Step 3. Estimate parameters in the phase domain - # Step is required to get proper intial estimates for the parameter estimation in the complex domain - # Unwrap the phases based on the filtered real and imaginary part - phase_arc_unwrap, _, _, _, _ = _unwrap_phases_filter( - filter_length_complex, dd_arc, np.angle(dd_arc), jump_percentage_2pi - ) - - # Compute the A matrix. Column order (cross_range, temperature, offset, velocity) must match how - # x_hat_arc_ph is indexed positionally below (and elsewhere in this function). - A_arc = md.construct_design_matrix( - ["cross_range", "temperature", "offset", "velocity"], - m2ph, - cross_range=cr2ph_arc, - temperature=temp, - n_epochs=len(Btemporal), - time=Btemporal, - ) - - # Define the observation vector for the arc, which is based on the 'unwrapped' phase based on the filter - y_arc = np.reshape(phase_arc_unwrap, (len(phase_arc_unwrap), 1)) - - # Estimate parameters in the phase domain - x_hat_arc_ph, Q_x_hat_arc_ph = est.blue_q_yy_inv(A_arc, y_arc, Q_yy_inv) - - # Step 4. VCM in the complex domain - # Here we will compute the VCM for the complex domain. - # It is possible to choose between the mean and sigma or mad and median amplitude per partition. - - # Estimate the DD sigma - if vcm_complex_method == "sigma_mean": - sigma_dd = np.abs(mean_ampl_sd_i * mean_ampl_sd_j) * np.sqrt( - (sigma_ampl_sd_i / mean_ampl_sd_i) ** 2 + (sigma_ampl_sd_j / mean_ampl_sd_j) ** 2 - ) - - elif vcm_complex_method == "mad_median": - sigma_dd = np.abs(median_ampl_sd_i * median_ampl_sd_j) * np.sqrt( - (mad_ampl_sd_i * 1.4826 / median_ampl_sd_i) ** 2 + (mad_ampl_sd_j * 1.4826 / median_ampl_sd_j) ** 2 - ) - else: - raise ValueError( - f"You specified an unknown vcm complex method. The method -- {vcm_complex_method} -- does not exist" - ) - - # Compute VCM in the complex domain - sigma_complex = np.append( - sigma_dd, sigma_dd - ) # Real and Imag are stacked together since we use both of the observations - Q_dd_cmplx = np.identity(len(sigma_complex)) - np.fill_diagonal(Q_dd_cmplx, sigma_complex**2) - - # Step 5. Parameter estimation in complex domain - # Complex data preparation for the arc - re_arc = dd_arc.real - im_arc = dd_arc.imag - arc_obs = np.append(re_arc, im_arc) - - # Combine all the independent variables in one independent variable - x_data = bkps, Btemporal, temp, cr2ph_arc - xx_data = Btemporal, temp, cr2ph_arc - - # Create initial value arrays - x0_2_p = np.zeros(3 * len(bkps) + 3) # Create empty array for the bounds - x0_2_p[0 : len(bkps)] = np.ones(len(bkps)) * np.max(re_arc) # The amplitude to be estimated - x0_2_p[len(bkps) + 1] = x_hat_arc_ph[ - 2, 0 - ] # Interception of the dispalcement polynomial. We use estimated values in the phase domain - x0_2_p[len(bkps) + 1 : 2 * len(bkps) + 1] = ( - np.ones(len(bkps)) * x_hat_arc_ph[3, 0] - ) # Value related displacement velocity in the displacement polynomial - x0_2_p[2 * len(bkps) + 1 : 3 * len(bkps) + 1] = np.zeros( - len(bkps) - ) # Value related to the second compont of the dispalcement polynomial - x0_2_p[-2] = x_hat_arc_ph[0, 0] # Cross range - x0_2_p[-1] = x_hat_arc_ph[1, 0] # Thermal expansion - - # Define bounds - ( - amp_lower, - a_lower, - b_lower, - c_lower, - cr_lower, - exp_lower, - amp_upper, - a_upper, - b_upper, - c_upper, - cr_upper, - exp_upper, - ) = bounds - - # define bounds for second order polynomial with partitions - bounds_upper_2_p = np.ones(len(bkps) * 3 + 3) - bounds_upper_2_p[0 : len(bkps)] = amp_upper - bounds_upper_2_p[len(bkps)] = a_upper - bounds_upper_2_p[len(bkps) + 1 : 2 * len(bkps) + 1] = np.ones(len(bkps)) * b_upper - bounds_upper_2_p[2 * len(bkps) + 1 : 3 * len(bkps) + 1] = np.ones(len(bkps)) * c_upper - bounds_upper_2_p[-2] = cr_upper - bounds_upper_2_p[-1] = exp_upper - - bounds_lower_2_p = np.ones(len(bkps) * 3 + 3) - bounds_lower_2_p[0 : len(bkps)] = amp_lower - bounds_lower_2_p[len(bkps)] = a_lower - bounds_lower_2_p[len(bkps) + 1 : 2 * len(bkps) + 1] = np.ones(len(bkps)) * b_lower - bounds_lower_2_p[2 * len(bkps) + 1 : 3 * len(bkps) + 1] = np.ones(len(bkps)) * c_lower - bounds_lower_2_p[-2] = cr_lower - bounds_lower_2_p[-1] = exp_lower - - bounds_2_p = (list(bounds_lower_2_p), list(bounds_upper_2_p)) - - # Estiamte parameters in the complex domain with a 2nd order displacement polynomial - # bounds and partitions - try: - x_hat_2_p_b, pcov_2_p_b = _scipy_fit_partition_2nd_order_bounds( - bkps, xx_data, arc_obs, x0_2_p, bounds_2_p, Q_dd_cmplx, n_max_iter - ) - except (RuntimeError, ValueError): - print(f"Optimal parameters not found. Skipping arc {(pnt_i_idx, pnt_j_idx)}") - - # Fill everything with nans - ts_length = len(ampl_i) - - for key in [ - "unwrap_phases_arc", - "sigma_phases_arc", - "estimated_phase", - "estimated_displ_phase", - "estimated_thermal_phase", - "estimated_cross_range_phase", - "cr2ph_arcs", - ]: - results[key].append(np.full([ts_length], np.nan)) - for key in [ - "estimated_thermal", - "estimated_cross_range", - "estimated_cross_range_sigma", - "estimated_thermal_sigma", - ]: - results[key].append(np.nan) - - results["succeeded_arcs"].append((np.nan, np.nan)) - - if test_stochastics: - for key in [ - "q_per_partition", - "std_residuals_partition", - "rmse_residuals_partition", - "mean_sigma_post_arc", - ]: - stochastic_results[key].append(np.nan) - stochastic_results["mean_a_priori_sigma_arc"].append(np.mean(Q_yy_diagonal)) - stochastic_results["arc_length"].append(arc_length) - stochastic_results["mean_sigma_p_i"].append(np.mean(slc_quality_i)) - stochastic_results["mean_sigma_p_j"].append(np.mean(slc_quality_j)) - - else: - # Estimate the phases: - ( - phase_est_2_p_b, - phase_th_2_p_b, - phase_cross_range_2_p_b, - phase_disp_2_p_b, - _, - _, - ) = _model_arc_partitions_2nd_order_phases(x_data, x_hat_2_p_b) - # Unwrap the observed phases: - phase_unwrap_2_p_b = _unwrap_phases(np.angle(dd_arc), phase_est_2_p_b) - # Estimate 'residual' phase: - phase_res_2_p_b = phase_unwrap_2_p_b - phase_est_2_p_b - - # Add the results for the arc to the dictionary - results["unwrap_phases_arc"].append(phase_unwrap_2_p_b) - results["sigma_phases_arc"].append(Q_yy_diagonal) - results["estimated_phase"].append(phase_est_2_p_b) - results["estimated_displ_phase"].append(phase_disp_2_p_b) - results["estimated_thermal"].append(x_hat_2_p_b[-1] * 1000 / m2ph) - results["estimated_cross_range"].append(x_hat_2_p_b[-2]) - results["estimated_cross_range_sigma"].append(np.sqrt(pcov_2_p_b[-2, -2])) - results["estimated_thermal_sigma"].append(np.sqrt(pcov_2_p_b[-1, -1]) * 1000 / m2ph) - results["estimated_thermal_phase"].append(phase_th_2_p_b) - results["estimated_cross_range_phase"].append(phase_cross_range_2_p_b) - results["cr2ph_arcs"].append(cr2ph_arc) - results["succeeded_arcs"].append((pnt_i_idx, pnt_j_idx)) - # Get the dictionaries in the right shape and format - # results = flatten_arrays_in_dict(results) - - if test_stochastics: - rmse_res_partition, std_res_partition, q_per_part = _compute_residuals_per_partition_stm( - phase_unwrap_2_p_b, phase_est_2_p_b, Q_yy, bkps - ) - stochastic_results["q_per_partition"].append(q_per_part) - stochastic_results["std_residuals_partition"].append(std_res_partition) - stochastic_results["rmse_residuals_partition"].append(rmse_res_partition) - stochastic_results["mean_sigma_post_arc"].append(np.std(phase_res_2_p_b)) - stochastic_results["mean_a_priori_sigma_arc"].append(np.mean(Q_yy_diagonal)) - stochastic_results["arc_length"].append(arc_length) - stochastic_results["mean_sigma_p_i"].append(np.mean(slc_quality_i)) - stochastic_results["mean_sigma_p_j"].append(np.mean(slc_quality_j)) - - # Get the dictionaries in the right shape and format - # stochastic_results = flatten_arrays_in_dict(stochastic_results) - - if print_output: - print( - "Estimated cross_range (phase domain NMAD):", - np.around(x_hat_arc_ph[0, 0], 2), - "+/-", - np.around((np.sqrt(Q_x_hat_arc_ph[0, 0])) / (-1 * m2ph), 2), - ) - print( - "Estimated cross_range (2nd order + partitions and bounds):", - np.around(x_hat_2_p_b[-2], 2), - "+/-", - np.around((np.sqrt(pcov_2_p_b[-2, -2])) / (-1 * m2ph), 2), - ) - print( - "Estimated thermal expansion (phase domain NMAD):", - np.around(x_hat_arc_ph[1, 0] * 1000 / m2ph, 4), - "+/-", - np.around(np.sqrt(Q_x_hat_arc_ph[1, 1]) * 1000 / m2ph, 2), - ) - print( - "Estimated thermal expansion (2nd order + partitions and bounds):", - np.around(x_hat_2_p_b[-1] * 1000 / m2ph, 4), - np.around(np.sqrt(pcov_2_p_b[-1, -1]) * 1000 / m2ph, 2), - ) - print("") - print("") - - p = p + 1 - - # tranform from list to np array - results = {key: np.array(value) for key, value in results.items()} - - if test_stochastics: - return results, stochastic_results - - return results - - -def periodogram( - stm: xr.Dataset, - key_dphase: str, - key_h2ph: str, - key_Btemporal: str, - std_obs: float = 1.0, - std_height: float = 50.0, - std_vel: float = 0.02, - init_height: float = 0.0, - init_vel: float = 0.0, - init_step_height: float = 3.0, - init_step_vel: float = 2e-3, - min_steps: int = 10, -): - """Periodogram algorithm. - - This function performs periodogram unwrapping on arcs. - - It uses a deformation model with two parameters: height and velocity to estimate the unwrapped phase. - - For computation efficiency, the design matrix is constructed only once for all arcs, utilizing the average - height-to-phase conversion factor (h2ph) across all arcs. The effect of using this average is corrected later. - - Parameters - ---------- - stm : xr.Dataset - Input Space-Time Matrix (STM) containing the wrapped phase, height-to-phase conversion factor, and year-time. - key_dphase : str - Key for the wrapped differential phase data variable in the STM. - key_h2ph : str - Key for the height-to-phase conversion factor in the STM. - key_Btemporal : str - Key for the temporal baseline in the STM. - The value should be in decimal years. - std_obs : float, optional - A-priori standard deviation of the observations in rads, by default 1.0. - This value is used to construct the stochastic model (Qyy) of the observations. - std_height : float, optional - A-priori standard deviation of the height in meters, by default 50.0. - This value is used to construct the boundaries of the initial search space for the height parameter. - std_vel : float, optional - A-priori standard deviation of the velocity in meters per year, by default 0.02. - This value is used to construct the boundaries of the initial search space for the velocity parameter. - init_height : float, optional - Initial value for the height parameter in meters, by default 0.0. - init_vel : float, optional - Initial value for the velocity parameter in meters per year, by default 0.0. - init_step_height : float, optional - Initial step size for the height parameter in meters, by default 3.0. - This value sets the resolution of the initial search space for the height parameter. - After every search, the step size will be reduced by a factor of 10. - init_step_vel : float, optional - Initial step size for the velocity parameter in meters per year, by default 2e-3. - This value sets the resolution of the initial search space for the velocity parameter. - After every search, the step size will be reduced by a factor of 10. - min_steps : int, optional - Minimum number of steps in the search space for the height and velocity parameters, by default 10. - If the number of steps in the initial search space is smaller than this value, it will be set to this value. - After the first search, the number of steps will be set to this value. - - Returns - ------- - Tuple[xr.DataArray, xr.DataArray, xr.DataArray, xr.DataArray, xr.DataArray] - Returns the unwrapped phase, ambiguities, estimated height, estimated velocity, and temporal coherence. - - Unwrapped phase: in rads, shape (n_arcs, n_obs), dtype np.float64. - - Ambiguities: unitless, shape (n_arcs, n_obs), dtype np.float64. - - Estimated height: in meters, shape (n_arcs,), dtype np.float64. - - Estimated velocity: in meters per year, shape (n_arcs,), dtype np.float64. - - Temporal coherence: unitless float number, norm of the complex coherence, scalar, dtype np.float64. - """ - # Compute m2ph (meters to phase) conversion factor from wavelength - if "wavelength" not in stm.attrs: - raise ValueError( - "Wavelength is not provided and not found in attributes of STM." - "Please make sure it is provided." - "For example: stm = stm.assign_attrs({'wavelength': wavelength})" - ) - wavelength = stm.attrs["wavelength"] - m2ph = -4 * np.pi / wavelength - - # Make sure year time only contains the time dimension - assert (len(stm[key_Btemporal].dims) == 1) and ("time" in stm[key_Btemporal].dims), ( - "year time should and only should contain the 'time' dimension." - ) - - # Load year time in memory - Btemporal = stm[key_Btemporal].values - - # Set up functional and stochastic model for all arcs - # Here we use the same h2ph (average over all arcs) for all arcs and correct the effect later - # Doing this avoids perform matrix inversion for each arc - h2ph_approx = stm[key_h2ph].mean(dim="space").values # Mean h2ph of all arcs - - # Design matrix B, size n_obs x n_params - # In B, h2ph should also be multiplied by m2ph since it did not when it was created - B = np.stack([h2ph_approx * m2ph, Btemporal * m2ph]).T - - # Stochastic model Qyy, size n_obs x n_obs - # This is the covariance matrix of the observations - n_obs = stm[key_dphase].sizes["time"] # number of observations - Qyy = np.diag(np.repeat(std_obs**2, n_obs)) - - # Normal matrix N and rhs for the least squares solution - N = B.T @ np.linalg.inv(Qyy) @ B # B.T * Qyy^-1 * B , size n_params x n_params - # Solve N * x = B.T * Qyy^-1, then rhs = N^-1 * B.T * Qyy^-1 - rhs = np.linalg.inv(N) @ B.T @ np.linalg.inv(Qyy) - - # check if the time dimension is not chunked, and unchunk it if necessary - if "time" in stm.chunks.keys(): - if len(stm.chunks["time"]) != 1: - stm = stm.chunk({"time": -1}) - - # Build initial search space for height and velocity - n_steps_height = max(round(2 * std_height / init_step_height), min_steps) - n_steps_vel = max(round(2 * std_vel / init_step_vel), min_steps) - init_search_space = _build_periodogram_search_space( - init_height, init_vel, init_step_height, init_step_vel, n_steps_height, n_steps_vel - ) - - # Perform one search for all arcs, and get the best initial estimates for height and velocity per arc - # This is motivated by the fact that the initial search space is the largest, and can be vectorized for all arcs - # First iteration, candidate modeled phases are identical for all arcs - # The residuals phase_residual_all_arcs is a large array with n_arcs x n_obs x n_search - # so use .data to avoid loading it into memory if it is a dask array - dphase_obs = stm[key_dphase].data # n_arcs x n_obs x 1 - - # If the memory size of phase_residual_all_arcs will exceed the threshold - # chunk dphase_obs and init_search_space to enable computation - mem_size_estimation = ( - dphase_obs.shape[0] * dphase_obs.shape[1] * init_search_space.shape[0] * dphase_obs.dtype.itemsize - ) / (1024**2) # in MB - if mem_size_estimation > THRES_TEMP_COH_MEMORY: - dphase_obs, init_search_space = _chunk_for_temp_coh_compute(dphase_obs, init_search_space) - - # Compute modelled phase for all arcs and all search candidates - phs_model = B @ init_search_space.T # n_obs x n_search - - # Expand dimensions and compute the phase residuals for all arcs and all search candidates - dphase_obs = dphase_obs[:, :, None] # n_arcs x n_obs x 1 - phs_model = phs_model[None, :, :] # 1 x n_obs x n_search - phase_residual_all_arcs = dphase_obs - phs_model - - # Find the best initial height and velocity based on the temporal coherence - coh_search_space_all_arcs = ( - np.cos(phase_residual_all_arcs).sum(axis=1) + 1j * np.sin(phase_residual_all_arcs).sum(axis=1) - ) / stm[key_dphase].sizes["time"] # n_arcs x n_search - coh_idx_all_arcs = np.argmax(np.abs(coh_search_space_all_arcs), axis=1) # n_arcs - - # Implicitly compute best coh index if dask array - coh_idx_all_arcs = coh_idx_all_arcs.compute() if isinstance(coh_idx_all_arcs, da.Array) else coh_idx_all_arcs - - # Build xr.DataArray for the initial height and velocity of all arcs - da_init_height_all_arcs = xr.DataArray( - init_search_space[coh_idx_all_arcs, 0], - dims=["space"], - ) - da_init_vel_all_arcs = xr.DataArray( - init_search_space[coh_idx_all_arcs, 1], - dims=["space"], - ) - - # Apply the _periodogram_arc on stm[key_dphase] along "space" dimension - # Set up input core dimensions, which are the dimensions _periodogram_arc will be applied to - # We are broadcasting _periodogram_arc on stm[key_dphase] and stm[key_h2ph] along the space dimension - # The height and velocity are scalars - # Therefore, we are only calling it on the "time" dimension for the first two parameters - # So we have the input_core_dims as [["time"], ["time"], [], []] - input_core_dims = [["time"], ["time"], [], []] - - # There are 5 outputs from _periodogram_arc - # The first two are np arrays with time dimension - # The other three are scalars, so they have no dimensions - output_core_dims = [["time"], ["time"], [], [], []] - - results = xr.apply_ufunc( - _periodogram_arc, - stm[key_dphase], - stm[key_h2ph], - da_init_height_all_arcs, - da_init_vel_all_arcs, - input_core_dims=input_core_dims, - output_core_dims=output_core_dims, - kwargs={ - "h2ph_approx": h2ph_approx, - "B": B, - "Qyy": Qyy, - "N": N, - "rhs": rhs, - "init_step_height": init_step_height, - "init_step_vel": init_step_vel, - "min_steps": min_steps, - }, - vectorize=True, - dask="parallelized", - output_dtypes=[np.float64, np.float64, np.float64, np.float64, np.float64], - ) - - return results - - -def _periodogram_arc( - phs_obs_wrapped: np.ndarray, - h2ph: np.ndarray, - init_height: float, - init_vel: float, - h2ph_approx: np.ndarray, - B: np.ndarray, - Qyy: np.ndarray, - N: np.ndarray, - rhs: np.ndarray, - init_step_height: float, - init_step_vel: float, - min_steps: float, -): - """Periodogram unwrapping for a single arc. - - Parameters - ---------- - phs_obs_wrapped : np.ndarray - Wrapped phase observations in radians, shape (n_obs,). - h2ph : np.ndarray: - Height-to-phase factor of the arc, shape (n_obs,). - init_height : float - Initial value for the height parameter in meters. - init_vel : float - Initial value for the velocity parameter in meters per year. - h2ph_approx : np.ndarray - Approximate height-to-phase factor calculated by spatial average of all h2ph, shape (n_obs,). - B : np.ndarray - Design matrix, size n_obs x n_params, where n_params = 2 (height and velocity). - Qyy : np.ndarray - Stochastic model of the observations, size n_obs x n_obs. - N : np.ndarray - Normal matrix, size n_params x n_params. - rhs : np.ndarray - Right-hand side matrix for the least squares solution, size n_params x n_obs. - init_step_height : float - Initial step size for the height parameter in meters. - init_step_vel : float - Initial step size for the velocity parameter in meters per year. - min_steps : float - Minimum number of steps in the search space for the height and velocity parameters. - - Returns - ------- - Tuple[np.ndarray, np.ndarray, float, float, float] - Returns the unwrapped phase, ambiguities, estimated height, estimated velocity, and temporal coherence. - - Unwrapped phase: in rads, shape (n_obs,), dtype np.float64. - - Ambiguities: unitless, shape (n_obs,), dtype np.float64. - - Estimated height: in meters, scalar, dtype np.float64. - - Estimated velocity: in meters per year, scalar, dtype np.float64. - - Temporal coherence: unitless float number, norm of the complex coherence, scalar, dtype np.float64. - """ - # Assign initial values for the search - step_height = init_step_height - step_vel = init_step_vel - param_height = init_height - param_vel = init_vel - - # Calculate the initial temporal coherence for the initial height and velocity, - # in case the search loop is not entered - phs_model = B @ np.array([param_height, param_vel]) # size n_obs - phase_residual = phs_obs_wrapped[:, None] - phs_model - coh_best = (np.cos(phase_residual).sum() + 1j * np.sin(phase_residual).sum()) / phs_obs_wrapped.shape[0] - - # Search loop - count = 0 - while step_height > STOP_HEIGHT and step_vel > STOP_VEL and count < MAX_COUNT: - # Build search space - search_space = _build_periodogram_search_space( - param_height, param_vel, step_height, step_vel, min_steps, min_steps - ) - - # Calculate the wrapped model phase for all candidates - phs_model = wrap_phase(B @ search_space.T) # size n_obs x n_search - - # Calculate the temporal coherence for all search candidates - # Expand dimension of phs_obs_wrapped to facilitate broadcasting - # No need to repeat phs_obs_wrapped since the minus operation will broadcast to the shape of phs_model - # Sum along axis=0 which is the observation axis - # Reference: van Leijen 2014, Eq. 4.55 - # The following implementation equivalent to: - # np.exp(1j * (np.expand_dims(phs_obs_wrapped, axis=1) - phs_model)).sum(axis=0) / phs_obs_wrapped.shape[0] - phase_residual = phs_obs_wrapped[:, None] - phs_model - coh_search_space = ( - np.cos(phase_residual).sum(axis=0) + 1j * np.sin(phase_residual).sum(axis=0) - ) / phs_obs_wrapped.shape[0] - - # Get the best temporal coherence value and its index - coh_idx = np.argmax(np.abs(coh_search_space)) - coh_best = coh_search_space[coh_idx] - - # Update values needed for search space - # Reduce step size to 1/10 - param_height = search_space[coh_idx, 0] - param_vel = search_space[coh_idx, 1] - step_height /= 10 - step_vel /= 10 - - count += 1 - - # Correct the height parameter for using h2ph_approx - # Method copied from MATLAB DePSI code - factor = np.median(h2ph / h2ph_approx) # correct factor - param_height = param_height / factor - - # Calculate the modelled phase and unwrapped phase - model_est = B @ np.array([param_height, param_vel]) + np.angle(coh_best) # Absolute modelled phase - dphase_new = wrap_phase(phs_obs_wrapped - model_est) # Wrapped modelled phase - ambiguities = np.round((model_est + dphase_new - phs_obs_wrapped) / (2 * np.pi)) # Ambiguities - phs_obs_unwrapped = 2 * np.pi * ambiguities + phs_obs_wrapped # Unwrapped phase - param = rhs @ phs_obs_unwrapped # [height_est, velocity_est] - - return phs_obs_unwrapped, ambiguities, param[0], param[1], np.abs(coh_best) - - -def _chunk_for_temp_coh_compute(phs_obs_wrapped, search_space): - """Chunk observations and search space for temporal coherence computation.""" - if isinstance(phs_obs_wrapped, da.Array): # Existing chunk size for the arc dimension - chunk_arcs = phs_obs_wrapped.chunks[0][0] - else: - # If phs_obs_wrapped is not a dask array, chunk it in the arc dimension, making each chunk about 10 MB - chunk_arcs = max(1, 10 * 1024**2 // (phs_obs_wrapped.shape[1] * phs_obs_wrapped.dtype.itemsize)) - phs_obs_wrapped = da.from_array(phs_obs_wrapped, chunks=(chunk_arcs, phs_obs_wrapped.shape[1])) - # Decide the chunk size for the search space dimension - # making each n_arcs x n_obs x n_search chunk about 100 MB - chunk_searches = max(1, 100 * 1024**2 // (chunk_arcs * phs_obs_wrapped.shape[1] * phs_obs_wrapped.dtype.itemsize)) - search_space = da.from_array(search_space, chunks=(chunk_searches, 2)) - - return phs_obs_wrapped, search_space - - -def _build_periodogram_search_space(init_height, init_vel, step_height, step_vel, n_steps_height, n_steps_vel): - """Construct the periodogram search space for height and velocity parameters. - - For both height and velocity, the candidates are generated around the initial values according to the step size - and the number of steps. On each side of the initial value, N candidates are generated with a step size, where - N is specified by `n_steps_height` and `n_steps_vel`, and the step size is specified by `step_height` and - `step_vel`. - - Then all possible combinations of height and velocity candidates are created to form - the search space. - - Parameters - ---------- - init_height : float - Initial height parameter in meters. - init_vel : float - Initial velocity parameter in meters per year. - step_height : int - Step size of search for height parameter, in meters. - step_vel : int - Step size of search for velocity parameter, in meters per year. - n_steps_height : int - Number of steps for height parameter on each side of the initial value. - n_steps_vel : int - Number of steps for velocity parameter on each side of the initial value. - - Returns - ------- - np.ndarray - Search space for height and velocity parameters, shape (n_candidates_vel * n_candidates_height, 2) - """ - height_candidates = np.arange( - init_height - n_steps_height * step_height, - init_height + n_steps_height * step_height + step_height, - step_height, - ) - - vel_candidates = np.arange( - init_vel - n_steps_vel * step_vel, init_vel + n_steps_vel * step_vel + step_vel, step_vel - ) - - # All possible combinations of height and velocity - search_space = np.array(np.meshgrid(height_candidates, vel_candidates)).T.reshape(-1, 2) - - return search_space diff --git a/constants.py b/constants.py deleted file mode 100644 index 14c5abad..00000000 --- a/constants.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Physical constants used in DePSI.""" - -# Earth radius -# Unit in meters -EARTH_RADIUS = 6378136 - -# Speed of light in vacuum -SPEED_OF_LIGHT = 299792458.0 # m/s - -# Sentinel-1 wavelength -WAVELENGTH_S1 = 0.055465763 # m diff --git a/depsi/arc_estimation.py b/depsi/arc_estimation.py index 7e649863..5a4d9ad8 100644 --- a/depsi/arc_estimation.py +++ b/depsi/arc_estimation.py @@ -1438,7 +1438,7 @@ def periodogram( Key for the temporal baseline in the STM. The value should be in decimal years. std_obs : float, optional - A-poriori standard deviation of the observations in rads, by default 1.0. + A-priori standard deviation of the observations in rads, by default 1.0. This value is used to construct the stochastic model (Qyy) of the observations. std_height : float, optional A-priori standard deviation of the height in meters, by default 50.0. diff --git a/depsi/constants.py b/depsi/constants.py index 6ec6a423..14c5abad 100644 --- a/depsi/constants.py +++ b/depsi/constants.py @@ -6,3 +6,6 @@ # Speed of light in vacuum SPEED_OF_LIGHT = 299792458.0 # m/s + +# Sentinel-1 wavelength +WAVELENGTH_S1 = 0.055465763 # m diff --git a/depsi/network.py b/depsi/network.py index 4ea7b92c..cfe12998 100644 --- a/depsi/network.py +++ b/depsi/network.py @@ -2007,3 +2007,48 @@ def _network_relation_matrix(idx_source, idx_target, n_points, idx_refpnt, spars A = np.array(A.todense()) return A + + +def _independent_arcs(arcs: np.ndarray) -> np.ndarray: + """Select independent arcs from a list of arcs. + + An arc is independent if its starting and ending points do not exist in any other arc's + starting or ending points. + + Parameters + ---------- + arcs : np.ndarray + A 2D array of shape (n_points, 2) where each row represents indices of the starting and ending points + of an arc. + + Returns + ------- + np.ndarray + A 2D array of independent arcs, where each row represents indices of the starting and ending points + of an arc. + """ + # Select arcs with unique starting points + _, unique_idx_start = np.unique(arcs[:, 0], return_index=True) + arcs = arcs[unique_idx_start, :] + + # Select arcs with unique ending points + _, unique_idx_end = np.unique(arcs[:, 1], return_index=True) + arcs = arcs[unique_idx_end, :] + + # After previous two steps, no arcs will share starting or ending points. + # However, there starting points may be the ending points of other arcs, and vice versa. + # To ensure independency, we loop through the rest arcs and add arc one by one + # In each interation, remove arcs that + # 1) start with the ending point of this arc, or + # 2) end with the starting point of this arc + arcs_selected = np.empty((0, 2), dtype=int) + while arcs.shape[0] > 0: + arc_current = arcs[0, :] + arcs_selected = np.append(arcs_selected, [arc_current], axis=0) + # Remove arcs which contain the starting point or ending point of the current arc + idx_remove = np.where((arcs[:, 1] == arc_current[0]) | (arcs[:, 0] == arc_current[1]))[0] + # add the index of the current arc to idx_remove + idx_remove = np.append(idx_remove, 0) + arcs = np.delete(arcs, idx_remove, axis=0) + + return arcs_selected diff --git a/stochastic.py b/depsi/stochastic.py similarity index 100% rename from stochastic.py rename to depsi/stochastic.py diff --git a/depsi/utils.py b/depsi/utils.py index b18b7eab..912f0028 100644 --- a/depsi/utils.py +++ b/depsi/utils.py @@ -31,7 +31,7 @@ import pytz import xarray as xr -from depsi.constants import EARTH_RADIUS +from depsi.constants import EARTH_RADIUS, WAVELENGTH_S1 logger = logging.getLogger(__name__) @@ -834,3 +834,8 @@ def concatenate_stms( stm_dens_pnts_output = stm_dens_pnts_output.reset_coords(names=coords_to_reset, drop=False) return stm_dens_pnts_output + + +def get_m2ph(wavelength: float = WAVELENGTH_S1): + """Get the conversion factor from meters to phase.""" + return -4 * np.pi / wavelength diff --git a/network.py b/network.py deleted file mode 100644 index cfe12998..00000000 --- a/network.py +++ /dev/null @@ -1,2054 +0,0 @@ -"""Module for creating networks from STM points.""" - -import logging -import math -from typing import Literal - -import matplotlib.pyplot as plt -import networkx as nx -import numpy as np -import scipy -import sparse -import xarray as xr -from scipy.spatial import Delaunay, KDTree, distance_matrix -from sklearn.neighbors import BallTree - -from depsi.arc_estimation import arc_estimation_control_network -from depsi.constants import EARTH_RADIUS -from depsi.stats import pretest -from depsi.utils import compute_phase_difference, get_distance - -logger = logging.getLogger(__name__) - - -# Constants for MHT in network integration -ALPHA0 = 0.1 # Significance level for 1-dimensional test -GAMMA0 = 0.5 # Power of the test -# Overall Model Test threshold for accepting the network -# In arc/point rejection phase, if OMT < OMT_THRES, stop rejection iteration -# In ambiguity fixing phase, if OMT < OMT_THRES, stop fixing iteration -# In arc/point rejection phase this is hardly triggered -OMT_THRES = 1e-10 -# Threshold for arc rejection statistics TT1 -# If for all arcs max(TT1) < TT1_THRES, stop rejection iteration -# For most cases this threshold is triggered in rejection phase -TT1_THRES = 1.0 - - -def spatial_integration( - stm_pnts: xr.Dataset, - stm_arcs: xr.Dataset, - key_sdphase: str = "sd_phase", - key_arc_quality: str = "temp_coh", - threshold_arc_quality: float = 0.5, - idx_refpnt: int | None = None, - min_arc_connections: int = 3, - largest_component_ratio: float = 0.8, - parallel: bool = False, - sparse_mode: bool = False, - ensure_network_while_mht: bool = False, - arc_estimation_method: Literal["periodogram"] = "periodogram", - skip_network_adaptation: bool = False, - max_iterations_adaptation: int = None, -) -> tuple[xr.Dataset, xr.Dataset]: - """Spatially integrate the ambiguities of network arcs to points. - - This function estimates the integer ambiguities of the points from arc ambiguities. It assumes a network - has been formed from the points by calling "network.form_network", and arc ambiguities have been estimated - by calling relevant functions in the "depsi.arc_estimation" module. - - The function returns an updated STM of arcs which contains the adjusted arc ambiguities after spatial integration, - as well as an updated STM of points which contains the estimated point ambiguities. - - The following steps are performed: - 1. Validate the network arcs (`stm_arcs`) against the point STM (`stm_pnts`) - 2. Select arcs based on quality threshold and ensure minimum connections for all points - 3. Select a reference point which assumes zero phase (hence zero ambiguity) - 4. Adjust the network by removing arcs/points which potentially cause errors using Multi-Hypothesis Testing (MHT) - 5. Adjust the ambiguities per time epoch to make sure spatial solutions give zero residuals. - 6. Calculate point ambiguities and unwrapped phases w.r.t. the reference point. - - Parameters - ---------- - stm_pnts : xr.Dataset - Space-Time Matrix of points. - stm_arcs : xr.Dataset - Space-Time Matrix of arcs. - The arcs should be formed from stm_pnts, using the "network.form_network" function. This function generates - coordinates "source" and "target" in stm_arcs which refer to the indices in stm_pnts. An "uid" data variable - is also generated by "form_network" for easy indexing. - An arc estimation should have been applied on stm_arcs before calling "spatial_integration". One can use - relevant functions in "depsi.arc_estimation" module for this purpose. Arc estimation adds the variable - "ambiguities" to stm_arcs, which are the estimated arc ambiguities. It also adds quality variables such as - "temp_coh" (ensemble coherence), which are used to select arcs for spatial integration. - key_sdphase : str, optional - Key of the single difference phase variable in stm_pnts, by default "sd_phase" - This phase is used to compute unwrapped phases after ambiguity estimation. - key_arc_quality : str, optional - Key of the arc quality variable in stm_arcs, by default "temp_coh" - threshold_arc_quality : float, optional - Threshold for arc quality, by default 0.5 - idx_refpnt : int | None, optional - Index of the reference point in stm_pnts. If None, the source point of the arc with highest quality is selected - as the reference point. - min_arc_connections : int, optional - Minimum number of connections for arcs, by default 3 - largest_component_ratio : float, optional - Threshold for determining the largest component when multiple components exist in the network, by default 0.8. - When removing arcs/points, it may happen that the network is split into multiple disconnected components. - In this case, only the largest component is kept and the others are discarded. - The largest component should contain at least this fraction of the total points, otherwise an error is raised. - Hence, when an error should be raised at all times, set this value to 1.0 . - parallel : bool, optional - Whether to use parallel processing, by default False - sparse_mode : bool, optional - Whether to use sparse matrix format for large networks, by default False - ensure_network_while_mht : bool, optional - Whether to ensure minimum connections in MHT network adaptation, by default False - arc_estimation_method : Literal["periodogram"], optional - Method used for arc estimation, by default "periodogram". - This constrains the method used for VCM computation. - skip_network_adaptation : bool, optional - Whether to skip network adaptation by MHT, by default False. - When enabling this option, it is recommended to set the threshold_arc_quality to a - high value (e.g. 0.75) to ensure only high-quality arcs are selected for spatial - integration. - max_iterations_adaptation : int, optional - Maximum number of iterations for network adaptation. - If None, the maximum number if iterations will be the number of arcs. - - Returns - ------- - xr.Dataset, xr.Dataset - Updated Space-Time Matrix of arcs and updated Space-Time Matrix of points. - For arcs, the "ambiguities" variable contains the adjusted arc ambiguities after spatial integration. - For points, the "ambiguities" variable contains the estimated point ambiguities, and "unwrapped_phase" - contains the unwrapped phase w.r.t. the reference point. - - References - ---------- - Van Leijen, F.J.. "Persistent scatterer interferometry based on geodetic estimation theory." (2014). - """ - # Check parallelization behavior - if parallel: - raise NotImplementedError("Dask support is not implemented yet for spatial_integration.") - else: - # Compute all data into memory - stm_pnts = stm_pnts.compute() - stm_arcs = stm_arcs.compute() - - # Validate that stm_arcs are formed from stm_pnts - if (stm_arcs["source"].max().values >= stm_pnts.sizes["space"]) or ( - stm_arcs["target"].max().values >= stm_pnts.sizes["space"] - ): - raise ValueError("stm_arcs contains source/target indices that exceed the number of points in stm_pnts.") - if "ambiguities" not in stm_arcs: - raise ValueError("stm_arcs does not contain 'ambiguities' variable. Please estimate arc ambiguities first.") - if key_arc_quality not in stm_arcs: - raise ValueError(f"stm_arcs does not contain '{key_arc_quality}' variable for arc quality assessment.") - - # Check arc estimation method, this constrains VCM computation method - if arc_estimation_method not in ["periodogram"]: - raise NotImplementedError(f"Unknown arc estimation method {arc_estimation_method}.") - - if sparse_mode: - raise NotImplementedError("Sparse mode is not implemented yet for spatial_integration.") - - # If idx_refpnt is specified - # Get radar coordinates of the reference point before any shape change - if idx_refpnt is not None: - azimuth_refpnt = stm_pnts["azimuth"].isel(space=idx_refpnt).values - range_refpnt = stm_pnts["range"].isel(space=idx_refpnt).values - - # Select arcs with quality > threshold_arc_quality - # Then ensure all points have at least min_arc_connections connections - mask = (np.abs(stm_arcs[key_arc_quality]) > threshold_arc_quality).compute() - stm_arcs = stm_arcs.where(mask, drop=True) - stm_arcs, stm_pnts = _ensure_network_min_connections(stm_arcs, stm_pnts, min_arc_connections) - - # Ensure the network is a single connected component after arc selection and point removal - stm_arcs, stm_pnts = _ensure_single_network(stm_arcs, stm_pnts, largest_component_ratio) - - # Select reference point as the source pnt of arcs with highest temp_coh - if idx_refpnt is None: - idx_arc_max_coh = stm_arcs[key_arc_quality].argmax().values - idx_refpnt = stm_arcs["source"].isel(space=idx_arc_max_coh).values - azimuth_refpnt = stm_pnts["azimuth"].isel(space=idx_refpnt).values - range_refpnt = stm_pnts["range"].isel(space=idx_refpnt).values - else: - # Make sure idx_refpnt is still valid after arc selection and point removal - mask_refpnt = (stm_pnts["azimuth"].values == azimuth_refpnt) & (stm_pnts["range"].values == range_refpnt) - if not np.any(mask_refpnt): - raise ValueError( - f"Reference point ({azimuth_refpnt}, {range_refpnt}) removed after arc selection. " - f"Please choose another reference point." - ) - - # Get reference point from radar coordinates - idx_refpnt = np.where(mask_refpnt)[0][0] - - # Adjust the network by removing bad arcs/points using MHT - if skip_network_adaptation: - logger.info("Skipping MHT network adaptation step.") - stm_arcs_adjusted, stm_pnts_adjusted = stm_arcs, stm_pnts - else: - stm_arcs_adjusted, stm_pnts_adjusted = _mht_network_adaptation( - stm_arcs, - stm_pnts, - idx_refpnt, - azimuth_refpnt, - range_refpnt, - ensure_network_while_mht, - sparse_mode, - arc_estimation_method, - max_iterations_adaptation, - largest_component_ratio, - ) - - # Update idx_refpnt after MHT adaptation - idx_refpnt = np.where( - (stm_pnts_adjusted["azimuth"].values == azimuth_refpnt) - & (stm_pnts_adjusted["range"].values == range_refpnt) - )[0][0] - - # Adjust ambiguities to fix unwrapping errors - stm_arcs_output, stm_pnts_output, idx_refpnt = _ambiguity_adaptation( - stm_arcs_adjusted, - stm_pnts_adjusted, - idx_refpnt, - sparse_mode, - arc_estimation_method, - ) - - # Assign idx_refpnt as attribute to stm_pnts_output - stm_pnts_output = stm_pnts_output.assign_attrs({"idx_refpnt": idx_refpnt}) - - # Add unwrapped phase to stm_pnts_output - # Unwrapped phase is w.r.t. the reference point - # Therefore the sd_phase of the reference point is subtracted - unwrapped_phase_pnts = ( - stm_pnts_output[key_sdphase].data - + stm_pnts_output["ambiguities"].data * 2 * np.pi - - np.tile( - stm_pnts_output[key_sdphase].isel(space=idx_refpnt).data, - (stm_pnts_output.sizes["space"], 1), - ) - ) - stm_pnts_output["unwrapped_phase"] = (("space", "time"), unwrapped_phase_pnts) - - return stm_arcs_output, stm_pnts_output - - -def form_network( - stm: xr.Dataset, - key_phase: str, - key_h2ph: str, - key_Btemporal: str, - key_complex: str = "complex", - key_xcrds: str = "lon", - key_ycrds: str = "lat", - network_method: Literal["redundant", "delaunay"] = "redundant", - max_length: float = None, - min_links: int = 16, - num_partitions: int = 8, - dphase_method: Literal["conjmult", "subtract"] = "subtract", -) -> xr.Dataset: - """Generate an STM of arcs from an STM of points. - - Parameters - ---------- - stm : xr.Dataset - Space-Time Matrix of scatterers. - key_phase : str - Key of the phase values in the STM. - This phase will be used to compute the differential arc phase. - key_h2ph : str - Key of the h2ph values in the STM. - The arc h2ph will be computed as the average between source and target. - key_Btemporal : str - Key of the temporal baseline values in the STM. - key_complex : str, optional - Key of the complex values, by default "complex" - key_xcrds : str, optional - Key of the x coordinates for calulating arc length, by default "lon" - key_ycrds : str, optional - Key of the y coordinates for calulating arc length, by default "lat" - network_method : Literal["redundant", "delaunay"], optional - network formation method, by default "redundant" - max_length : float, optional - maximum arc length, by default None - min_links : int, optional - minimum links per point, by default 16 - only effective when network_method is "redundant" - num_partitions : int, optional - number of partitions of searching when forming redundant network, by default 8 - only effective when network_method is "redundant" - dphase_method : Literal["conjmult", "subtract"], optional - method of computing phase difference, by default "subtract" - "subtract" method subtracts the source phase from the target phase (without re-wrapping); - "conjmult" method computes the phase difference by conjugate multiplication: - d_phase = np.angle(complex_target * complex_source.conj()) - - Returns - ------- - xr.Dataset - Space-Time Matrix of arcs, containing the following variables: - - d_phase: the arc phase, which is the difference between source and target points - - Btemp: the temporal baseline, which is the same for all arcs - - h2ph: the arc h2ph, which is the average between source and target points - """ - # Generate the network arcs. - if network_method == "redundant": - if min_links <= 0: - logger.error(f"min_links must be strictly positive (currently: {min_links})") - return - if num_partitions <= 0: - logger.error(f"num_partitions must be strictly positive (currently: {num_partitions})") - return - elif network_method != "delaunay": - raise NotImplementedError(f"Unknown network method {network_method}, known are delaunay and redundant") - - # Collect point coordinates. - indices = [stm[coord] for coord in [key_xcrds, key_ycrds]] - coordinates = np.column_stack(indices) - - arcs = None - - # Create network arcs as list of tuples of point ids. - if network_method == "delaunay": - arcs = _generate_arcs_delaunay(coordinates, max_length) - elif network_method == "redundant": - arcs = _generate_arcs_redundant(coordinates, max_length, min_links, num_partitions) - - # Compute the phase difference. - arcs_unzipped = list(zip(*arcs, strict=False)) - source_idx = list(arcs_unzipped[0]) - target_idx = list(arcs_unzipped[1]) - - if dphase_method not in ["conjmult", "subtract"]: - raise NotImplementedError(f"Unknown dphase_method '{dphase_method}'.") - dict_key_method = {"subtract": key_phase, "conjmult": key_complex} # mapping for selecting the correct key - d_phase = compute_phase_difference( - stm.isel(space=source_idx)[dict_key_method[dphase_method]].data, - stm.isel(space=target_idx)[dict_key_method[dphase_method]].data, - method=dphase_method, - ) - - # Temporal baseline - Btemp = stm[key_Btemporal].data - - # Height to phase factor - h2ph = (stm[key_h2ph].isel(space=source_idx).data + stm[key_h2ph].isel(space=target_idx).data) / 2 - - # Generate a unique identifier of arcs based on source and target for easy indexing - # This is because when updating network, points can be removed and reindexed - # Therefore we cannot use 2d index (source, target) as uid - # NOTE: This encoding is safe for typical networks (<10,000 points, <100,000 arcs). - # For 10,000 points, max UID ≈ 1e9, well within int64 max (2^63-1 ≈ 9.22e18), providing - # a safety margin of >9 billion times. Even for 1M points, safety margin is >900,000x. - scale = 10 ** (math.floor(math.log10(stm.sizes["space"])) + 1) # Scale to ensure no overlap - uid = scale * (np.array(source_idx) + 1) + (np.array(target_idx) + 1) # Plus one to avoid zero uid - uid = uid.astype(np.int64) - - arcs = xr.Dataset( - data_vars={ - "d_phase": (["space", "time"], d_phase), - "Btemp": (["time"], Btemp), - "h2ph": (["space", "time"], h2ph), - }, - coords={"source": (["space"], source_idx), "target": (["space"], target_idx), "uid": (["space"], uid)}, - attrs=stm.attrs, - ) - - return arcs - - -def _mht_network_adaptation( - stm_arcs: xr.Dataset, - stm_pnts: xr.Dataset, - idx_refpnt: int, - azimuth_refpnt: int | float, - range_refpnt: int | float, - ensure_network_while_mht: bool, - sparse_mode: bool, - arc_estimation_method: str, - max_iterations_adaptation: int, - largest_component_ratio: float, -) -> tuple[xr.Dataset, xr.Dataset]: - """Adjust the network by removing bad arcs/points by applying MHT. - - This function implements the Multi-Hypothesis Testing (MHT) approach iteratively to identify and remove - arcs and points to reduce the overall residual in ambiguity estimation. - - Parameters - ---------- - stm_arcs : xr.Dataset - Space-Time Matrix of arcs. - stm_pnts : xr.Dataset - Space-Time Matrix of points. - idx_refpnt : int - Index of the reference point in stm_pnts. - azimuth_refpnt : int | float - Azimuth coordinate of the reference point. - range_refpnt : int | float - Range coordinate of the reference point. - ensure_network_while_mht : bool - Whether to ensure minimum connections in MHT network adaptation. - sparse_mode : bool - Whether to use sparse matrix format for large networks. - arc_estimation_method : str - Method used for arc estimation. - max_iterations_adaptation : int - Maximum number of iterations for network adaptation. - largest_component_ratio : float - Threshold for determining the largest component when multiple components exist in the network. - - Returns - ------- - xr.Dataset, xr.Dataset - Updated Space-Time Matrix of arcs and points. - """ - # Setup functional and stochastic model - A = _network_relation_matrix( - stm_arcs["source"], stm_arcs["target"], stm_pnts.sizes["space"], idx_refpnt, sparse_mode - ) # Network relation matrix A - - if arc_estimation_method == "periodogram": - Qyy_diag = 1 - stm_arcs["temp_coh"].values - else: - raise NotImplementedError(f"arc_estimation_method '{arc_estimation_method}' is not supported.") - invQy = np.diag(1 / Qyy_diag) - - _, echeck, _ = _solve_float_ambiguities(A, stm_arcs["ambiguities"].data, invQy) # Estimate initial residual - OMT = np.diag(echeck.T @ invQy @ echeck).sum() # Test statistics for Overall Model Test - - # Setup test parameters - kb_dict = {} - max_con = np.abs(A).sum(axis=0).max() - for n_con in range(1, max_con + 1): - _, k1, kb, _ = pretest(n_con, ALPHA0, GAMMA0) - kb_dict[n_con] = kb - - # By default, set max_iterations_adaptation to the number of arcs - if max_iterations_adaptation is None: - max_iterations_adaptation = stm_arcs.sizes["space"] - - # Iteratively remove arcs/points until OMT and all arc statistics pass the test - stm_pnts_updated = stm_pnts.copy() - stm_arcs_updated = stm_arcs.copy() - TT1max = TT1_THRES + 1.0 # Initial TT1_max to trigger the while loop - niter = 0 - while (OMT >= OMT_THRES) and (TT1max >= TT1_THRES) and (niter < max_iterations_adaptation): - # The iteration stops when one of the following conditions is met: - # 1) overall model test pass: OMT < OMT_THRES (very rare case) - # 2) all arc test statistics smaller than threshold: max(TT1) < TT1_THRES (most common case) - # 3) maximum number of iterations reached (fail case) - logger.debug(f"MHT iteration {niter}: OMT={OMT:.2e}") - - # Because OMT failed, choose from two Ha: 1) remove an arc; 2) remove a point - # Decision is made based on flag_rm - flag_rm, idx_rm, TT1max, TTqmax = _mht_network_adaptation_reject_one( - A, stm_arcs_updated["ambiguities"].data, Qyy_diag, k1, kb_dict - ) - - if flag_rm == 0: # remove arc - logger.debug(f"MHT iteration {niter}: removing arc index {idx_rm} with TT1={TT1max:.2f}") - stm_arcs_updated = stm_arcs_updated.drop_isel(space=idx_rm) # Remove the arc - elif flag_rm == 1: # remove point - if idx_rm >= idx_refpnt: - idx_rm += 1 # Adjust index due to removed reference point column in A - - logger.debug(f"MHT iteration {niter}: removing point index {idx_rm} with TT1={TT1max:.2f}") - - # Removing points is achieved by removing all arcs connects to the point - # Later the points will be actually removed when ensuring minimum connections - # Arc indices connecting to the point to remove - idx_arcs_selected = np.where( - ((stm_arcs_updated["source"] != idx_rm) & (stm_arcs_updated["target"] != idx_rm)).data - )[0] - # Remove all arcs connects to the point to remove - stm_arcs_updated = stm_arcs_updated.isel(space=idx_arcs_selected) - - if ensure_network_while_mht: - min_connections_to_ensure = 3 # Ensure all points in the network have at least 3 connections - else: - min_connections_to_ensure = 2 # Just ensure all points are connected in the network - - # This makes sure all points can be tested in case of disagreement between arcs - stm_arcs_updated, stm_pnts_updated = _ensure_network_min_connections( - stm_arcs_updated, stm_pnts_updated, min_connections=min_connections_to_ensure - ) - - # Ensure the network is a single connected component after arc/point removal - stm_arcs_updated, stm_pnts_updated = _ensure_single_network( - stm_arcs_updated, stm_pnts_updated, largest_component_ratio - ) - - # Make sure the reference point is still in stm_pnts_updated, by checking its azimuth and range - mask_refpnt = (stm_pnts_updated["azimuth"].values == azimuth_refpnt) & ( - stm_pnts_updated["range"].values == range_refpnt - ) - if not np.any(mask_refpnt): - raise ValueError( - f"Reference point ({azimuth_refpnt}, {range_refpnt}) removed in the MHT process. " - f"Please choose another reference point." - ) - idx_refpnt = np.where(mask_refpnt)[0][0] # Update idx_refpnt - - # Get indices of selected arcs based on uid - if arc_estimation_method == "periodogram": - Qyy_diag = 1 - stm_arcs_updated["temp_coh"].values # VCM diagonal from temporal coherence - invQy = np.diag(1 / Qyy_diag) - - A = _network_relation_matrix( - stm_arcs_updated["source"], - stm_arcs_updated["target"], - stm_pnts_updated.sizes["space"], - idx_refpnt, - sparse_mode, - ) # Update A matrix - _, echeck, _ = _solve_float_ambiguities( - A, stm_arcs_updated["ambiguities"].data, invQy - ) # Estimate residual again - OMT = np.diag(echeck.T @ invQy @ echeck).sum() # Update OMT statistic - - niter += 1 - - if niter >= stm_arcs.sizes["space"]: - raise RuntimeError( - "Maximum number of iterations reached in MHT network adaptation. " - "The network may still contain bad arcs or points." - ) - - return stm_arcs_updated, stm_pnts_updated - - -def _mht_network_adaptation_reject_one( - A: np.ndarray | scipy.sparse.spmatrix, - y: np.ndarray, - Qyy_diag: np.ndarray, - k1: float, - kb_dict: dict, -) -> tuple[int, int, float, float]: - """Remove one point/arc from the network to reduce the residual in ambiguity estimation.""" - # Retrieve shapes - N_arcs, N_epochs = y.shape - N_points = A.shape[1] - - # Inverse of VCM of observations - if Qyy_diag.ndim == 1: # Diagonal VCM - invQy = np.diag(1 / Qyy_diag) - Qyy = np.diag(Qyy_diag) - else: - raise NotImplementedError("Currently only diagonal VCM is supported. Qyy_diag should be a 1d array.") - - # Solve ambiguities as float - _, echeck, invAtQyA = _solve_float_ambiguities(A, y, invQy) - - # Post-priori VCM of residuals - Qecheck = Qyy - (A @ invAtQyA @ A.T) # TODO: check how to handle large Qecheck - - # Test statistics TT1 per arc - Qecheck_diag = np.array(np.diag(Qecheck).flatten()).squeeze() - w = echeck**2 / np.tile(np.abs(Qecheck_diag), (N_epochs, 1)).T - TT1 = np.sum(w, axis=1) / k1**2 - TT1max = max(TT1) - - # Test statistics per point - # Build arc-point connectivity mask and - connected_mask = A != 0 - # Drop one connected arc per point to create the basis (see e.g. verhoef97). - has_connection = connected_mask.any(axis=0) - first_connected_idx = np.argmax(connected_mask, axis=0) - selected_mask = connected_mask.copy() - selected_mask[first_connected_idx[has_connection], np.where(has_connection)[0]] = False - # Tq for point q: sum_i(sum_t(e_i,t^2) / Qe_i) over selected arcs i connected to point q. - e2_sum = np.sum(echeck**2, axis=1) - e2_sum_weighted = e2_sum / np.abs(Qecheck_diag) - Tq_num = selected_mask.T @ e2_sum_weighted - # Find where to calculate TTq based on connectivity - narcs_connected = selected_mask.sum(axis=0).astype(int) - kb_vals = np.array([kb_dict.get(d, np.nan) for d in narcs_connected], dtype=float) - valid = (narcs_connected > 0) & np.isfinite(kb_vals) & (kb_vals != 0) - # Calculate TTq for points with valid kb values - # assign -inf to invalid ones to make sure they won't be selected for removal - TTq = np.full(N_points, -np.inf, dtype=float) - with np.errstate(divide="ignore", invalid="ignore"): - TTq[valid] = Tq_num[valid] / kb_vals[valid] - TTqmax = max(TTq) - - # Decision one removal strategy - if TT1max > TTqmax: - idx_removal = np.argmax(TT1) # index of arc to remove - flag_removal = 0 # remove arc - else: - idx_removal = np.argmax(TTq) # index of point to remove - flag_removal = 1 # remove point - - return flag_removal, idx_removal, TT1max, TTqmax - - -def _ambiguity_adaptation( - stm_arcs: xr.Dataset, - stm_pnts: xr.Dataset, - idx_refpnt: int, - sparse_mode: bool, - arc_estimation_method: str, -) -> tuple[xr.Dataset, xr.Dataset, int]: - """Fix unwrapping errors by adjusting ambiguities per epoch. - - This function iterates over each epoch and adjusts the ambiguities to make sure the spatial - solutions of ambiguities gives zero residuals for each epoch. - - Parameters - ---------- - stm_arcs : xr.Dataset - Space-Time Matrix of arcs. - stm_pnts : xr.Dataset - Space-Time Matrix of points. - idx_refpnt : int - Index of the reference point. - sparse_mode : bool - Whether to use sparse matrix format for large networks. - arc_estimation_method : str - Method used for arc estimation. - - Returns - ------- - (xr.Dataset, xr.Dataset) - Updated Space-Time Matrices of arcs and points. - For arcs, the "ambiguities" variable contains the adjusted arc ambiguities. - For points, the "ambiguities" variable are estimated from the adjusted arc ambiguities. - """ - # Setup functional and stochastic model - A = _network_relation_matrix( - stm_arcs["source"], stm_arcs["target"], stm_pnts.sizes["space"], idx_refpnt, sparse_mode - ) - if arc_estimation_method == "periodogram": - Qyy_diag = 1 - stm_arcs["temp_coh"].values # VCM diagonal from temporal coherence - else: - raise NotImplementedError(f"arc_estimation_method '{arc_estimation_method}' is not supported in form_network.") - invQy = np.diag(1 / Qyy_diag) - - # Initialize adjusted ambiguities storage, shape: (n_points-1, n_epochs) - # Space dimension is n_points-1 because reference point is excluded - acheck = np.zeros((stm_pnts.sizes["space"] - 1, stm_pnts.sizes["time"])) - - # Fix unwrapping ambiguities by looping over epochs - stm_arcs_updated = stm_arcs.copy() - stm_pnts_updated = stm_pnts.copy() - for epoch in range(stm_pnts.sizes["time"]): - logger.debug(f"Adjusting ambiguities for epoch {epoch}") - y = stm_arcs["ambiguities"].isel(time=epoch).data - acheck_ifg, echeck_ifg, _ = _solve_float_ambiguities(A, y, invQy) - OMT = echeck_ifg.T @ invQy @ echeck_ifg - idx_previous_arc_fix = -1 # Avoid fixing the same arc again in the same epoch - - while OMT >= OMT_THRES: # While the OMT fails, fix for this epoch - # Find arc index with largest abs echeck - # When OMT > kOMT, echeck_ifg[idx_max_echeck] is guaranteed to be non-zero - idx_sort = np.argsort(np.abs(echeck_ifg))[::-1] # Indices of echeck sorted by abs value, descending - idx_max_echeck = idx_sort[0] # Index of arc with largest abs echeck - if idx_max_echeck == idx_previous_arc_fix: - # If get same arc as previous fix, take the second largest - idx_max_echeck = idx_sort[1] - - if np.round(abs(echeck_ifg[idx_max_echeck])) >= 1: # If >= 1, minus closest integer - y[idx_max_echeck] -= np.round(echeck_ifg[idx_max_echeck]) - elif echeck_ifg[idx_max_echeck] > 0: # if (0, 1), minus 1 - y[idx_max_echeck] -= 1.0 - elif echeck_ifg[idx_max_echeck] < 0: # if (-1, 0), plus 1 - y[idx_max_echeck] += 1.0 - - idx_previous_arc_fix = idx_max_echeck # record the fixed arc index - - # Recalculate OMT - acheck_ifg, echeck_ifg, _ = _solve_float_ambiguities(A, y, invQy) - OMT = echeck_ifg.T @ invQy @ echeck_ifg - - logger.debug(f"Fixing arc index {idx_max_echeck}, new OMT={OMT:.2e}") - - stm_arcs_updated["ambiguities"][:, epoch] = y # Store adjusted arc ambiguities - acheck[:, epoch] = acheck_ifg # Store adjusted point ambiguities - - # Round acheck to closest integer - acheck = np.round(acheck).astype(np.int16) - - # Assign reference point ambiguities as zero - acheck_full = np.zeros((stm_pnts.sizes["space"], stm_pnts.sizes["time"])).astype(np.int16) - acheck_full[np.arange(stm_pnts.sizes["space"]) != idx_refpnt, :] = acheck - - # Assign acheck_full to stm_pnts_updated - stm_pnts_updated["ambiguities"] = (["space", "time"], acheck_full) - - return stm_arcs_updated, stm_pnts_updated, idx_refpnt - - -def _ensure_network_min_connections( - stm_arcs: xr.Dataset, - stm_pnts: xr.Dataset, - min_connections: int, -) -> tuple[xr.Dataset, xr.Dataset]: - """Ensure that all points in the network have at least min_connections arcs. - - This is achieved by an iterative process of removing points which have less than - min_connections arcs, and removing all arcs connected to these points. - The process is repeated until no more points are removed. - - Parameters - ---------- - stm_arcs : xr.Dataset - Space-Time Matrix of arcs. - stm_pnts : xr.Dataset - Space-Time Matrix of points. - min_connections : int - Minimum number of connections for each point. - - Returns - ------- - xr.Dataset, xr.Dataset - Updated Space-Time Matrix of arcs and points. - """ - # Ensure all points have at least min_connections connections - previous_size = -1 # Initialize with an impossible value to trigger the while loop - # Keep iterating until no more points are removed - while stm_pnts.sizes["space"] != previous_size: - previous_size = stm_pnts.sizes["space"] - # Remove points with < min_connections connections - stm_pnts, stm_arcs = _remove_network_points_min_connections(stm_pnts, stm_arcs, min_connections) - - return stm_arcs, stm_pnts - - -def _ensure_single_network( - stm_arcs: xr.Dataset, stm_pnts: xr.Dataset, largest_component_ratio: float -) -> tuple[xr.Dataset, xr.Dataset]: - """Ensure the network is connected and discard the smaller disconnected sub-network(s). - - This function utilizes the NetworkX library to identify connected components in the network - formed by stm_arcs and stm_pnts. When building the graph, the point indices are used as node identifiers, - and the "source" and "target" coordinates in stm_arcs are used to add edges between the corresponding nodes. - Note that "source" and "target" coordinates in stm_arcs are indices of the points STM stm_pnts, - but not necessarily the same as the "space" coordinate of stm_pnts. - - If there are multiple connected components, only the largest one is kept and the others are discarded. - However, if the largest component is smaller than a certain ratio (largest_component_ratio) of the total - points, an error is raised. - """ - G = nx.Graph() - G.add_nodes_from(np.arange(stm_pnts.sizes["space"])) # Use point indices as node identifiers - G.add_edges_from( - zip( - stm_arcs["source"].values, - stm_arcs["target"].values, - strict=True, - ) - ) - - # Get list of connected components - list_components = [cc for cc in nx.connected_components(G)] - - # If there are multiple connected components, keep only the largest one and discard the others - if len(list_components) > 1: - nodes_largest = max(list_components, key=len) # set of node indices in the largest connected component - - # Check if the largest component is significantly larger than the second largest one - if (len(nodes_largest) / stm_pnts.sizes["space"]) < largest_component_ratio: - raise RuntimeError( - f"The largest connected component contains only {len(nodes_largest)} points, which is less than " - f"{largest_component_ratio * 100:.1f}% of the total {stm_pnts.sizes['space']} points. " - "In this case DePSI cannot automatically decide which component to keep. " - "This may indicate a problem with the network formation. " - "Please check the input data and parameters." - ) - - mask_arcs = xr.DataArray( - np.isin(stm_arcs["source"].values, list(nodes_largest)) - & np.isin(stm_arcs["target"].values, list(nodes_largest)), - dims=["space"], - ) - stm_arcs_output = stm_arcs.where(mask_arcs, drop=True) - stm_pnts_output = stm_pnts.isel(space=list(nodes_largest)) - - # Update the source and target indices in stm_arcs_output to match the new stm_pnts_output - idx_map = {old_idx: new_idx for new_idx, old_idx in enumerate(sorted(nodes_largest))} - stm_arcs_output_updated = stm_arcs_output.copy() - stm_arcs_output_updated["source"] = xr.DataArray( - np.vectorize(idx_map.get)(stm_arcs_output["source"].values), dims="space" - ) - stm_arcs_output_updated["target"] = xr.DataArray( - np.vectorize(idx_map.get)(stm_arcs_output["target"].values), dims="space" - ) - stm_arcs_output = stm_arcs_output_updated - - n_components = len(list_components) - logger.info("Separated components detected in the network!") - logger.info(f"Network has {n_components} connected components.") - logger.info( - f"Keeping only the largest component with {stm_pnts_output.sizes['space']} points " - f"and {stm_arcs_output.sizes['space']} arcs." - ) - logger.info( - f"Discarded {stm_pnts.sizes['space'] - stm_pnts_output.sizes['space']} points " - f"and {stm_arcs.sizes['space'] - stm_arcs_output.sizes['space']} arcs." - ) - else: - stm_arcs_output = stm_arcs - stm_pnts_output = stm_pnts - - return stm_arcs_output, stm_pnts_output - - -def _solve_float_ambiguities(A, y, invQy, sparse_mode: bool = False): - """Solve ambiguities as a float based on Least-Squares.""" - # Solve ambiguities as they are float numbers - # This solves the equation y = Ax in least-squares sense - # With A a sparse matrix - # And stochastic model Qyy taken into account - invQyA = invQy @ A # Avoid repeated computation in vectorized lsmr - invAtQyA = np.linalg.inv(A.T @ invQyA) - - if sparse_mode: - - @np.vectorize(signature="(i)->(j)") - def lsmr(y): - """Least-squares iterative solver for sparse data.""" - x, *_ = scipy.sparse.linalg.lsmr(invQyA, invQy @ y) - return x - - acheck = lsmr(y.T).T # float ambiguity estimation - else: - acheck = invAtQyA @ (A.T @ invQy @ y) - echeck = y - A @ acheck # residuals estimation - - return acheck, echeck, invAtQyA - - -def _remove_network_points_min_connections( - stm: xr.Dataset, arcs: xr.Dataset, min_connections: int -) -> tuple[xr.Dataset, xr.Dataset]: - """Remove points which have less than min_connections arc connections. - - The following steps are performed: - - 1. Remove points from stm which have less than min_connections connections in arcs. - 2. Remove arcs which connect to the removed points. - 3. Update the space indices in points/arcs STM accordingly. - The point indices are always a 0-based continuous array. - - Note that this function does not perform iterative removal to assure that all points have - at least min_connections connections, but only performs one round of removal. - """ - if min_connections < 1: - raise ValueError("min_connections must be at least 1") - - # Load source and target indices from arcs - # these are 1d arrays so should fit in memory - idx_source = arcs["source"].values - idx_target = arcs["target"].values - - # Select STM points that are in arcs - # Only keep points which ids are in arcs, isolated points are removed in idx_selected - idx_selected, counts = np.unique(np.concatenate([idx_source, idx_target]), return_counts=True) - idx_selected = idx_selected[counts >= min_connections] # only keep points with at least min_connections connections - - # If no change, return directly - if len(idx_selected) == stm.sizes["space"]: - return stm, arcs - - # Select points - stm_pnts_updated = stm.isel(space=idx_selected) - - # Select arcs that connect selected points (Some arcs may be dropped together with points) - mask_source = np.isin(idx_source, idx_selected) - mask_target = np.isin(idx_target, idx_selected) - mask_arcs = mask_source & mask_target - arcs = arcs.isel(space=np.where(mask_arcs)[0]) - - # The space size of the STM changes, resulting non-contiguous indices in space dimension - # hence an update in arcs space coordinates is needed - # Here we use a mapping solution, since the maximum number of network points is usually <100k - # Map old indices in arcs to new indices - idx_map = {old_idx: new_idx for new_idx, old_idx in enumerate(idx_selected)} - # apply the mapping to the source and target indices in arcs - arcs_updated = arcs.copy() - arcs_updated["source"] = xr.DataArray(np.vectorize(idx_map.get)(arcs["source"].values), dims="space") - arcs_updated["target"] = xr.DataArray(np.vectorize(idx_map.get)(arcs["target"].values), dims="space") - - return stm_pnts_updated, arcs_updated - - -def _generate_arcs_delaunay(coordinates, max_length): - """Create a network using Delaunay triangulation.""" - # Create network and collect neighbors. - network = Delaunay(coordinates) - neighbors_ptr, neighbors_idx = network.vertex_neighbor_vertices - - # Convert ptr and idx arrays into list of sorted index pairs. - arcs = [] - for s in range(len(neighbors_ptr) - 1): - for t in range(neighbors_ptr[s], neighbors_ptr[s + 1]): - length = get_distance(coordinates[int(s)], coordinates[neighbors_idx[t]], mode="euclidean") - if max_length is None or length <= max_length: - arcs.append(tuple(sorted([int(s), int(neighbors_idx[t])]))) - - # Remove duplicates and make the list canonical. - arcs = sorted(list(set(arcs))) - - return arcs - - -def _generate_arcs_redundant(coordinates, max_length, min_links, num_partitions): - """Create a network with at least min_links arcs per node. - - The redundant network is formed with the following steps: - - 1. Create a KDTree and find all pairs of points within the maximum distance. - 2. Loop through each point and find its neighbors within the maximum distance. - 3. Divide neighbors into partitions based on their direction. - 4. Select the nth nearest neighbors from all partitions, starting from n=1. - 5. Sort the selected neighbors by distance, add them to the arcs list. If min_links is not - exceeded, continue to the n+1th nearest neighbors of all partitions. - 6. Repeat until min_links is reached. - """ - arcs = [] - indices = range(len(coordinates)) - - # Create a KDTree for fast distance queries. - tree = KDTree(coordinates) - if max_length is not None: - pairs = tree.query_pairs(r=max_length, output_type="ndarray") - else: - pairs = tree.query_pairs(r=np.inf, output_type="ndarray") - - # Duplicate pairs with reversed indices to ensure that arcs are undirected. - pairs = np.concatenate((pairs, np.flip(pairs, axis=1)), axis=0) - pairs = pairs[np.argsort(pairs[:, 0])] # Sort pairs by first column (source index). - - for cur_index in indices: - # Get the neighbors of the current node - neighbors = pairs[pairs[:, 0] == cur_index][:, 1].tolist() - - if len(neighbors) == 0: # skip if there are no neighbors - continue - elif len(neighbors) <= min_links: - # If there are not enough neighbors, connect them all. - for idx in neighbors: - arc_to_add = (min(cur_index, idx), max(cur_index, idx)) - arcs.append(arc_to_add) - else: - # Calculate partitions and distances for neighbors - partitions = [ - int(math.floor(num_partitions * (0.5 + math.atan2(coordinate[1], coordinate[0]) / math.tau))) - for coordinate in coordinates[neighbors] - coordinates[cur_index] - ] - distances = [math.dist(coordinates[cur_index], coordinates[idx]) for idx in neighbors] - - # Create sorted array by partition and then distance - sorted_arr = np.array(sorted(list(zip(partitions, distances, neighbors, strict=False)))) - - # Split into partitions - partitions_diff = sorted_arr[1:, 0] - sorted_arr[:-1, 0] - separators = np.where(partitions_diff > 0)[0] - partitions_split = np.split(sorted_arr, separators + 1) - partitions_split = [partition[:min_links] for partition in partitions_split] - - # Collect the neighbor 'hierarchies' - neighbor_hierarchies = [[] for _ in range(min_links)] - count = 0 - for n in range(min_links): - # Break early if we have gathered enough neighbors. - if min_links <= count: - break - for partition in partitions_split: - # Note that we do not break inside this loop, - # because we want the nth nearest neighbors from all partitions. - if n < len(partition) and (max_length is None or partition[n][1] <= max_length): - neighbor_hierarchies[n].append(partition[n]) - count = count + 1 - - # Sort hierarchies per partition by distance to the current node - neighbor_hierarchies = [ - sorted(hierarchy, key=lambda x: x[1]) for hierarchy in neighbor_hierarchies if len(hierarchy) != 0 - ] - - # Add sorted arcs to at least min_links neighbors - cur_arcs = [ - (min(cur_index, int(neighbor[2])), max(cur_index, int(neighbor[2]))) - for hierarchy in neighbor_hierarchies - for neighbor in hierarchy - ] - cur_arcs = cur_arcs[:min_links] - - arcs.extend(cur_arcs) - - # Remove duplicates and make the list canonical. - arcs = sorted(list(set(arcs))) - - return arcs - - -def get_ordered_arcs( - x_ref_search, - y_ref_search, - buffer_radius_ref, - dist_to_quality, - n_max_arcs, - nad_nmad_max, - x_coordinates, - y_coordinates, - slc_quality, - nad_nmad_vals, - coordinate_type, -): - """Get a list with ordered arcs based on pnt quality and a search area. - - Args: - ---- - x_ref_search (float): x-coordinate of the centre of the search area - y_ref_search (float): y-coordinate of the centre of the search area - buffer_radius_ref (float): The buffer (in m) around the centre coordinates where potential arcs are computed - dist_to_quality (float): parameter that relates arc length to additional sigma - n_max_arcs (float): The maximum nr of arcs to be outputed - nad_nmad_max (float): The maximum NAD / NMAD, for the entire time series, for a point to be considered - x_coordinates (xr.DataArray): x-coordinates of all points in the STM - y_coordinates (xr.DataArray): y-coordinates of all points in the STM - slc_quality (xr.DataArray): quality of the estimates per partition - nad_nmad_vals (xr.DataArray): NAD / NMAD of the entire time series per point - coordinate_type: 'Euclidean' (RD) / 'geographic' (lon/lat) - - Returns: - ------- - arcs (list): ordered arcs - arcs_and_quality (list): ordered list with the points and quality - quality_dict (dictionary): dictionary with the arcs and their quality - """ - # Find all points within the buffer around the starting location x, y - idx_pnts_buffer_ref = find_points_within_buffer( - x_coordinates, y_coordinates, x_ref_search, y_ref_search, buffer_radius_ref, coordinate_type - ) - - # Get the a-priori quality of all potential arcs that can be made - arcs_and_quality = _ordered_arcs_all_points( - x_coordinates, - y_coordinates, - slc_quality, - idx_pnts_buffer_ref, - dist_to_quality, - nad_nmad_vals, - nad_max=nad_nmad_max, - ) - - # We will only work with n_max_arcs, otherwise we need to load an extensive dataset everytime - arcs_and_quality = arcs_and_quality[0:n_max_arcs] - - # Only store the arcs, and remove the quality values - arcs = [] - for _, arc in arcs_and_quality: - arcs.append(arc) - - # Create a dictionary with the quality of the arcs - quality_dict = {tuple(sorted(arc[1])): arc[0] for arc in arcs_and_quality} - - return arcs, arcs_and_quality, quality_dict - - -def find_points_within_buffer( - x_coords, - y_coords, - x_pnts, - y_pnts, - buffer_radius, - coordinate_type: Literal["euclidean", "geographic"] = "euclidean", - return_aggregate_point_buffer: bool = True, -): - """Find all points located within a specified buffer radius around a given location. - - This function determines which points in a set of coordinates are located within a defined buffer - distance around one or more specified points. - - The function: - 1. Converts the input x and y coordinates of the search point(s) to arrays if they are not already. - 2. Computes the distance between each point in `x_coords` and `y_coords` and the reference points. - 3. Returns the indices of these points for further processing or analysis. - - Args: - ---- - x_coords (numpy.ndarray): Array of x-coordinates for all points in the dataset. - y_coords (numpy.ndarray): Array of y-coordinates for all points in the dataset. - x_pnts (float or array-like): The x-coordinate or array of x-coordinates of the reference point(s). - y_pnts (float or array-like): The y-coordinate or array of y-coordinates of the reference point(s). - buffer_radius (float): The radius of the buffer zone around the reference points, specified in meters. - coordinate_type (Literal["euclidean", "geographic"]): whether the coordinates provided are Euclidean (such as - RD) or geographic (such as lon / lat) - return_aggregate_point_buffer (bool): if True, a list of indices of the points within the buffer of any of the - points in x_pnts/y_pnts is returned. If False, the list of indices within the buffer of each point in - x_pnts/y_pnts is returned individually. Default True - - Returns: - ------- - indices (numpy.ndarray): Array of indices of the points located within the buffer zone, either the aggregate - buffer zone (`return_aggregate_point_buffer`=True) or per point in `x_pnts`/`y_pnts` - (`return_aggregate_point_buffer`=False) - - Example: - indices = find_points_within_buffer(x_coords, y_coords, x_pnts=10.5, y_pnts=20.3, buffer_radius=5.0) - (with Euclidean coordinates and an aggregate buffer output) - """ - # Make sure x_pnts and y_pnts are arrays - x_pnts = np.atleast_1d(x_pnts) - y_pnts = np.atleast_1d(y_pnts) - assert len(x_pnts.shape) == 1, f"Received more than 1 dimension in x_pnts (shape {x_pnts.shape})!" - assert len(y_pnts.shape) == 1, f"Received more than 1 dimension in y_pnts (shape {y_pnts.shape})!" - - if coordinate_type == "euclidean": - # could possibly also be a KDTree but for consistency inside the function the minkowski metric does Euclidean - tree = BallTree(np.vstack([x_coords, y_coords]).T, metric="minkowski") - search_radius = buffer_radius - search_points = np.vstack([x_pnts, y_pnts]).T - elif coordinate_type == "geographic": - # the geographic tree assumes an Earth radius of 1, so we need to divide the search radius by the radius of - # the Earth - # It also expects input in radians instead of degrees, and first latitude (y), then longitude (x) - tree = BallTree(np.vstack([np.radians(y_coords), np.radians(x_coords)]).T, metric="haversine") - search_radius = buffer_radius / EARTH_RADIUS - search_points = np.vstack([np.radians(y_pnts), np.radians(x_pnts)]).T - else: - raise ValueError(f"Unknown coordinate type {coordinate_type}! Known are euclidean and geographic.") - - res_indices = tree.query_radius(search_points, search_radius) - - if not return_aggregate_point_buffer: - return res_indices - - all_indices = [] - for i in res_indices: # flatten won't work since res_indices is an irregularly shaped numpy array - for j in i: - all_indices.append(j) - indices = np.array(list(sorted(list(set(all_indices))))) - - return indices - - -def _ordered_arcs_all_points(rdx, rdy, slc_quality, idx_pnts_buffer, dist_to_quality, nad_vals, nad_max): - """Generate a sorted list of unique arcs between all points within buffer, ranked by combined quality and distance. - - This function computes arcs between all points in a given buffer based on their spatial distance and - quality metrics. The arcs are ranked according to a combination of the Euclidean distance between - points and the maximum quality value between them. - - The function: - 1. Computes the Euclidean distance matrix for all points within the buffer. - 2. Calculates the arc quality time series for all point pairs based on their SLC quality values. - 3. Extracts the maximum quality for each arc and combines it with the distance between the points. - 4. Uses the lower triangular matrix to avoid duplicate arcs - 5. Sorts the arcs based on the combined distance and quality metric. - - Args: - ---- - rdx (numpy.ndarray): Array of x-coordinates for all points. - rdy (numpy.ndarray): Array of y-coordinates for all points. - slc_quality (numpy.ndarray): Array of SLC quality time series for all points. - idx_pnts_buffer (numpy.ndarray): Indices of the points within the buffer. - dist_to_quality (float): Scaling factor for weighting the distance in combination with the quality. - nad_vals (numpy.ndarray): Array of NAD values of all points. - nad_max (float): The maximum NAD, for the entire time series, for a point to be considered - - Returns: - ------- - arcs_and_quality (list): Sorted list of tuples, each tuple contains the combined quality and distance value - and the indices of the two points forming the arc. - - Example: - ------- - arcs_and_quality = ordered_arcs_all_points(rdx, rdy, slc_quality, idx_pnts_buffer, dist_to_quality) - """ - # Compute NAD values of the points within the buffer - nad_buffer_vals = nad_vals[idx_pnts_buffer] - - # Mask points that have an NAD value above the threshold - mask_nad = nad_buffer_vals < nad_max - idx_pnts_buffer = idx_pnts_buffer[mask_nad] - - # Compute the distance matrix for all points (below the NAD threshold) within the buffer - rdx_buffer = rdx[idx_pnts_buffer] - rdy_buffer = rdy[idx_pnts_buffer] - coords_points = np.vstack((rdx_buffer, rdy_buffer)).T - dist_matrix = distance_matrix(coords_points, coords_points) - - # Compute the quality matrix for all potential arcs within the buffer - slc_quality_buffer = slc_quality[idx_pnts_buffer, :] - slc_quality_i_buffer = slc_quality_buffer[:, np.newaxis] - slc_quality_j_buffer = slc_quality_buffer[np.newaxis, :] - - # Compute the arc quality time series - # quality is based on both sides of the arc - arc_quality_ts = np.sqrt(slc_quality_i_buffer**2 + slc_quality_j_buffer**2) - - # Get the maximum value in the time dimension per arc - # This is considered as the 'worst' quality for the entire time period - arcs_quality_max = np.max(arc_quality_ts, axis=2) - - # Get only the lower triangular matrix - arcs_quality_max_lower = np.tril(arcs_quality_max) - arcs_dist = np.tril(dist_matrix) - - # Add additional sigma because of the arc length - arc_quality_dist_max = arcs_quality_max_lower + arcs_dist * dist_to_quality - - # Order the arcs - arcs_and_quality = [] - - # Loop trough the lower tringular matrix and add values list - rows, cols = arc_quality_dist_max.shape - for i in range(rows): - for j in range(i): # j < i makes sure that we only get the lower triangular matrix - arcs_and_quality.append((arc_quality_dist_max[i, j], (idx_pnts_buffer[i], idx_pnts_buffer[j]))) - - # Sort the list with all the arcs from best to worse - arcs_and_quality.sort() - - return arcs_and_quality - - -def construct_control_network( - arcs, - quality_dict_arcs, - excluded_arcs, - n_top, - n_batch, - deg_threshold, - min_nodes, - min_redundancy, - visualize_network=False, -): - """Construct a control network of arcs based on quality metrics and structural requirements. - - This function iteratively builds a network from a search area of arcs, filtering out excluded arcs - (where an excluded arc is an arc we know we don't want to have) and ensuring that the resulting - network meets specified criteria, such as a minimum number of nodes and redundancy. - The network is constructed by: - 1. Ranking arcs based on quality metrics provided in `quality_dict_arcs`. - 2. Excluding arcs that are in the `excluded_arcs` list. - 3. Iteratively adding arcs in batches and refining the network to remove nodes with low centrality. - 4. Evaluating the network against requirements such as minimum nodes and average redundancy. - - If the requirements are not met with the given arcs, a warning is issued. Optionally, the network - construction process can be visualized. - - Args: - ---- - arcs (list of tuples): List of arcs (pairs of points) within the search area. - quality_dict_arcs (dict): Dictionary mapping arcs (tuples) to a sigma value. - excluded_arcs (list of tuples): List of arcs that should be excluded from the network. - n_top (int): Number of top-ranked arcs to start with. - n_batch (int): Number of additional arcs to add in each iteration. - deg_threshold (float): Degree threshold for removing low-centrality nodes from the network. - min_nodes (int): Minimum number of nodes required in the final network. - min_redundancy (float): Minimum average redundancy (degree) required in the final network. - visualize_network (bolean, optional): If set to True, the network construction process is visualized. - - Returns: - ------- - ref_pnt (int): The reference point used in the final iteration of the network. - arcs_updated_network_sorted (list of tuples): The sorted list of arcs in the final network - based on quality metrics. - ref_pnt_initial (int): The reference point used in the initial network. - arcs_initial_network (list of tuples): The list of arcs in the network before refinement. - - Example: - ------- - ref_pnt, final_arcs, initial_ref_pnt, initial_arcs = construct_control_network( - arcs, quality_dict_arcs, excluded_arcs, - n_top=50, n_batch=10, - deg_threshold=1.5, min_nodes=20, min_redundancy=2.0, - visualize_network=False - ) - """ - # Sort arcs by quality, removing excluded arcs from the ranked arcs variable - excluded_arcs_sorted = [tuple(sorted(arc)) for arc in excluded_arcs] - arcs_without_excluded = [arc for arc in arcs if tuple(sorted(arc)) not in excluded_arcs_sorted] - - # Initialize variables - current_network = None - avg_degree = 0 - iteration = 0 - - more_arcs_to_test = True - network_reqs_not_met = True - - # the following while loop will stop when either the network requirements (# nodes and redundancy level) are met, - # OR when we run out of arcs to text - while more_arcs_to_test and network_reqs_not_met: - # Select the arcs for the current iteration - iteration += 1 - end_idx = n_top + iteration * n_batch - arcs_to_test = arcs_without_excluded[0:end_idx] - - if not arcs_to_test: # Break if no more arcs to add - print("No more arcs to test.") - more_arcs_to_test = False - continue - - # Create a new network using the selected arcs - current_network = nx.Graph() - current_network, _, ref_pnt = _from_arcs_to_graph(arcs_without_excluded[:end_idx], plot=visualize_network) - if iteration == 1: - ref_pnt_initial = ref_pnt - arcs_initial_network = [tuple(sorted(arc)) for arc in current_network.edges()] - - # Remove nodes with degree equal or lower than the threshold - current_network, _, ref_pnt = _remove_low_centrality_nodes( - current_network, deg_threshold=deg_threshold, plot=visualize_network - ) - arcs_updated_network = [tuple(sorted(arc)) for arc in current_network.edges()] - - # Test the network against requirements - avg_degree = len(current_network.edges) / len(current_network.nodes) if len(current_network.nodes) > 0 else 0 - print(f"The average degree is {avg_degree:.2f}") - - if len(current_network.nodes) >= min_nodes and avg_degree >= min_redundancy: - network_reqs_not_met = False - # Stop if requirements are met - - # Final check if the network meets requirements - if len(current_network.nodes) < min_nodes or avg_degree < min_redundancy: - print("Warning: Network could not meet all requirements with the given arcs.") - - # The arcs constructed above are no longer sorted based on their quality - arcs_updated_network_sorted = sorted( - arcs_updated_network, key=lambda arc: quality_dict_arcs.get(tuple(sorted(arc)), float("inf")) - ) - - return ref_pnt, arcs_updated_network_sorted, ref_pnt_initial, arcs_initial_network - - -def _from_arcs_to_graph(arcs, plot=False, save_path="./network.png"): - """Construct a graph from a set of arcs and identifies key properties of the network. - - Args: - ---- - arcs : (list of tuple) - List of arcs (edges) in the graph. Each arc is represented as a tuple - of two nodes, e.g., [(node1, node2), (node2, node3)]. - plot : (boolean) - Default is False, If True, the function visualizes the graph structure, including highlighting - connected components. Default is False (no visualization). - save_path : (string) - Default = "./network.png" - - Returns: - ------- - tuple: A tuple containing: - - network (networkx.Graph): The constructed graph based on the provided arcs. - - degree_centrality (dict): A dictionary where keys are nodes and values are their - degree centrality scores. - - ref_pnt (int): The index of the reference point (point with most connections). - """ - if not arcs: - raise ValueError("The input arcs list is empty. Please provide a valid list of arcs.") - - # Compute the network - network = nx.Graph() - network.add_edges_from(arcs) - - # Compute the degree centrality - # Which says something on how connected one particular node is with other nodes - degree_centrality = nx.degree_centrality(network) - - # Get the reference point (node with the most connections) - ref_pnt = max(degree_centrality, key=degree_centrality.get) - - if plot: - # Visualize the network - pos = nx.spring_layout(network) # Lay-out for the graph - nx.draw( - network, - pos, - with_labels=True, - node_size=500, - node_color="skyblue", - font_size=10, - font_weight="bold", - edge_color="gray", - ) - - connected_components = list(nx.connected_components(network)) - # Highlight the connected components - for component in connected_components: - nx.draw_networkx_nodes(network, pos, nodelist=component, node_color="orange", node_size=700) - - # Highlight the reference point - nx.draw_networkx_nodes(network, pos, nodelist=[ref_pnt], node_color="red", node_size=800) - plt.title(f"Network Visualization - {len(connected_components)} Connected Components") - plt.savefig(save_path, bbox_inches="tight") - - return network, degree_centrality, ref_pnt - - -def _remove_low_centrality_nodes(network, deg_threshold, plot=False, save_path="./network.png"): - """Remove nodes with a low degree of centrality. - - Args: - ---- - network (networkx.Graph): The input graph from which nodes will be removed. - deg_threshold (int): The degree threshold; nodes with a degree equal to or less than this value will be removed. - plot (bolean, optional): If True, the function visualizes the updated graph, including its connected components. - Default is False (no visualization). - save_path : (string) - Default = "./network.png" - - Returns: - ------- - tuple: A tuple containing: - - network (networkx.Graph): The updated graph after removing nodes with the specified degree threshold. - - degree_centrality (dict): A dictionary where keys are nodes and values are their degree centrality scores. - - ref_pnt (int or None): The node with the highest degree centrality in the updated graph. - - Example: - ------- - updated_network, degree_centrality, ref_point = remove_low_centrality_nodes( - network, deg_threshold=1, plot=False - ) - """ - # Identify nodes to remove - nodes_to_remove = [node for node, degree in network.degree() if degree <= deg_threshold] - removed_nodes = set(nodes_to_remove) # For plot highlighting - - # Remove these nodes from the network - network.remove_nodes_from(nodes_to_remove) - - # Handle empty graph case - if not network.nodes: - return network, {}, None - - # Degree centrality - degree_centrality = nx.degree_centrality(network) - - # Get the reference point (node with the highest degree centrality) - ref_pnt = max(degree_centrality, key=degree_centrality.get) - - if plot: - # Visualize the updated network - pos = nx.spring_layout(network) - nx.draw( - network, - pos, - with_labels=True, - node_size=500, - node_color="skyblue", - font_size=10, - font_weight="bold", - edge_color="gray", - ) - - connected_components = list(nx.connected_components(network)) - - # Highlight connected components - for component in connected_components: - nx.draw_networkx_nodes(network, pos, nodelist=component, node_color="orange", node_size=700) - - # Highlight removed nodes in the plot (optional) - if removed_nodes: - # Filter valid nodes that exist in the layout - valid_removed_nodes = [node for node in removed_nodes if node in pos] - nx.draw_networkx_nodes(network, pos, nodelist=valid_removed_nodes, node_color="red", node_size=800) - - nx.draw_networkx_nodes(network, pos, nodelist=[ref_pnt], node_color="red", node_size=800) - - plt.title(f"Network Visualization after Removing Nodes with Degree ≤ {deg_threshold}") - plt.savefig(save_path, bbox_inches="tight") - - return network, degree_centrality, ref_pnt - - -def test_succeeded_arcs_control_network( - succeeded_arcs, quality_dict_arcs, deg_threshold, min_nodes, min_redundancy, visualize_network=False -): - """Evaluate a network constructed from a set of arcs to determine if it meets structural requirements. - - This function tests a network built from a set of arcs to ensure it satisfies minimum structural criteria, - including the number of nodes, redundancy, and centrality. It checks whether any nodes with low degree should - be removed and then evaluates the network again. The arcs are sorted by quality after refinement. - - Parameters - ---------- - succeeded_arcs : list of tuples - A list of arcs (pairs of points) that are used to construct the network. - quality_dict_arcs : dict - A dictionary where the keys are the arcs and the values are their respective quality scores. - deg_threshold : float - The degree threshold for identifying low-centrality nodes. Nodes with degrees smaller will be removed. - min_nodes : int - The minimum number of nodes required for the network to be valid. - min_redundancy : float - The minimum average degree (redundancy) required for the network. - visualize_network : bool, optional - A flag to control network visualization of the network during evaluation. Default is False (no visualization). - - Returns - ------- - network_check : int - A flag indicating whether the network meets the requirements (1 if successful, 0 if not). - arcs_updated_network_sorted : list of tuples - The sorted list of arcs based on their quality after refining the network. - ref_pnt : int - The reference point used in the network. - - Example - ------- - network_check, final_arcs, ref_pnt = test_succeeded_arcs_control_network( - succeeded_arcs, deg_threshold=1.5, min_nodes=20, - min_redundancy=2.0, visualize_network=1 - ) - """ - arcs_updated_network_sorted = [] - network_check = 0 - - # Construct the network based on the input arcs - current_network = nx.Graph() - current_network, _, ref_pnt = _from_arcs_to_graph(succeeded_arcs, plot=visualize_network) - arcs_updated_network = [tuple(sorted(arc)) for arc in current_network.edges()] - - # Check whether there are nodes with degree smaller than one - nodes_to_remove = [node for node, degree in current_network.degree() if degree <= deg_threshold] - - if not nodes_to_remove: - # Test the network against requirements - avg_degree = len(current_network.edges) / len(current_network.nodes) if len(current_network.nodes) > 0 else 0 - print(f"The average degree is {avg_degree:.2f}") - - if len(current_network.nodes) >= min_nodes and avg_degree >= min_redundancy: - network_check = 1 - - # The arcs constructed above are no longer sorted based on their quality - arcs_updated_network_sorted = sorted( - arcs_updated_network, key=lambda arc: quality_dict_arcs.get(tuple(sorted(arc)), float("inf")) - ) - - else: - print("We remove low degree nodes") - # Remove the nodes with low degree and test the network again - current_network, _, ref_pnt = _remove_low_centrality_nodes( - current_network, deg_threshold=deg_threshold, plot=visualize_network - ) - arcs_updated_network = [tuple(sorted(arc)) for arc in current_network.edges()] - - # Test the network against requirements - avg_degree = len(current_network.edges) / len(current_network.nodes) if len(current_network.nodes) > 0 else 0 - print(f"The average degree is {avg_degree:.2f}") - - if len(current_network.nodes) >= min_nodes and avg_degree >= min_redundancy: - network_check = 1 - - # The arcs constructed above are no longer sorted based on their quality - arcs_updated_network_sorted = sorted( - arcs_updated_network, key=lambda arc: quality_dict_arcs.get(tuple(sorted(arc)), float("inf")) - ) - - else: - network_check = 0 - - return network_check, arcs_updated_network_sorted, ref_pnt - - -def ordered_arcs_connection_point_and_control_network( - rdx_connection_point, - rdy_connection_point, - slc_quality_connection_point, - connection_point_idx, - rdx_control, - rdy_control, - slc_quality_control, - control_idx, - dist_to_quality, - coordinate_type: Literal["euclidean", "geometric"], -): - """Generate a sorted array of unique arcs between a 'connection_point' and the control network. - - This function computes arcs between a connection point and control points, evaluating each arc based on - a combination of spatial distance and SLC quality. The arcs are sorted from best to worst, where "best" - is determined by the lowest combined quality and distance value. - - The function: - 1. Computes the Euclidean distance between the connection point and the control points. - 2. Calculates the arc quality time series for each arc. - 3. Combines the distance and maximum quality for each arc to define an overall quality value. - 4. Sorts the arcs by quality values from best (lowest) to worst (highest). - 5. Constructs a sorted array of arcs, ensuring the control point appears first in each arc. - - Args: - ---- - rdx_connection_point (xarray.DataArray): x-coordinate of the connection point. - rdy_connection_point (xarray.DataArray): y-coordinate of the connection point. - slc_quality_connection_point (xarray.DataArray): Array of SLC quality time series for the connection point. - connection_point_idx (int): Index of the connection point. - rdx_control (xarray.DataArray): x-coordinates of the control points. - rdy_control (xarray.DataArray): y-coordinates of the control points. - slc_quality_control (xarray.DataArray): Array of SLC quality time series for the control points. - control_idx (xarray.DataArray): Indices of the control points. - dist_to_quality (float): Conversion factor to scale distance relative to quality. - coordinate_type (Literal["euclidean", "geographic"]): whether the provided coordinates are Euclidean or - geographic - - Returns: - ------- - tuple: - - arcs (numpy.ndarray): A sorted array of arcs, where each row contains a control point - followed by the connection point. - - sorted_quality_values (numpy.ndarray): Sorted quality values corresponding to the arcs. - """ - # Stack the coordinates of the connection point and the control points - coords_connection_point = np.vstack((rdx_connection_point, rdy_connection_point)).T - coords_control = np.vstack((rdx_control, rdy_control)).T - - # Distance matrix between the connection point and the control points - if coordinate_type == "euclidean": - dist_matrix = distance_matrix(coords_connection_point, coords_control).squeeze() - else: - dist_matrix = np.zeros((coords_connection_point.shape[0], coords_control.shape[0])) - for m in range(coords_connection_point.shape[0]): - for n in range(coords_control.shape[0]): - dist_matrix[m, n] = get_distance(coords_connection_point[m], coords_control[n], mode=coordinate_type) - - # # Compute the quality matrix for connection point (pnt i) and all control points (point j) - slc_quality_j = slc_quality_control.values # Quality values of the control points - slc_quality_i = np.expand_dims(slc_quality_connection_point.values, axis=0) # Make sure the dimensions match - slc_quality_i = np.repeat(slc_quality_i, repeats=slc_quality_j.shape[0], axis=0) - - # Compute arc quality time series (which is a function of the slc_quality of point i and point j) - arc_quality_ts = np.sqrt(slc_quality_i**2 + slc_quality_j**2) - arc_quality_max = np.max(arc_quality_ts, axis=1) # Compute the maximum value - - # Combine distance and quality - arc_quality_dist_max = arc_quality_max + dist_matrix * dist_to_quality - - # Sort the arcs and compute the indices of the control points (since point i, the connection_point, is in all arcs) - sorted_control = np.argsort(arc_quality_dist_max) - sorted_control_idx = control_idx.values[sorted_control] # sort the indices of the control points as well - sorted_quality_values = np.sort(arc_quality_dist_max) - - # Compute the arcs between the connection_point and the control points. - # Make sure that the control_points comes first - arcs = np.zeros((len(sorted_control_idx), 2), dtype=int) - arcs[:, 0] = sorted_control_idx # control network points - arcs[:, 1] = connection_point_idx # connection_point - - return arcs, sorted_quality_values - - -def construct_control_network_test_arcs( - x_ref_search, - y_ref_search, - buffer_radius_ref, - dist_to_quality, - N_max_arcs, - N_top, - N_batch, - deg_threshold, - min_nodes, - min_redundancy, - nad_nmad_max, - visualize_network, - sigma_post_over_sigma_prior, - nr_max_iter_control, - bounds, - m2ph, - years, - dates, - temperature, - sd_complex, - slc_quality, - cr2ph, - ampl_ts, - bkps_stm, - mean_ampl_sd, - sigma_ampl_sd, - mad_ampl_sd, - median_ampl_sd, - x_coordinates, - y_coordinates, - nad_nmad, - coordinate_type: Literal["euclidean", "geographic"] = "euclidean", -): - """Construct the control network while testing the arcs inside. - - Parameters - ---------- - x_ref_search: - X coordinate of central point to search for reference point - y_ref_search: - Y coordinate of central point to search for reference point - buffer_radius_ref: - Size of search window for reference point - dist_to_quality: - relates arc length to additional sigma - N_max_arcs: - Maximum number of arcs - N_top: - Number of top-ranked arcs to start with. - N_batch: - Number of additional arcs to add in each iteration. - deg_threshold: - Degree threshold for removing low-centrality nodes from the network. - min_nodes: - Minimum number of nodes required in the final network. - min_redundancy: - Minimum average redundancy (degree) required in the final network. - nad_nmad_max: - Maximum NAD / NMAD for a point to be considered - visualize_network: - Boolean whether or not to visualize the network - sigma_post_over_sigma_prior: - Upper limit on how much the aposteriori sigma of an arc is allowed to differ from the apriori sigma - nr_max_iter_control: - Maximum number of iterations in the network testing - bounds: list of tuples - Bounds for parameter estimation in the format (lower_bounds, upper_bounds). - m2ph: float - Meters to phase conversion factor - years: np.ndarray - Array of decimal years corresponding to the time series epochs. - dates: np.ndarray - Array of date indices or timestamps corresponding to the time series. - temperature: np.ndarray - Array of temperature values for thermal expansion modeling. - sd_complex: np.ndarray - Single-difference complex - slc_quality: np.ndarray - Complex-valued standard deviations of the signal for all points. - cr2ph: np.ndarray - crossrange-to-phase for each point - ampl_ts: np.ndarray - amplitude timeseries - bkps_stm: np.ndarray - Breakpoints stm - mean_ampl_sd: np.ndarray - Single difference mean amplitude - sigma_ampl_sd: np.ndarray - Single difference mean amplitude standard deviation - mad_ampl_sd: np.ndarray - Median absolute deviation of the single difference amplitude - median_ampl_sd: np.ndarray - Median single difference amplitude - x_coordinates: np.ndarray - x coordinates (RD or longitude) - y_coordinates: np.ndarray - y coordinates (RD or latitude) - nad_nmad: np.ndarray - NAD or NMAD to be used in the point selection upon which `nad_nmad_max` is imposed. - coordinate_type: Literal["euclidean", "geographic"] = "euclidean" - Whether the provided coordinates are Euclidean (such as RD) or geographic (such as lon/lat) - - Returns - ------- - dict - Results of the accepted network - list - Reference point - list - Results of the accepted arcs - """ - # At the start, no arcs are tested yet, so we don't have failed_arcs, and noisy_arcs - failed_arcs = [] - noisy_arcs = [] - - # Get an ordered list of all potential arcs with a buffer area - arcs_search_area, _, quality_dict_arcs = get_ordered_arcs( - x_ref_search, - y_ref_search, - buffer_radius_ref, - dist_to_quality, - N_max_arcs, - nad_nmad_max, - x_coordinates, - y_coordinates, - slc_quality, - nad_nmad, - coordinate_type, - ) - - network_meets_requirements = False - - while not network_meets_requirements: - # Create a first network based on the ranked quality and leaving out the failed_arcs and noisy_arcs - ref_pnt, arcs_updated_network, ref_pnt_initial, arcs_initial_network = construct_control_network( - arcs_search_area, - quality_dict_arcs, - failed_arcs, - N_top, - N_batch, - deg_threshold, - min_nodes, - min_redundancy, - False, - ) - - print("Computing the solutions for the arcs") - # Test whether solutions for all arcs can be found (sometimes it happens that because of the complex - # functions no solutions can be found) - results_initial_control_network_v1 = arc_estimation_control_network( - arcs_updated_network, - bounds, - m2ph, - nr_max_iter_control, - years, - dates, - temperature, - sd_complex, - slc_quality, - cr2ph, - ampl_ts, - bkps_stm, - mean_ampl_sd, - sigma_ampl_sd, - mad_ampl_sd, - median_ampl_sd, - x_coordinates, - y_coordinates, - coordinate_type, - ) - - # Save the failed arcs to an array (such that they are not taken into account any more) - succeeded_arcs = [ - arc - for arc, succeeded in zip( - arcs_updated_network, results_initial_control_network_v1["succeeded_arcs"], strict=True - ) - if not np.isnan(succeeded).any() - ] - - failed_arcs_temp = [ - arc - for arc, succeeded in zip( - arcs_updated_network, results_initial_control_network_v1["succeeded_arcs"], strict=True - ) - if np.isnan(succeeded).any() or not succeeded.any() - ] - failed_arcs = list(set(failed_arcs).union(failed_arcs_temp)) - print(f"Failed arcs {failed_arcs}") - - # Remove the failed arcs from the dictionary with all the results (as the estimated parameters etc) - results_initial_control_network_v2 = {} # Make a new dictionary where we will not save - # the results of the failed_arcs - - # Find the indices where the arcs are saved where no solution was found. - # These arcs have nan values - valid_indices = ~np.isnan(results_initial_control_network_v1["succeeded_arcs"]).any(axis=1) - - # Filter all variables in the dictionary and only copy the values for the arcs where we found a solution - for key, value in results_initial_control_network_v1.items(): - if ( - isinstance(value, np.ndarray) - and value.shape[0] == results_initial_control_network_v1["succeeded_arcs"].shape[0] - ): - results_initial_control_network_v2[key] = value[valid_indices] - else: - results_initial_control_network_v2[key] = value # Keep values that are not row-based unchanged - - print("Check if the network with the solved arcs still meets our requirements") - - # Since we have 'failed_arcs', where no solution was found, the new network need to be tested - # It can happen that we have isolated points, - # or that the average degree of the network is not high enough anymore - network_check, arcs_updated_network, ref_pnt = test_succeeded_arcs_control_network( - succeeded_arcs, quality_dict_arcs, deg_threshold, min_nodes, min_redundancy, visualize_network - ) - - if network_check == 0: - print("Network fails requirements, starting again") - - if network_check == 1: - print("Network meets requirements") - # After the last test, the isolated arcs are removed (so they are still in - # 'succeeded_arcs' and in the dictionary) - # And these arcs need to be removed from the dictionary - missing_indices = [ - idx for idx, arc in enumerate(succeeded_arcs) if arc not in arcs_updated_network - ] # missing indices are the arcs that were isolated and removed. But they are still in the - # dictionary so there they need to be removed as well - - results_control_network_temp = {} - for key, value in results_initial_control_network_v2.items(): - if isinstance(value, np.ndarray): - if value.ndim in {1, 2}: # Both 1D and 2D arrays here - results_control_network_temp[key] = np.delete(value, missing_indices, axis=0) - else: - results_control_network_temp[key] = value - else: - results_control_network_temp[key] = value - - # Now we have a network that fullfills requirements but there might be noisy arcs - # We compute solutions for all arcs and computed RMSE - # In the next part, we will remove any arcs that are too noisy for the requirements, and then test if - # the network still fulfills the requirements - print("Calculate whether there are arcs where the solution that we found is noisy") - - est_displ_phase = ( - results_control_network_temp["unwrap_phases_arc"] - - results_control_network_temp["estimated_cross_range_phase"] - - results_control_network_temp["estimated_thermal_phase"] - ) - displ_phase = results_control_network_temp["estimated_displ_phase"] - - residues_per_arc = est_displ_phase - displ_phase - sigma_post_arc = np.std(residues_per_arc, axis=1) - mean_sigma_prior_arc = np.mean(results_control_network_temp["sigma_phases_arc"], axis=1) - - idx_bad_arcs = np.where(sigma_post_arc >= sigma_post_over_sigma_prior * mean_sigma_prior_arc)[0] - idx_good_arcs = np.where(sigma_post_arc < sigma_post_over_sigma_prior * mean_sigma_prior_arc)[0] - - bad_arcs = results_control_network_temp["succeeded_arcs"][idx_bad_arcs] - good_arcs = results_control_network_temp["succeeded_arcs"][idx_good_arcs].astype(int) - good_arcs = [tuple(row) for row in good_arcs] # Change the output to a list - - print(f"The value for sigma_post_over_prior is {sigma_post_over_sigma_prior}") - - print("we removed arcs") - - print("The bad arcs are") - print(idx_bad_arcs) - print("The good arcs are") - print(idx_good_arcs) - - # Add the noisy arcs to the list with failed_arcs - failed_arcs = list(set(failed_arcs).union([tuple(row.astype(int)) for row in bad_arcs])) - noisy_arcs = list(set(noisy_arcs).union([tuple(row.astype(int)) for row in bad_arcs])) - - # Check whether the network without the noisy arcs still meets requirements - print("Check whether the network stil meets the requirements, even after the removal of bad arcs") - network_check_good, arcs_updated_network, ref_pnt = test_succeeded_arcs_control_network( - good_arcs, quality_dict_arcs, deg_threshold, min_nodes, min_redundancy, visualize_network=False - ) - - if network_check_good == 0: - print("Network does not meet requirements, start over") - - if network_check_good == 1: - print("We are happy! The network consisting of the good arcs fulfills the requirements.") - - # If there are noisy arcs, they need to be removed from the final network - # Convert noisy_arcs to set for quicker lookup - noisy_arcs_set = {tuple(map(float, arc)) for arc in noisy_arcs} - - # Get the current succeeded_arcs - succeeded_arcs = results_control_network_temp["succeeded_arcs"] - - # Deterimine which arcs to keep (so NOT in noisy_arcs_set) - valid_indices = np.array([tuple(row) not in noisy_arcs_set for row in succeeded_arcs]) - - # New dict with only valid rows - results_control_network = {} - - for key, value in results_control_network_temp.items(): - if isinstance(value, np.ndarray) and value.shape[0] == succeeded_arcs.shape[0]: - results_control_network[key] = value[valid_indices] - else: - results_control_network[key] = value # Leave unrelated variables unchanged - - network_meets_requirements = True # Stop the loop - - if not network_meets_requirements: - # If the network doesn't meet the requirements, start over - print("Network does not meet the requirements. Recomputing the network with updated failed_arcs...") - - return results_control_network, ref_pnt, arcs_updated_network - - -def _network_relation_matrix(idx_source, idx_target, n_points, idx_refpnt, sparse_mode: bool = False): - """Create the network relation matrix A as a sparse matrix. - - A network relation matrix has shape (n_arcs, n_points - 1). - Each row corresponds to an arc, and each column corresponds to a point, excluding the reference point. - For each arc, the column corresponding to the source point has a value of -1, and the column corresponding - to the target point has a value of +1. - All other entries are zero. - - The reference point column removal refers to Eq.4.11 of Kampes (2006). - - Parameters - ---------- - idx_source : list or np.ndarray - List of source point indices for each arc. - idx_target : list or np.ndarray - List of target point indices for each arc. - n_points : int - Total number of points in the network. - idx_refpnt : int - Index of the reference point to be excluded from the matrix. This index assumes 0-based indexing of the points. - sparse_mode : bool - Whether to return the matrix in sparse format. If False, returns a dense numpy array. - - References - ---------- - Kampes, Bert M. Radar interferometry: persistent scatterer technique. Dordrecht: Springer Netherlands, 2006. - DOI: 10.1007/978-1-4020-4723-7 - """ - n_arcs = len(idx_source) - A_start = sparse.COO( - (np.arange(n_arcs), idx_source), - np.full_like(np.arange(n_arcs), -1, dtype=np.int8), - shape=(n_arcs, n_points), - ) - A_end = sparse.COO( - (np.arange(n_arcs), idx_target), - np.full_like(np.arange(n_arcs), 1, dtype=np.int8), - shape=(n_arcs, n_points), - ) - A = A_start + A_end - - # Convert to Compressed Sparse Row (CSR) matrix for efficient arithmetic and matrix vector operations - A = A.tocsr() - - # Remove reference point column - A = scipy.sparse.hstack([A[:, :idx_refpnt], A[:, idx_refpnt + 1 :]]) - - if not sparse_mode: - A = np.array(A.todense()) - - return A - - -def _independent_arcs(arcs: np.ndarray) -> np.ndarray: - """Select independent arcs from a list of arcs. - - An arc is independent if its starting and ending points do not exist in any other arc's - starting or ending points. - - Parameters - ---------- - arcs : np.ndarray - A 2D array of shape (n_points, 2) where each row represents indices of the starting and ending points - of an arc. - - Returns - ------- - np.ndarray - A 2D array of independent arcs, where each row represents indices of the starting and ending points - of an arc. - """ - # Select arcs with unique starting points - _, unique_idx_start = np.unique(arcs[:, 0], return_index=True) - arcs = arcs[unique_idx_start, :] - - # Select arcs with unique ending points - _, unique_idx_end = np.unique(arcs[:, 1], return_index=True) - arcs = arcs[unique_idx_end, :] - - # After previous two steps, no arcs will share starting or ending points. - # However, there starting points may be the ending points of other arcs, and vice versa. - # To ensure independency, we loop through the rest arcs and add arc one by one - # In each interation, remove arcs that - # 1) start with the ending point of this arc, or - # 2) end with the starting point of this arc - arcs_selected = np.empty((0, 2), dtype=int) - while arcs.shape[0] > 0: - arc_current = arcs[0, :] - arcs_selected = np.append(arcs_selected, [arc_current], axis=0) - # Remove arcs which contain the starting point or ending point of the current arc - idx_remove = np.where((arcs[:, 1] == arc_current[0]) | (arcs[:, 0] == arc_current[1]))[0] - # add the index of the current arc to idx_remove - idx_remove = np.append(idx_remove, 0) - arcs = np.delete(arcs, idx_remove, axis=0) - - return arcs_selected diff --git a/test_network.py b/test_network.py deleted file mode 100644 index a1188a54..00000000 --- a/test_network.py +++ /dev/null @@ -1,531 +0,0 @@ -"""test_network.py""" - -import numpy as np -import pytest -import xarray as xr - -from depsi.network import ( - _ensure_network_min_connections, - _ensure_single_network, - _independent_arcs, - _network_relation_matrix, - _remove_network_points_min_connections, - form_network, - spatial_integration, -) - - -@pytest.fixture -def stm_random(): - """Fixture to create a random STM dataset.""" - rng = np.random.default_rng(42) - Npoints = 12 # Number of points - Ntimes = 31 # Number of epochs - # Coordinates and time - lat = rng.uniform(51.14, 51.15, Npoints) - lon = rng.uniform(6.9, 7.0, Npoints) - time = np.arange(Ntimes) - # Data - complex = rng.uniform(-1, 1, (Npoints, Ntimes)) + 1j * rng.uniform(-1, 1, (Npoints, Ntimes)) - phase = np.angle(complex) - h2ph = rng.uniform(1e3, 1e4, (Npoints, Ntimes)) - # Create the xarray Dataset - stm = xr.Dataset( - data_vars={ - "phase": (("space", "time"), phase), - "h2ph": (("space", "time"), h2ph), - "complex": (("space", "time"), complex), - }, - coords={ - "space": ("space", np.arange(Npoints)), - "time": ("time", time), - "lat": ("space", lat), - "lon": ("space", lon), - }, - ) - - return stm - - -@pytest.fixture -def stm_random_grid(): - """STM points forming a 10x10 grid""" - N_time = 50 - grid_shape = 10 - x_grid, y_grid = np.meshgrid(np.arange(0, 100, grid_shape), np.arange(0, 100, grid_shape)) - N_points = x_grid.flatten().shape[0] - - stm = xr.Dataset( - coords={ - "space": (["space"], np.arange(N_points)), - "time": (["time"], np.arange(N_time)), - "x": (["space"], x_grid.flatten()), - "y": (["space"], y_grid.flatten()), - }, - data_vars={ - "phase": (["space", "time"], np.random.uniform(0, 1, (N_points, N_time))), - "h2ph": (["space", "time"], np.random.uniform(0, 1, (N_points, N_time))), - "ambiguity": (["space", "time"], np.random.choice([-1, 0, 1], (N_points, N_time), p=[0.02, 0.96, 0.02])), - }, - ) - - return stm - - -@pytest.fixture -def arcs_random(stm_random): - """Fixture of fully connected arcs from stm_random.""" - # Fully connected arcs - # Defaul method is redundant - # No max_length, so all points are connected - arcs = form_network(stm_random, key_phase="phase", key_h2ph="h2ph", key_Btemporal="time") - - # Most arcs have quality 0.9 - # Except the last two have quality 0.0 - # And the first five have quality 0.99 - temp_coh = np.zeros((arcs.sizes["space"],)) - temp_coh[:-2] = 0.9 - temp_coh[:5] = 0.99 - arcs["temp_coh"] = (("space"), temp_coh) - - return arcs - - -def _build_network_components(component_sizes: list[int]) -> tuple[xr.Dataset, xr.Dataset]: - """Build coordinate only point/arcs STMs from connected-component sizes.""" - n_points = int(sum(component_sizes)) - stm_pnts = xr.Dataset(coords={"space": ("space", np.arange(n_points))}) - - source = [] - target = [] - offset = 0 - for size in component_sizes: - # Build each component as a simple chain graph. - for idx in range(offset, offset + size - 1): - source.append(idx) - target.append(idx + 1) - offset += size - - stm_arcs = xr.Dataset( - coords={ - "source": ("space", np.array(source, dtype=int)), - "target": ("space", np.array(target, dtype=int)), - } - ) - - return stm_arcs, stm_pnts - - -class TestNetworkFormation: - def test_form_network_simulated_grid(self, stm_random_grid): - arcs = form_network( - stm_random_grid, - key_phase="phase", - key_h2ph="h2ph", - key_Btemporal="time", - key_xcrds="x", - key_ycrds="y", - max_length=25, - min_links=8, - num_partitions=8, - ) - - source = arcs["source"].values - target = arcs["target"].values - - assert arcs.sizes["space"] == 428 # nr arcs should be 428 with a 10x10 grid setting - assert np.all(np.diff(source) >= 0) # check if source is mono-increasing - assert np.all(source < target) # check if all sources < targets - assert ( - np.unique(np.column_stack((source, target)), axis=0).shape[0] == source.shape[0] - ) # check if all (source, target) pairs are unique - assert np.unique(arcs["uid"].values).shape[0] == arcs.sizes["space"] - - def test_stm_to_arcs_subtract(self, stm_random): - # Generate arcs of a Delaunay network with subtracted phase differences. - stm_arcs = form_network( - stm_random, - key_phase="phase", - key_h2ph="h2ph", - key_Btemporal="time", - network_method="delaunay", - max_length=0.05, - dphase_method="subtract", - ) - - assert all( - [all([-2 * np.pi <= phase <= 2 * np.pi for phase in phases]) for phases in stm_arcs["d_phase"].values] - ) - - def test_stm_to_arcs_conjmult(self, stm_random): - # Generate arcs of a Delaunay network with conjugate multiplication phase differences. - stm_arcs = form_network( - stm_random, - key_phase="phase", - key_h2ph="h2ph", - key_Btemporal="time", - network_method="delaunay", - max_length=0.05, - dphase_method="conjmult", - ) - - assert all([all([-np.pi <= phase <= np.pi for phase in phases]) for phases in stm_arcs["d_phase"].values]) - - def test_stm_to_arcs_fail(self, stm_random): - # Test incorrect method fail. - with pytest.raises(NotImplementedError): - form_network( - stm_random, - key_phase="phase", - key_h2ph="h2ph", - key_Btemporal="time", - network_method="unknown", - dphase_method="subtract", - ) - with pytest.raises(NotImplementedError): - form_network( - stm_random, - key_phase="phase", - key_h2ph="h2ph", - key_Btemporal="time", - network_method="delaunay", - dphase_method="unknown", - ) - - -class TestNetworkEnsure: - @pytest.mark.parametrize("thres, min_n_connections", [(0.5, 2), (0.5, 1)]) - def test_select_arcs_discard_two(self, arcs_random, stm_random, thres, min_n_connections): - """Should only discard two arcs, with temp_coh < 0.5.""" - # Select arcs based on temp_coh threshold. - mask = np.abs(arcs_random["temp_coh"]) > thres # mask as DataArray - arcs_selected = arcs_random.where(mask, drop=True) - arcs_results, _ = _ensure_network_min_connections(arcs_selected, stm_random, min_connections=min_n_connections) - - # Threshold is 0.5, so only the last two arcs are discarded - # The min_n_connections should not affect the selection - assert arcs_results.sizes["space"] == arcs_random.sizes["space"] - 2 - - def test__remove_network_points_min_connections_nconnection_zero(self, stm_random, arcs_random): - """Raise error when min_connections <1.""" - with pytest.raises(ValueError): - _remove_network_points_min_connections(stm_random, arcs_random, min_connections=-1) - with pytest.raises(ValueError): - _remove_network_points_min_connections(stm_random, arcs_random, min_connections=0) - - def test__remove_network_points_min_connections_keep_all_pnts(self, stm_random, arcs_random): - """No STM points removed since no arc is discarded.""" - stm_updated, arcs_updated = _remove_network_points_min_connections(stm_random, arcs_random, min_connections=1) - - assert stm_updated.sizes["space"] == stm_random.sizes["space"] - assert arcs_updated.sizes["space"] == arcs_random.sizes["space"] - - def test__remove_network_points_min_connections_discard_one(self, stm_random, arcs_random): - """Remove one STM point.""" - # remove arcs with source or target == 1 - arcs = arcs_random.copy(deep=True) - arcs = arcs.where((arcs["source"] != 1) & (arcs["target"] != 1), drop=True) - - stm_updated, arcs_updated = _remove_network_points_min_connections(stm_random, arcs, min_connections=1) - - # Should remove the point with index 1 - assert stm_updated.sizes["space"] == stm_random.sizes["space"] - 1 - - @pytest.mark.parametrize("component_sizes", [[12]]) - def test_ensure_single_network_no_separated_part(self, component_sizes): - """Keep the network unchanged when there is only one connected component.""" - stm_arcs, stm_pnts = _build_network_components(component_sizes) - - stm_arcs_out, stm_pnts_out = _ensure_single_network(stm_arcs, stm_pnts, largest_component_ratio=0.8) - - assert stm_pnts_out.sizes["space"] == stm_pnts.sizes["space"] - assert stm_arcs_out.sizes["space"] == stm_arcs.sizes["space"] - assert np.array_equal(stm_arcs_out["source"].values, stm_arcs["source"].values) - assert np.array_equal(stm_arcs_out["target"].values, stm_arcs["target"].values) - - @pytest.mark.parametrize( - ["component_sizes", "largest_component_ratio"], - [([15, 1, 1, 1], 0.8), ([8, 6], 0.5), ([9, 1, 1], 0.8), ([5, 3, 2], 0.4)], - ) - def test_ensure_single_network_keep_largest_significant(self, component_sizes, largest_component_ratio): - """Keep only the largest component when it is significant enough.""" - stm_arcs, stm_pnts = _build_network_components(component_sizes) - largest_size = max(component_sizes) - - stm_arcs_out, stm_pnts_out = _ensure_single_network(stm_arcs, stm_pnts, largest_component_ratio) - - assert stm_pnts_out.sizes["space"] == largest_size - assert stm_arcs_out.sizes["space"] == largest_size - 1 - assert np.all(stm_arcs_out["source"].values >= 0) - assert np.all(stm_arcs_out["target"].values >= 0) - assert np.all(stm_arcs_out["source"].values < largest_size) - assert np.all(stm_arcs_out["target"].values < largest_size) - - @pytest.mark.parametrize( - "component_sizes", - [ - [6, 4], - [8, 7, 1], - [10, 9, 1, 1], - ], - ) - def test_ensure_single_network_raise_when_largest_not_significant(self, component_sizes): - """Raise when the largest component is not clearly dominant.""" - largest_component_ratio = 0.8 - stm_arcs, stm_pnts = _build_network_components(component_sizes) - - with pytest.raises(RuntimeError): - _ensure_single_network(stm_arcs, stm_pnts, largest_component_ratio) - - -class TestNetworkUnwrap: - @pytest.mark.parametrize( - ["id_ref", "idx_err_space", "idx_err_time", "error_values", "skip_network_adaptation"], - [ - (3, [], [], [], False), # No error - (3, [2, 11], [7, 13], [-1, 1], False), # Two errors in arc ambiguities - (3, [2, 11], [7, 13], [-1, 1], True), # Two errors, skip network adjustment, should still be corrected - (9, [0, 4, 8], [5, 10, 15], [1, -100, 1], False), # Three errors, one large, but should be corrected - ], - ) - def test_spatial_integration(self, id_ref, idx_err_space, idx_err_time, error_values, skip_network_adaptation): - """Test spatial unwrapping based on arc ambiguities. - - Build points with true value of ambiguities. - Construct arcs with arc ambiguities derived from true ambiguities. - Add tiny errors to arc ambiguities at certain space/time indices. - - Then perform spatial unwrapping with a specified reference point. - - The spatial unwrapping should be able to solve the point ambiguities correctly. - The solved ambiguities should w.r.t. the reference point. - """ - # Set up test parameters - rng = np.random.default_rng(42) - Npoints = 17 # Number of points - Ntimes = 29 # Number of epochs - time = np.arange(Ntimes) - complex = rng.uniform(-1, 1, (Npoints, Ntimes)) + 1j * rng.uniform(-1, 1, (Npoints, Ntimes)) - phase = np.angle(complex) - h2ph = rng.uniform(1e3, 1e4, (Npoints, Ntimes)) - - # Create the points - stm_pnts = xr.Dataset( - data_vars={ - "phase": (("space", "time"), phase), - "h2ph": (("space", "time"), h2ph), - "complex": (("space", "time"), complex), - "ambiguities_true": ( - ("space", "time"), - np.round(rng.normal(0, 0.5, (Npoints, Ntimes))).astype(int).clip(-1, 1), - ), - }, - coords={ - "space": ("space", np.arange(Npoints)), - "time": ("time", time), - "azimuth": ("space", np.round(rng.normal(0, 10, (Npoints))).astype(int)), - "range": ("space", np.round(rng.normal(0, 10, (Npoints))).astype(int)), - }, - attrs={"wavelength": 0.056}, # Wavelength in meters - ) - - # Construct arcs based on true ambiguities - # All arcs by default have 0.99 temp_coh - stm_arcs = form_network( - stm_pnts, - key_xcrds="azimuth", - key_ycrds="range", - key_phase="phase", - key_h2ph="h2ph", - key_Btemporal="time", - network_method="redundant", - max_length=30, - ) - temp_coh = np.zeros((stm_arcs.sizes["space"],)) + 0.99 - stm_arcs["temp_coh"] = (("space"), temp_coh) - - # Compute arc ambiguities from true point ambiguities - ambigs = ( - stm_pnts["ambiguities_true"].values[stm_arcs["target"].values, :] - - stm_pnts["ambiguities_true"].values[stm_arcs["source"].values, :] - ) - # Introduce some errors in ambiguities - ambigs_errors = np.zeros_like(ambigs) - for idx_s, idx_t, err in zip(idx_err_space, idx_err_time, error_values, strict=False): - ambigs_errors[idx_s, idx_t] += err - stm_arcs["ambiguities"] = (("space", "time"), ambigs + ambigs_errors) - - stm_arcs_output, stm_pnts_output = spatial_integration( - stm_pnts, stm_arcs, idx_refpnt=id_ref, key_sdphase="phase", skip_network_adaptation=skip_network_adaptation - ) - - # Verify output dimensions, no points should be rejected - assert stm_pnts_output.sizes["space"] == stm_pnts.sizes["space"] - - # Check that the solved ambiguities match the true ambiguities w.r.t. the reference point - assert np.allclose( - stm_pnts_output["ambiguities"].values - - stm_pnts["ambiguities_true"].values - + np.tile(stm_pnts["ambiguities_true"].isel(space=id_ref).values, (stm_pnts.sizes["space"], 1)), - 0, - ) - - # Check that the unwrapped phase is correct - ref_phase = stm_pnts["phase"].isel(space=id_ref).values - relative_ambiguities = stm_pnts["ambiguities_true"].values - np.tile( - stm_pnts["ambiguities_true"].isel(space=id_ref).values, (stm_pnts.sizes["space"], 1) - ) # true ambiguities relative to reference point - unwrapped_phase_expected = ( - stm_pnts["phase"].values - + relative_ambiguities * 2 * np.pi - - np.tile(ref_phase, (stm_pnts.sizes["space"], 1)) - ) - assert np.allclose(stm_pnts_output["unwrapped_phase"].values, unwrapped_phase_expected) - - # Check that the reference point index remains the same - assert stm_pnts_output.attrs["idx_refpnt"] == id_ref - - @pytest.mark.parametrize("idx_ref", [0, 5, 10, 16]) - def test_spatial_integration_ref_pnt_removed(self, idx_ref): - """Raise error when reference point is removed""" - # Set up test parameters - rng = np.random.default_rng(42) - Npoints = 17 # Number of points - Ntimes = 29 # Number of epochs - time = np.arange(Ntimes) - complex = rng.uniform(-1, 1, (Npoints, Ntimes)) + 1j * rng.uniform(-1, 1, (Npoints, Ntimes)) - phase = np.angle(complex) - h2ph = rng.uniform(1e3, 1e4, (Npoints, Ntimes)) - - # Create the points - stm_pnts = xr.Dataset( - data_vars={ - "phase": (("space", "time"), phase), - "h2ph": (("space", "time"), h2ph), - "complex": (("space", "time"), complex), - "ambiguities_true": ( - ("space", "time"), - np.round(rng.normal(0, 0.5, (Npoints, Ntimes))).astype(int).clip(-1, 1), - ), - }, - coords={ - "space": ("space", np.arange(Npoints)), - "time": ("time", time), - "azimuth": ("space", np.round(rng.normal(0, 10, (Npoints))).astype(int)), - "range": ("space", np.round(rng.normal(0, 10, (Npoints))).astype(int)), - }, - attrs={"wavelength": 0.056}, - ) - - # Construct arcs based on true ambiguities - # All arcs by default have 0.99 temp_coh - stm_arcs = form_network( - stm_pnts, - key_xcrds="azimuth", - key_ycrds="range", - key_phase="phase", - key_h2ph="h2ph", - key_Btemporal="time", - network_method="redundant", - max_length=30, - ) - temp_coh = np.zeros((stm_arcs.sizes["space"],)) + 0.99 - stm_arcs["temp_coh"] = (("space"), temp_coh) - - # Set temp_coh of all arcs connected to reference point to 0.01 - mask_ref_arcs = (stm_arcs["source"] == idx_ref) | (stm_arcs["target"] == idx_ref) - stm_arcs["temp_coh"] = stm_arcs["temp_coh"].where(~mask_ref_arcs, other=0.01) - - # Compute arc ambiguities from true point ambiguities - ambigs = ( - stm_pnts["ambiguities_true"].values[stm_arcs["target"].values, :] - - stm_pnts["ambiguities_true"].values[stm_arcs["source"].values, :] - ) - stm_arcs["ambiguities"] = (("space", "time"), ambigs) - - with pytest.raises(ValueError): - spatial_integration(stm_pnts, stm_arcs, idx_refpnt=idx_ref, key_sdphase="phase") - - @pytest.mark.parametrize( - ["idx_source", "idx_target", "n_points", "idx_refpnt"], - [ - (np.array([0, 1, 2]), np.array([1, 2, 3]), 4, 0), # 4 points, 3 arcs - (np.array([0, 1, 2]), np.array([1, 2, 3]), 7, 0), # 7 points, 3 arcs - (np.array([1, 1, 2, 2]), np.array([0, 2, 1, 3]), 4, 2), # 4 points, 4 arcs, unsorted - (np.array([0, 0, 0, 1, 1, 2, 2]), np.array([1, 2, 3, 3, 4, 3, 4]), 5, 3), # 5 points, 6 arcs - ], - ) - def test_init_network_relation_matrix( - self, - idx_source, - idx_target, - n_points, - idx_refpnt, - ): - A = _network_relation_matrix(idx_source, idx_target, n_points, idx_refpnt) - - # Create expected matrix in a for loop - A_exp = np.zeros((idx_source.shape[0], n_points), dtype=int) - for i, (src, tgt) in enumerate(zip(idx_source, idx_target, strict=False)): - A_exp[i, src] = -1 - A_exp[i, tgt] = 1 - A_exp = np.delete(A_exp, idx_refpnt, axis=1) # Remove reference point column - - assert A.shape == A_exp.shape - assert np.all(A == A_exp) - - @pytest.mark.parametrize( - ["idx_source", "idx_target", "n_points", "idx_refpnt"], - [ - (np.array([0, 1, 2]), np.array([1, 2, 3]), 4, 0), # 4 points, 3 arcs - (np.array([0, 1, 2]), np.array([1, 2, 3]), 7, 0), # 7 points, 3 arcs - (np.array([1, 1, 2, 2]), np.array([0, 2, 1, 3]), 4, 2), # 4 points, 4 arcs, unsorted - (np.array([0, 0, 0, 1, 1, 2, 2]), np.array([1, 2, 3, 3, 4, 3, 4]), 5, 3), # 5 points, 6 arcs - ], - ) - def test_init_network_relation_matrix_sparse( - self, - idx_source, - idx_target, - n_points, - idx_refpnt, - ): - A = _network_relation_matrix(idx_source, idx_target, n_points, idx_refpnt, sparse_mode=True) - - # Create expected matrix in a for loop - A_exp = np.zeros((idx_source.shape[0], n_points), dtype=int) - for i, (src, tgt) in enumerate(zip(idx_source, idx_target, strict=False)): - A_exp[i, src] = -1 - A_exp[i, tgt] = 1 - A_exp = np.delete(A_exp, idx_refpnt, axis=1) # Remove reference point column - - assert A.shape == A_exp.shape - assert np.all(A.todense() == A_exp) - - -class TestArcsUtils: - @pytest.mark.timeout(10) # Each should finish in 10 seconds - @pytest.mark.parametrize( - "npoints, narcs", - [ - (103, 1000), - (1923, 10000), - (12, 30), - ], - ) - def test_independent_arcs(self, npoints, narcs): - # Simulate random arcs - rng = np.random.default_rng(42) - arcs = rng.integers(0, npoints, size=(narcs, 2)) - # remove arcs which has the same start and end point - arcs = arcs[arcs[:, 0] != arcs[:, 1]] - # Remove duplicate arcs - arcs = np.unique(np.sort(arcs, axis=1), axis=0) - - # Test that the arcs are independent. - independent_arcs = _independent_arcs(arcs) - - # A point index should only appear once - # either as a start or end point of an arc. - all_idx = independent_arcs.flatten() - assert all_idx.shape == np.unique(all_idx).shape diff --git a/tests/test_network.py b/tests/test_network.py index a36c2efe..a1188a54 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -7,6 +7,7 @@ from depsi.network import ( _ensure_network_min_connections, _ensure_single_network, + _independent_arcs, _network_relation_matrix, _remove_network_points_min_connections, form_network, @@ -500,3 +501,31 @@ def test_init_network_relation_matrix_sparse( assert A.shape == A_exp.shape assert np.all(A.todense() == A_exp) + + +class TestArcsUtils: + @pytest.mark.timeout(10) # Each should finish in 10 seconds + @pytest.mark.parametrize( + "npoints, narcs", + [ + (103, 1000), + (1923, 10000), + (12, 30), + ], + ) + def test_independent_arcs(self, npoints, narcs): + # Simulate random arcs + rng = np.random.default_rng(42) + arcs = rng.integers(0, npoints, size=(narcs, 2)) + # remove arcs which has the same start and end point + arcs = arcs[arcs[:, 0] != arcs[:, 1]] + # Remove duplicate arcs + arcs = np.unique(np.sort(arcs, axis=1), axis=0) + + # Test that the arcs are independent. + independent_arcs = _independent_arcs(arcs) + + # A point index should only appear once + # either as a start or end point of an arc. + all_idx = independent_arcs.flatten() + assert all_idx.shape == np.unique(all_idx).shape diff --git a/test_stochastic.py b/tests/test_stochastic.py similarity index 97% rename from test_stochastic.py rename to tests/test_stochastic.py index 036b6b9d..f5aa44bf 100644 --- a/test_stochastic.py +++ b/tests/test_stochastic.py @@ -6,7 +6,7 @@ def simulated_stm(n_ifg, n_points): - """function to simulate arcs stm for testing.""" + """Function to simulate arcs stm for testing.""" rng = np.random.default_rng(31) # Simulate coordinates diff --git a/utils.py b/utils.py deleted file mode 100644 index 912f0028..00000000 --- a/utils.py +++ /dev/null @@ -1,841 +0,0 @@ -import math -import os -from typing import Literal - -import asf_search as asf -import dask.array as da -import pandas as pd -import pyproj - -try: - from datetime import UTC, datetime, timedelta -except ImportError: # UTC can only be imported from Python 3.11 onwards - import warnings - from datetime import datetime, timedelta, timezone - - UTC = timezone.utc - warnings.warn( - """ - DePSI uses datetime.UTC, which is only supported from Python 3.11 onwards. - For older Python versions, datetime.timezone.utc is used. - This might be deprecated in newer DePSI versions. - """, - DeprecationWarning, - stacklevel=1, # necessary to start the call stack here. - ) - -import logging - -import geopandas -import numpy as np -import pytz -import xarray as xr - -from depsi.constants import EARTH_RADIUS, WAVELENGTH_S1 - -logger = logging.getLogger(__name__) - - -def wrap_phase(phs_abs): - """Wrap the absolute phase to the range [-pi, pi). - - Parameters - ---------- - phs_abs : array_like or float - The absolute phase. - - Returns - ------- - ndarray or float - The wrapped phase in the range [-pi, pi). - """ - phs_wrapped = np.remainder(phs_abs + np.pi, 2 * np.pi) - np.pi - - return phs_wrapped - - -def _orbit_fit(orbit, verbose=0, der=True): - """Return a orbit_fit dict. - - Modified from the "orbitFit" function: - https://github.com/Pbaz98/Caroline-Radar-Coding-Toolbox/blob/main/gecoris/geoUtils.py#L325 - - Satellite state vector interpolation using Chebyshev polynomials of - 7th order (according to DLR recommendations). Function returns Chebyshev - polynomial coefficients. Use these to evaluate orbit state at given time - via function 'orbitVal'. - - input: snappy 'orbit' object (as read by 'read_metadata' function) - - CHANGE LOG - - 30/6/2023: Modified to adapt the input to a np.array Nx4 (N number of timesamples) - - 22/09/23: add the flag for derivative or not - """ - # parse masterorb: - t = orbit[:, 0] - x = orbit[:, 1] - y = orbit[:, 2] - z = orbit[:, 3] - - # interpolate orbits using Chebyshev polynomials of 7th order: - t0 = (min(t) + max(t)) / 2 - px = t - t0 # time argument px (centered around mid interval) - cx = np.polynomial.chebyshev.chebfit(px, x, 7) # position - cy = np.polynomial.chebyshev.chebfit(px, y, 7) - cz = np.polynomial.chebyshev.chebfit(px, z, 7) - - if der: - cvx = np.polynomial.chebyshev.chebder(cx) # velocity - cvy = np.polynomial.chebyshev.chebder(cy) - cvz = np.polynomial.chebyshev.chebder(cz) - else: - x_vel = orbit[:, 4] - y_vel = orbit[:, 5] - z_vel = orbit[:, 6] - - cvx = np.polynomial.chebyshev.chebfit(px, x_vel, 7) # velocity - cvy = np.polynomial.chebyshev.chebfit(px, y_vel, 7) - cvz = np.polynomial.chebyshev.chebfit(px, z_vel, 7) - - cax = np.polynomial.chebyshev.chebder(cvx) # acceleration - cay = np.polynomial.chebyshev.chebder(cvy) - caz = np.polynomial.chebyshev.chebder(cvz) - - if verbose: - # position fit residuals: - x_res = np.polynomial.chebyshev.chebval(px, cx) - x - y_res = np.polynomial.chebyshev.chebval(px, cy) - y - z_res = np.polynomial.chebyshev.chebval(px, cz) - z - x_std = np.std(x_res) - y_std = np.std(y_res) - z_std = np.std(z_res) - print(f"Orbit fit position residuals: X {x_std:.4f} m, Y {y_std:.4f} m, Z {z_std:.4f} m. ") - # velocity residuals: - vx_res = np.polynomial.chebyshev.chebval(px, np.polynomial.chebyshev.chebder(cx)) - x_vel - vy_res = np.polynomial.chebyshev.chebval(px, np.polynomial.chebyshev.chebder(cy)) - y_vel - vz_res = np.polynomial.chebyshev.chebval(px, np.polynomial.chebyshev.chebder(cz)) - z_vel - vx_std = np.std(vx_res) - vy_std = np.std(vy_res) - vz_std = np.std(vz_res) - print(f"Orbit fit velocity residuals: vX {vx_std:.4f} m/s, vY {vy_std:.4f} m/s, vZ {vz_std:.4f} m/s. ") - - orbit_fit = dict() - orbit_fit["t0"] = t0 - orbit_fit["cx"] = cx - orbit_fit["cy"] = cy - orbit_fit["cz"] = cz - orbit_fit["cvx"] = cvx - orbit_fit["cvy"] = cvy - orbit_fit["cvz"] = cvz - orbit_fit["cax"] = cax - orbit_fit["cay"] = cay - orbit_fit["caz"] = caz - - return orbit_fit - - -def get_distance( - source: list | tuple | np.ndarray, - target: list | tuple | np.ndarray, - mode: Literal["euclidean", "geographic"] = "euclidean", -): - """Calculate the distance between two points. - - The Euclidean mode calculates distance on a 2D XY-plane. The Geographic mode approximates the Earth as a sphere - with radius 6378136 meter (the polar radius). On long north-south oriented arcs, distance errors of up to 0.3% are - possible. - - Parameters - ---------- - source: list | tuple | np.ndarray - The source point, formatted as (x, y) / (lon, lat) - target: list | tuple | np.ndarray - The target point, formatted as (x, y) / (lon, lat) - mode: Literal["euclidean", "geographic"], default "euclidean" - Whether the source and target points are given in (x, y) (units meters) or (lon, lat) (units degrees) - - Returns - ------- - The distance between the two points in meters. - """ - if mode == "euclidean": - return math.dist(source, target) - elif mode == "geographic": - # this is the Haversine formula - lat1 = source[1] - lat2 = target[1] - dphi = np.radians(lat1 - lat2) - dlambda = np.radians(source[0] - target[0]) - dist = ( - 2 - * EARTH_RADIUS - * np.arcsin( - np.sqrt( - (1 - np.cos(dphi) + np.cos(np.radians(lat1)) * np.cos(np.radians(lat2)) * (1 - np.cos(dlambda))) / 2 - ) - ) - ) - return dist - raise ValueError(f"Unknown mode {mode}, only know euclidean and geographic!") - - -def npdatetime64_to_datetime(date: np.datetime64, tz_aware: bool = True) -> datetime: - """Convert a numpy datetime64 object to a python datetime object. - - Parses the np.datetime64 object into a datetime object. - - Parameters - ---------- - date : np.datetime64 - the date to be converted - tz_aware: bool, default True - whether the returned datetime object should be timezone-aware or not - - Returns - ------- - datetime.datetime - The same date converted to a datetime object - """ - timestamp = (date - np.datetime64("1970-01-01T00:00:00")) / np.timedelta64(1, "s") - dt_obj = datetime.fromtimestamp(timestamp, UTC) - if not tz_aware: - microseconds = timedelta(microseconds=dt_obj.microsecond) - dt_obj = datetime.strptime(dt_obj.strftime("%Y%m%d:%H%M%S"), "%Y%m%d:%H%M%S") + microseconds - return dt_obj - - -def _get_aoi_shapefile_bounding_box(aoi_filename: str) -> tuple: - """Read a .shp shapefile and return the bounding box. - - The shapefile is read and the area of interest is retrieved. The bounding box is then computed, and a tuple of the - coordinates of the bounding box is returned. - - Parameters - ---------- - aoi_filename: str | None - full path to the AoI shapefile, expects .shp format. - - Returns - ------- - tuple - tuple of two lists, the first containing the longitude extent, the second the latitude extent of the - bounding box. - - Raises - ------ - AssertionError - Raised when: - - the aoi_filename does not exist - - the aoi_filename does not end in .shp - - ValueError - Raised when: - - the provided shapefile contains zero polygons, or more than one polygon - - the provided shapefile contains an invalid polygon - """ - assert os.path.exists(aoi_filename), f"The file {aoi_filename} does not exist!" - assert aoi_filename.split(".")[-1] == "shp", f"The provided file {aoi_filename} is not of .shp type!" - - # open the file, and iterate through the geometry - shape = geopandas.read_file(aoi_filename) - # calculate the coordinates of the bounding box of the provided AoI - bounding_box = shape.total_bounds - # format as longitude extent (in x), latitude extent (in y) - bounding_box_formatted = ([bounding_box[0], bounding_box[2]], [bounding_box[1], bounding_box[3]]) - - return bounding_box_formatted - - -def crop_slc_spacetime( - slcs: xr.Dataset, - aoi_filename: str | None = None, - start_date: datetime | str | None = None, - end_date: datetime | str | int | None = None, -) -> xr.Dataset: - """Crop an SLC stack in both space and time. - - To crop in space an AoI shapefile is processed, and the SLC stack is cropped to the bounding box. - To crop in time, a start date is provided, and two options for the end date are available: - - datetime | str - this directly provides the end date - - int - this provides a number of SLCs intended to be in the crop. The end date is set automatically. - - If the aoi_filename is not provided and left to None, only a crop in time is performed. - If the start_date and end_date are not provided and left to None, only a crop in space is performed. - - Parameters - ---------- - slcs : xr.Dataset - the SLC stack to be cropped. Requires at least the following coordinates or variables: - In case of a crop in time: - - time -> the dates of the images - In case of a crop in space: - - lat -> the latitude of the pixels - - lon -> the longitude of the pixels - aoi_filename: str | None - full path to the AoI shapefile, expects .shp format. Set to None if no crop in space is requested. - start_date : datetime | str | None - the start date of the crop, in one of three formats: - - datetime object - - str object, formatted as YYYYMMDD - - None, no cropping in time requested - end_date : datetime | str | int | None - the end date of the crop, in one of four formats: - - datetime object - - str object, formatted as YYYYMMDD - - int object, which is interpreted as the number of images intended in the crop (including the start date). If - more images are requested than exist since the start date, all images from start_date until the last image - are provided. - - None, no cropping in time requested - - Returns - ------- - xr.Dataset - The cropped dataset - - Raises - ------ - AssertionError - Raised when: - - a start_date or end_date is provided in string format, but not in YYYYMMDD format - - a start_date is provided, but the end_date is set to None - - an end_date is provided, but the start_date is set to None - - the aoi_filename does not exist - - the aoi_filename does not end in .shp - - one of the required coordinates or variables is not available in slcs - - ValueError - Raised when: - - start_date is not of type datetime | str | None - - end_date is not of type datetime | str | int | None - """ - # Check the input - - if aoi_filename is not None: - assert os.path.exists(aoi_filename), f"The file {aoi_filename} does not exist!" - assert aoi_filename.split(".")[-1] == "shp", f"The provided file {aoi_filename} is not of .shp type!" - for axis in ["lat", "lon"]: - assert axis in slcs.keys(), f"Expected axis {axis} in SLCs but it is not present!" - - # convert the input to a timezone-aware datetime object - if isinstance(start_date, str): - assert len(start_date) == 8, f"Unknown start_date format {start_date}, expected YYYYMMDD!" - format_start_date = datetime( - eval(start_date[:4]), eval(start_date[4:6].lstrip("0")), eval(start_date[6:].lstrip("0")), tzinfo=pytz.UTC - ) - elif isinstance(start_date, datetime): - format_start_date = datetime(start_date.year, start_date.month, start_date.day, tzinfo=pytz.UTC) - elif start_date is None: - assert end_date is None, f"Start date is None while end date is {end_date} (not None!)" - format_start_date = None - else: - raise ValueError(f'Expected start_date of type "str" | "datetime" | None, got {type(start_date)}!') - - if isinstance(end_date, str): - assert len(end_date) == 8, f"Unknown end_date format {end_date}, expected YYYYMMDD!" - format_end_date = datetime( - eval(end_date[:4]), eval(end_date[4:6].lstrip("0")), eval(end_date[6:].lstrip("0")), tzinfo=pytz.UTC - ) - elif isinstance(end_date, datetime): - format_end_date = datetime(end_date.year, end_date.month, end_date.day, tzinfo=pytz.UTC) - elif isinstance(end_date, int): - fmt_dates = [npdatetime64_to_datetime(date) for date in slcs["time"].values] - valid_dates = [date for date in fmt_dates if date >= format_start_date] - end_idx = fmt_dates.index(valid_dates[0]) + end_date - 1 - end_idx = min(end_idx, len(fmt_dates) - 1) - format_end_date = fmt_dates[end_idx] - elif end_date is None: - assert start_date is None, f"Start date is None while end date is {end_date} (not None!)" - format_end_date = None - else: - raise ValueError(f'Expected end_date of type "str" | "datetime" | "int" | None, got {type(end_date)}!') - - # TIME CROP - if format_start_date is not None and format_end_date is not None: - # first the last assertion - assert "time" in slcs.keys(), "Expected axis 'time' in SLCs but it is not present!" - - fmt_dates_raw = [npdatetime64_to_datetime(date) for date in slcs["time"].values] - fmt_dates = np.array([datetime(date.year, date.month, date.day, tzinfo=pytz.UTC) for date in fmt_dates_raw]) - time_mask = (format_start_date <= fmt_dates) & (fmt_dates <= format_end_date) - slcs = slcs.sel(time=slcs["time"].values[time_mask]) - - # SPACE CROP - if aoi_filename is not None: - bounding_box = _get_aoi_shapefile_bounding_box(aoi_filename) - space_mask = ( - (slcs["lat"] >= min(bounding_box[1])) - & (slcs["lat"] <= max(bounding_box[1])) - & (slcs["lon"] >= min(bounding_box[0])) - & (slcs["lon"] <= max(bounding_box[0])) - ) - - comp_space_mask = space_mask.compute() - az_sum = comp_space_mask.sum(dim="azimuth") - rg_sum = comp_space_mask.sum(dim="range") - - # first and last non zero - min_range, max_range = ( - az_sum.where(az_sum > 0, drop=True)["range"].min().values, - az_sum.where(az_sum > 0, drop=True)["range"].max().values, - ) - min_azimuth, max_azimuth = ( - rg_sum.where(rg_sum > 0, drop=True)["azimuth"].min().values, - rg_sum.where(rg_sum > 0, drop=True)["azimuth"].max().values, - ) - # data at original locations not nan - slcs = slcs.sel(azimuth=range(min_azimuth, max_azimuth), range=range(min_range, max_range)) - - return slcs - - -def project_stm_coordinates(stm: xr.Dataset, projection: str = "RD") -> xr.Dataset: - """Project the latitude and longitude of a space-time matrix to another reference frame. - - The latitude and longitude layers are transformed into the desired projection, default Rijksdriehoek or RD. - - Parameters - ---------- - stm: xr.Dataset - Space-time matrix with the layers `lat` (latitude) and `lon` (longitude in WGS84 (EPSG:4326), and coordinate - `space` - projection: str, optional - Projection to which the latitude and longitude coordinates should be transformed. "RD" defaults to "EPSG:28992". - Default "RD" - - Returns - ------- - xr.Dataset - Space-time matrix with the added layers `projection_x` and `projection_y`, where projection is the requested - parameter `projection` in lower case. - - Raises - ------ - AssertionError - When layers "lon" or "lat" do not exist in `stm`. - """ - assert "lon" in stm.keys(), "Expected a space-time matrix with longitude layer named lon but it is not there!" - assert "lat" in stm.keys(), "Expected a space-time matrix with latitude layer named lat but it is not there!" - if projection == "RD": - projection_formatted = "EPSG:28992" - elif projection[:5] == "EPSG:": - projection_formatted = projection - else: - raise ValueError(f"Invalid projection provided! Expected 'RD' or 'EPSG:###' but got {projection}!") - - wgs84 = pyproj.Transformer.from_crs("EPSG:4326", projection_formatted, always_xy=True).transform - # Convert Lat and Lon to coordinates - proj_x, proj_y = wgs84(stm["lon"], stm["lat"]) - - # Add coordinates to the dataset - stm = stm.assign({f"{projection.lower()}_x": (["space"], proj_x)}) - stm = stm.assign({f"{projection.lower()}_y": (["space"], proj_y)}) - - return stm - - -def add_stm_time_deltas(stm: xr.Dataset) -> xr.Dataset: - """Add the time differences since the first image to a space-time matrix. - - Parameters - ---------- - stm: xr.Dataset - the space-time matrix with an axis "time" - - Returns - ------- - xr.Dataset - the space-time matrix with two new variables: - - `days_since_first_img`, the number of days since the first epoch in the STM - - `years_since_first_img`, the number of years since the first epoch in the STM, assuming 365.2425 days per year - - """ - assert "time" in stm.keys(), "Expected STM to have time axis but it's not there!" - # Add extra time coordinate variables for time intervals since first image - days = np.array( - [ - (npdatetime64_to_datetime(date) - npdatetime64_to_datetime(stm["time"].values[0])).days - for date in stm["time"].values - ] - ) - stm = stm.assign({"days_since_first_img": (["time"], days)}) - stm = stm.assign({"years_since_first_img": (["time"], days / 365.2425)}) - - return stm - - -def stm_compute_single_time_differences( - stm: xr.Dataset, single_difference_mother: str | datetime = "auto" -) -> xr.Dataset: - """Compute the single differences of an STM in time with respect to a given mother image. - - This computes the single difference complex value, phase, unnormalized amplitude, and h2ph values with respect - to the provided single difference mother. The mother image is the first image acquired on or after the provided - date (if a datetime object or str object is provided), or the mother image of the input dataset (if 'auto' mode - is selected). - - Parameters - ---------- - stm: xr.Dataset - the space-time matrix with an axis "time" and "space", and variables `h2ph` and `complex` - single_difference_mother: datetime | str - the date to be used as the mother image for the single difference computations, in one of three formats: - - 'auto' : will detect the mother image in the input SLC dataset, and use that epoch. - - datetime object - - str object, formatted as YYYYMMDD - - Returns - ------- - xr.Dataset - the space-time matrix with four new variables: - - sd_h2ph (space, time): single difference height to phase conversion with respect to single_difference_mother - - sd_complex (space, time): single difference complex phasor with respect to single_difference_mother - - sd_amplitude_unnormalized (space, time): single difference complex phasor amplitude to - single_difference_mother, not normalized - - sd_phase (space, time): single difference phase with respect to single_difference_mother - - Raises - ------ - ValueError - Raised when: - - single_difference_mother is of an unsupported format - - the date provided to single_difference_mother is not in the input stack date range - """ - # Identify the mother image - if isinstance(single_difference_mother, datetime): - format_mother_date = datetime( - single_difference_mother.year, single_difference_mother.month, single_difference_mother.day, tzinfo=pytz.UTC - ) - mother_index = [ - idx for idx, date in enumerate(stm["time"].values) if format_mother_date <= npdatetime64_to_datetime(date) - ] # select all images beyond the mother date - elif isinstance(single_difference_mother, str): - if single_difference_mother == "auto": - mother_index = np.where(abs(stm["h2ph"]).sum(axis=0).values == 0)[0] - elif len(single_difference_mother) == 8: - format_mother_date = datetime( - eval(single_difference_mother[:4]), - eval(single_difference_mother[4:6].lstrip("0")), - eval(single_difference_mother[6:].lstrip("0")), - tzinfo=pytz.UTC, - ) - mother_index = [ - idx - for idx, date in enumerate(stm["time"].values) - if format_mother_date <= npdatetime64_to_datetime(date) - ] # select all images beyond the mother date - else: - raise ValueError(f'Cannot parse {single_difference_mother}, not of type "auto" or "YYYYMMDD"!') - else: - raise ValueError(f"Unknown format {type(single_difference_mother)} for single_difference_mother!") - if len(mother_index) == 0: - raise ValueError( - f"Cannot find provided mother date {single_difference_mother}, " - "please provide a date that is within the range of the stack! Possible dates: " - f"{stm.time.values[0]}--{stm.time.values[-1]}" - ) - sd_mother_index = mother_index[0] # 0 in case more than 1 image is detected - # In that case we take the first image that was detected, as this is expected - sd_mother = npdatetime64_to_datetime(stm["time"].values[sd_mother_index]) - - # Format the single difference mother, and save it to the STM - stm.attrs["ps_sd_mother"] = sd_mother.strftime("%Y%m%d") - - # calculate the h2ph single difference (= daughter - mother) - sd_h2ph = stm["h2ph"] - stm["h2ph"][:, sd_mother_index] - stm = stm.assign({"sd_h2ph": (["space", "time"], sd_h2ph.data)}) - - # calculate the complex single difference, the amplitude, and the phase - mother_comp = stm["complex"][:, sd_mother_index].conj() - sd_complex_transposed = stm["complex"].transpose() * mother_comp - sd_complex = sd_complex_transposed.transpose() - sd_phase = da.angle(sd_complex) - sd_amplitude_unnormalized = da.abs(sd_complex) - stm = stm.assign({"sd_complex": (["space", "time"], sd_complex.data)}) - stm = stm.assign({"sd_amplitude_unnormalized": (["space", "time"], sd_amplitude_unnormalized.data)}) - stm = stm.assign({"sd_phase": (["space", "time"], sd_phase.data)}) - - return stm - - -def identify_s1_orbits_in_aoi(lon: list | np.ndarray, lat: list | np.ndarray) -> tuple[list[str], dict]: - """Identify the Sentinel-1 orbit numbers and directions crossing a AoI. - - Parameters - ---------- - lon: list | np.ndarray - List of all the longitudes of all the points of interest in the AoI - lat: list | np.ndarray - List of all the latitudes of all the points of interest in the AoI - - Returns - ------- - list - The orbits overlapping with the AoI - dict - The footprints of the overlapping SLCs per track - """ - bbox = [[np.min(lon), np.max(lon)], [np.min(lat), np.max(lat)]] - wkt = ( - f"POLYGON((" - f"{bbox[0][0]} {bbox[1][0]}, " - f"{bbox[0][1]} {bbox[1][0]}, " - f"{bbox[0][1]} {bbox[1][1]}, " - f"{bbox[0][0]} {bbox[1][1]}, " - f"{bbox[0][0]} {bbox[1][0]}))" - ) - slcs = None - counter = 0 - while slcs is None: - try: - slcs = asf.geo_search( - intersectsWith=wkt, - platform=asf.PLATFORM.SENTINEL1, - beamMode="IW", - processingLevel="SLC", - start="one month ago", - end="now", - ) - except (asf.exceptions.ASFSearch5xxError, asf.exceptions.ASFSearchError, TimeoutError): - counter += 1 - print(f"ASF encountered an internal error. Retrying... (#{counter})") - - orbits = [ - f"s1_{slc.properties['flightDirection'].lower().replace('e', '')[:3]}_t{slc.properties['pathNumber']:0>3d}" - for slc in slcs - ] - filtered_orbits = list(sorted(list(set(orbits)))) - - extents = [slc.geojson()["geometry"]["coordinates"][0] for slc in slcs] - footprints = {} - for orbit in filtered_orbits: - footprints[orbit] = [] - - for extent in range(len(extents)): - footprints[orbits[extent]].append(extents[extent]) - - return filtered_orbits, footprints - - -def generate_pnt_uids(stm: xr.Dataset, ensure_unique: bool = True, overwrite: bool = False) -> xr.Dataset: - """Generate unique identifiers based on radar coordinates and assign them to the STM. - - The unique identifiers are assigned as a new coordinate "pnt_uid" in the STM. - - Parameters - ---------- - stm: xr.Dataset - The space-time matrix with coordinate "azimuth" and "range". - ensure_unique: bool, optional - Whether to ensure that the generated unique identifiers are unique. Default is True. - When True, numpy.unique is used to check for uniqueness and raise an error if duplicates are found. - For very large STMs, this can be computationally expensive. Consider setting to False if the radar - coordinates are known to be unique. - overwrite: bool, optional - Whether to overwrite existing "pnt_uid" coordinate in the STM. Default is False. - If False and "pnt_uid" already exists in coordinates or data variables, a warning - is logged and the STM is returned unchanged. - If True and "pnt_uid" exists in data variables, it is dropped before generating new identifiers. - - Returns - ------- - xr.Dataset - The space-time matrix with an added unique identifier coordinate "pnt_uid" - """ - # Copy the input STM to avoid modifying it directly - stm_output = stm.copy() - - # Check if pnt_uid already exists - if "pnt_uid" in stm.coords and not overwrite: - warning_msg = ( - "No pnt_uid has been generated. " - "STM already contains 'pnt_uid' coordinate. " - "Set 'overwrite=True' to regenerate unique identifiers." - ) - logger.warning(warning_msg) - return stm_output - - # Check if pnt_uid is in data variables - if "pnt_uid" in stm.data_vars: - if not overwrite: - warning_msg = ( - "No pnt_uid has been generated. " - "STM already contains 'pnt_uid' data variable. " - "Setting 'overwrite=True' will drop this data variable " - "and regenerate unique identifiers as coordinates." - ) - logger.warning(warning_msg) - return stm_output - else: - stm_output = stm_output.drop_vars("pnt_uid") - - # Check input: - # stm should have coordinates "azimuth" and "range" - # they should only have space dimension - # there should be no nan values in these coordinates - for key_dim in ["azimuth", "range"]: - assert key_dim in stm.coords, f"Expected STM to have coordinate '{key_dim}'." - assert stm[key_dim].dims == ("space",), f"Coordinate '{key_dim}' should have and only have 'space' dimension." - assert not np.any(np.isnan(stm[key_dim].values)), f"Coordinate '{key_dim}' contains NaN values." - - # Generate unique identifiers - # This is done by pandas hashing the azimuth and range coordinates together - # Index is set to False to avoid including the index in the hash - # reset coords to avoid involving other coords in the hash - df = stm.reset_coords()[["azimuth", "range"]].to_dataframe() - uid = pd.util.hash_pandas_object(df, index=False).values - - # Ensure uniqueness if requested - if ensure_unique: - unique_uids = np.unique(uid) - if unique_uids.shape[0] != uid.shape[0]: - logger.error("Duplicate unique identifiers detected in STM!") - raise ValueError("Generated unique identifiers are not unique. Check radar coordinates for duplicates.") - - # Assign unique identifiers to the STM - stm_output = stm_output.assign_coords({"pnt_uid": (["space"], uid)}) - - return stm_output - - -def compute_phase_difference( - value_source: np.ndarray, - value_target: np.ndarray, - method: Literal["subtract", "conjmult"], -) -> np.ndarray: - """Calculate the phase difference between two STMs. - - When method is "subtract", the expected input values are phases, and the phase difference is calculated - by simple subtraction. - When method is "conjmult", the expected input values are complex values, and the phase difference is - calculated by conjugate multiplication of the complex values, then taking the angle of the result. - - Parameters - ---------- - value_source: np.ndarray - The values of the source STM. - value_target: np.ndarray - The values of the target STM. - method: Literal["subtract", "conjmult"] - The method to calculate the phase difference. Can be either "subtract" or "conjmult". - - Returns - ------- - np.ndarray - Array containing the phase differences between ``value_target`` and ``value_source``, - computed according to the selected method. - """ - if method == "subtract": - d_phase = value_target - value_source - elif method == "conjmult": - # check that the input values are complex - if not np.iscomplexobj(value_source) or not np.iscomplexobj(value_target): - raise ValueError("Input values must be complex when using 'conjmult' method.") - d_phase = np.angle(value_target * value_source.conj()) - else: - raise NotImplementedError(f"Unknown difference method {method}, known are subtract and conjmult") - - return d_phase - - -def convert_geographic_coords_to_euclidean( - lon: np.ndarray | list, - lat: np.ndarray | list, - target_crs: str = "EPSG:28992", -) -> tuple[np.ndarray, np.ndarray]: - """Convert geographic coordinates (lon, lat) to projected Euclidean coordinates (x, y). - - Parameters - ---------- - lon: np.ndarray | list - Longitudes in degrees. - lat: np.ndarray | list - Latitudes in degrees. - target_crs: str, optional - Target coordinate reference system in EPSG format. Default is - "EPSG:28992" (Amersfoort / RD New). - - Returns - ------- - tuple[np.ndarray, np.ndarray] - Tuple containing: - - x: Projected x coordinates in meters. - - y: Projected y coordinates in meters. - """ - transformer = pyproj.Transformer.from_crs("EPSG:4326", target_crs, always_xy=True) - x, y = transformer.transform(lon, lat) - return np.array(x), np.array(y) - - -def concatenate_stms( - stm_list: list[xr.Dataset], -) -> xr.Dataset: - """Concatenate multiple STMs along the space dimension. - - This function concatenates a list of xarray Datasets representing space-time matrices (STMs) - along the space dimension. - It ensures that all STMs have the required dimensions and coordinates, and handles time-only variables appropriately - to prevent broadcasting issues during concatenation. - If the same time-only variable exists in multiple STMs, it is assumed to be identical across those STMs. Otherwise, - an error will be raised during the concatenation process. - If a time-only variable exists in only a subset of the STMs, it will be included in the final concatenated STM, - assuming this data variable applies to all points in the concatenated STM. - - Parameters - ---------- - stm_list : list[xr.Dataset] - List of xarray Datasets representing STMs to concatenate. - - Returns - ------- - xr.Dataset - Concatenated STM along the space dimension. - """ - # Check stm_list is a non-empty list - assert isinstance(stm_list, list) and len(stm_list) > 0, "stm_list must be a non-empty list of xarray Datasets." - - # All STMs should have space and time dimensions - for idx, stm in enumerate(stm_list): - assert isinstance(stm, xr.Dataset), f"STM at index {idx} is not a Dataset." - assert "space" in stm.dims, f"STM at index {idx} does not have 'space' dimension." - assert "time" in stm.dims, f"STM at index {idx} does not have 'time' dimension." - - # All STMs should have the same size for the time dimension - reference_time_size = stm_list[0].sizes["time"] - for idx, stm in enumerate(stm_list[1:], start=1): - current_time_size = stm.sizes["time"] - assert current_time_size == reference_time_size, ( - f"STM at index {idx} has time dimension size {current_time_size}, expected {reference_time_size}." - ) - - # All coordinates of all STMs should be 1D, only space or time dimension - for idx, stm in enumerate(stm_list): - for coord in stm.coords: - assert len(stm[coord].dims) == 1, ( - f"Coordinate '{coord}' in STM at index {idx} is not 1D (dims {stm[coord].dims})." - ) - - # Identify and temporarily convert time-only variables to coords to prevent broadcasting during concat - time_only_vars = set() - for ds in stm_list: - time_only_vars.update(v for v in ds.data_vars if ds[v].dims == ("time",)) - - # Set identified time-only variables as coordinates in each dataset where they appear - for idx, ds in enumerate(stm_list): - for var in time_only_vars: - if var in ds.data_vars: - ds = ds.set_coords(var) - stm_list[idx] = ds - - # Concat in space dimension with all (space, time) data variables, filling NaNs for missing vars - stm_dens_pnts_output = xr.concat(stm_list, dim="space", data_vars="all") - - # Reset coordinates back to data variables; only reset those that are still coordinates - coords_to_reset = [v for v in time_only_vars if v in stm_dens_pnts_output.coords] - stm_dens_pnts_output = stm_dens_pnts_output.reset_coords(names=coords_to_reset, drop=False) - - return stm_dens_pnts_output - - -def get_m2ph(wavelength: float = WAVELENGTH_S1): - """Get the conversion factor from meters to phase.""" - return -4 * np.pi / wavelength