diff --git a/src/cellcharter/gr/_build.py b/src/cellcharter/gr/_build.py
index b9b1170..e6f75e2 100644
--- a/src/cellcharter/gr/_build.py
+++ b/src/cellcharter/gr/_build.py
@@ -75,6 +75,9 @@ def remove_long_links(
 
 
 def _remove_intra_cluster_links(labels, adjacency):
+    if not adjacency.data.flags.writeable:
+        adjacency = adjacency.copy()
+
     target_labels = np.array(labels.iloc[adjacency.indices])
     source_labels = np.array(
         labels.iloc[np.repeat(np.arange(adjacency.indptr.shape[0] - 1), np.diff(adjacency.indptr))]
@@ -132,11 +135,15 @@ def remove_intra_cluster_links(
     conns = adata.obsp[connectivity_key].copy() if copy else adata.obsp[connectivity_key]
     dists = adata.obsp[distances_key].copy() if copy else adata.obsp[distances_key]
 
-    conns, dists = (_remove_intra_cluster_links(adata.obs[cluster_key], adjacency) for adjacency in [conns, dists])
+    conns = _remove_intra_cluster_links(adata.obs[cluster_key], conns)
+    dists = _remove_intra_cluster_links(adata.obs[cluster_key], dists)
 
     if copy:
         return conns, dists
 
+    adata.obsp[connectivity_key] = conns
+    adata.obsp[distances_key] = dists
+
 
 def _connected_components(adj: sps.spmatrix, min_cells: int = 250, count: int = 0) -> np.ndarray:
     n_components, labels = sps.csgraph.connected_components(adj, return_labels=True)
diff --git a/src/cellcharter/gr/_nhood.py b/src/cellcharter/gr/_nhood.py
index b39e47b..9c30b2a 100644
--- a/src/cellcharter/gr/_nhood.py
+++ b/src/cellcharter/gr/_nhood.py
@@ -70,6 +70,12 @@ def _observed_permuted(adj, labels, observed, symmetric=True, batch_size=1):
     return results
 
 
+def _with_diagonal(df: pd.DataFrame, value: float) -> pd.DataFrame:
+    values = df.to_numpy(copy=True)
+    np.fill_diagonal(values, value)
+    return pd.DataFrame(values, columns=df.columns, index=df.index)
+
+
 def _nhood_enrichment(
     adj,
     labels,
@@ -83,7 +89,7 @@ def _nhood_enrichment(
     observed_expected=False,
 ):
     if only_inter:
-        adj = _remove_intra_cluster_links(labels, adj)
+        adj = _remove_intra_cluster_links(labels, adj.copy())
 
     cluster_categories = labels.cat.categories
 
@@ -140,9 +146,9 @@ def _nhood_enrichment(
 
     enrichment = np.log2(observed / expected) if log_fold_change else observed - expected
     if only_inter:
-        np.fill_diagonal(observed.values, np.nan)
-        np.fill_diagonal(expected.values, np.nan)
-        np.fill_diagonal(enrichment.values, np.nan)
+        observed = _with_diagonal(observed, np.nan)
+        expected = _with_diagonal(expected, np.nan)
+        enrichment = _with_diagonal(enrichment, np.nan)
 
     result = {"enrichment": enrichment}
 
diff --git a/tests/conftest.py b/tests/conftest.py
index 70c245d..7688d2f 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -45,7 +45,7 @@ def _download_codex(path: Path) -> None:
 @pytest.fixture()
 def non_visium_adata() -> ad.AnnData:
     non_visium_coords = np.array([[1, 0], [3, 0], [5, 6], [0, 4]])
-    adata = ad.AnnData(X=non_visium_coords, dtype=int)
+    adata = ad.AnnData(X=non_visium_coords.astype(int))
     adata.obsm[Key.obsm.spatial] = non_visium_coords
     return adata