[10.x] fix(azure): retry torn membership boundary reads

(cherry picked from commit 9fd511c)

Author: ReubenBond

src/Azure/Orleans.Clustering.AzureStorage/OrleansSiloInstanceManager.cs

@@ -338,15 +338,15 @@ private static bool CanAcceptSnapshot(List<(SiloInstanceTableEntry Entity, strin
             var closing = entries[^1].Entity.RowKey == SiloInstanceTableEntry.TABLE_VERSION_ROW_MAX;
             // Both markers absent identifies the legacy format, which uses legacy read semantics.
             // The first membership write atomically establishes both markers with the new version.
-            // Once either marker is observed, both are required to validate the snapshot.
+            // A query overlapping that write can see a partial pair, so retry partial or mismatched pairs.
             if (!opening && !closing)
             {
                 return true;
             }
 
             if (!opening || !closing)
             {
-                throw new KeyNotFoundException("The membership query must include both ordered boundary version rows.");
+                return false;
             }
 
             var before = entries[0].Entity.MembershipVersion

test/Extensions/Orleans.Azure.Tests/AzureMembershipPaginationTests.cs

@@ -125,19 +125,43 @@ public async Task PerpetualChurnFailsAfterBoundWithClusterContext()
     [Theory]
     [InlineData(true, false)]
     [InlineData(false, true)]
-    public async Task SingleMissingBoundaryRowFails(bool opening, bool closing)
+    public async Task SingleMissingBoundaryRowRetries(bool opening, bool closing)
     {
         var storage = new ScriptedMembershipTableReadStorage();
-        var query = FencedQuery(1, Silo("silo-1", "s1"));
+        var query = FencedQuery(1, Silo("silo-1", "stale"));
         if (!opening) query.Entries.RemoveAt(0);
         if (!closing) query.Entries.RemoveAt(query.Entries.Count - 1);
         storage.AddQuery(query);
+        storage.AddQuery(FencedQuery(2, Silo("silo-1", "current")));
 
-        var exception = await Assert.ThrowsAsync<KeyNotFoundException>(
+        var result = await CreateManager(storage).FindAllSiloEntries(TestContext.Current.CancellationToken);
+
+        Assert.Equal("current", result.Single(entry => entry.Entity.RowKey == "silo-1").ETag);
+        Assert.Equal("2", result.Single(entry => entry.Entity.RowKey == SiloInstanceTableEntry.TABLE_VERSION_ROW).Entity.MembershipVersion);
+        Assert.Equal(2, storage.QueryCount);
+    }
+
+    [Theory]
+    [InlineData(true, false)]
+    [InlineData(false, true)]
+    public async Task PersistentSingleBoundaryRowFailsAfterBound(bool opening, bool closing)
+    {
+        var storage = new ScriptedMembershipTableReadStorage();
+        for (var attempt = 0; attempt < OrleansSiloInstanceManager.MaxMembershipSnapshotAttempts; attempt++)
+        {
+            var query = FencedQuery(1, Silo("silo-1", "s1"));
+            if (!opening) query.Entries.RemoveAt(0);
+            if (!closing) query.Entries.RemoveAt(query.Entries.Count - 1);
+            storage.AddQuery(query);
+        }
+
+        var exception = await Assert.ThrowsAsync<InconsistentStateException>(
             () => CreateManager(storage).FindAllSiloEntries(TestContext.Current.CancellationToken));
 
-        Assert.Equal("The membership query must include both ordered boundary version rows.", exception.Message);
-        Assert.Equal(1, storage.QueryCount);
+        Assert.Equal(
+            $"Unable to read a consistent membership snapshot for cluster '{ClusterId}' from table '{TableName}' after {OrleansSiloInstanceManager.MaxMembershipSnapshotAttempts} attempts.",
+            exception.Message);
+        Assert.Equal(OrleansSiloInstanceManager.MaxMembershipSnapshotAttempts, storage.QueryCount);
     }
 
     [Theory]