Skip to content

Commit 19bab0e

Browse files
caol-ilacaol-ila
authored
Merge pull request #90 from github/caol-ila-follower-configs
Add configurations for follower clusters
2 parents 455541e + 7bbdaeb commit 19bab0e

File tree

11 files changed

+354
-10
lines changed

11 files changed

+354
-10
lines changed

‎KustoSchemaTools/Changes/DatabaseChanges.cs‎

Lines changed: 121 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1-
using KustoSchemaTools.Model;
1+
using Kusto.Language;
2+
using KustoSchemaTools.Model;
23
using Microsoft.Extensions.Logging;
34
using System.Data;
5+
using System.Text;
46

57
namespace KustoSchemaTools.Changes
68
{
@@ -175,6 +177,124 @@ private static List<IChange> GenerateScriptCompareChanges<T>(Database oldState,
175177

176178
return tmp;
177179
}
180+
181+
public static List<IChange> GenerateFollowerChanges(FollowerDatabase oldState, FollowerDatabase newState, ILogger log)
182+
{
183+
List<IChange> result =
184+
[
185+
.. GenerateFollowerCachingChanges(oldState, newState, db => db.Tables, "Table", "table"),
186+
.. GenerateFollowerCachingChanges(oldState, newState, db => db.MaterializedViews, "MV", "materialized-view"),
187+
188+
];
189+
190+
if (oldState.Permissions.ModificationKind != newState.Permissions.ModificationKind)
191+
{
192+
var kind = newState.Permissions.ModificationKind.ToString().ToLower();
193+
result.Add(new BasicChange("FollowerDatabase", "PermissionsModificationKind", $" Change Permission-Modification-Kind from {oldState.Permissions.ModificationKind} to {newState.Permissions.ModificationKind}", new List<DatabaseScriptContainer>
194+
{
195+
new DatabaseScriptContainer(new DatabaseScript($".alter follower database {newState.DatabaseName} principals-modification-kind = {kind}", 0), "FollowerChangePolicyModificationKind")
196+
}));
197+
}
198+
if (oldState.Cache.ModificationKind != newState.Cache.ModificationKind)
199+
{
200+
var kind = newState.Cache.ModificationKind.ToString().ToLower();
201+
result.Add(new BasicChange("FollowerDatabase", "ChangeModificationKind", $"Change Caching-Modification-Kind from {oldState.Cache.ModificationKind} to {newState.Cache.ModificationKind}", new List<DatabaseScriptContainer>
202+
{
203+
new DatabaseScriptContainer(new DatabaseScript($".alter follower database {newState.DatabaseName} caching-policies-modification-kind = {kind}", 0), "FollowerChangePolicyModificationKind")
204+
}));
205+
}
206+
207+
if (oldState.Cache.DefaultHotCache != newState.Cache.DefaultHotCache)
208+
{
209+
if (newState.Cache.DefaultHotCache != null)
210+
{
211+
result.Add(new BasicChange("FollowerDatabase", "ChangeDefaultHotCache", $"From {oldState.Cache.DefaultHotCache} to {newState.Cache.DefaultHotCache}", new List<DatabaseScriptContainer>
212+
{
213+
new DatabaseScriptContainer(new DatabaseScript($".alter follower database {newState.DatabaseName} policy caching hot = {newState.Cache.DefaultHotCache}", 0), "FollowerChangeDefaultHotCache")
214+
}));
215+
}
216+
else
217+
{
218+
result.Add(new BasicChange("FollowerDatabase", "DeleteDefaultHotCache", $"Remove Default Hot Cache", new List<DatabaseScriptContainer>
219+
{
220+
new DatabaseScriptContainer(new DatabaseScript($".delete follower database {newState.DatabaseName} policy caching", 0), "FollowerDeleteDefaultHotCache")
221+
}));
222+
}
223+
}
224+
225+
foreach(var script in result.SelectMany(itm => itm.Scripts))
226+
{
227+
var code = KustoCode.Parse(script.Text);
228+
var diagnostics = code.GetDiagnostics();
229+
script.IsValid = diagnostics.Any() == false;
230+
}
231+
232+
return result;
233+
234+
}
235+
236+
private static List<IChange> GenerateFollowerCachingChanges(FollowerDatabase oldState, FollowerDatabase newState, Func<FollowerCache, Dictionary<string,string>> selector, string type, string kustoType)
237+
{
238+
var result = new List<IChange>();
239+
var oldEntities = selector(oldState.Cache);
240+
var newEntities = selector(newState.Cache);
241+
242+
243+
var removedPolicyScripts = oldEntities.Keys.Except(newEntities.Keys)
244+
.Select(itm =>
245+
new
246+
{
247+
Name = itm,
248+
Script = new DatabaseScriptContainer(new DatabaseScript($".delete follower database {newState.DatabaseName} {kustoType} {itm} policy caching", 0), $"FollowerDelete{type}CachingPolicies")
249+
})
250+
.ToList();
251+
var changedPolicyScripts = newEntities
252+
.Where(itm => oldEntities.ContainsKey(itm.Key) == false
253+
|| oldEntities[itm.Key] != itm.Value)
254+
.Select(itm => new
255+
{
256+
Name = itm.Key,
257+
From = oldEntities.ContainsKey(itm.Key) ? oldEntities[itm.Key] : "default",
258+
To = itm.Value,
259+
Script = new DatabaseScriptContainer(new DatabaseScript($".alter follower database {newState.DatabaseName} {kustoType} {itm.Key} policy caching hot = {itm.Value}", 0), $"FollowerChange{type}CachingPolicies")
260+
})
261+
.ToList();
262+
263+
if (removedPolicyScripts.Any())
264+
{
265+
var deletePolicySb = new StringBuilder();
266+
deletePolicySb.AppendLine($"## Delete {type} Caching Policies");
267+
foreach (var change in removedPolicyScripts)
268+
{
269+
deletePolicySb.AppendLine($"* {change.Name}");
270+
}
271+
272+
result.Add(new BasicChange(
273+
"FollowerDatabase",
274+
$"Delete{type}CachingPolicy",
275+
deletePolicySb.ToString(),
276+
removedPolicyScripts.Select(itm => itm.Script).ToList()));
277+
}
278+
if (changedPolicyScripts.Any())
279+
{
280+
var changePolicies = new StringBuilder();
281+
changePolicies.AppendLine($"## Changed {type} Caching Policies");
282+
changePolicies.AppendLine($"{type} | From | To");
283+
changePolicies.AppendLine("--|--|--");
284+
foreach (var change in changedPolicyScripts)
285+
{
286+
changePolicies.AppendLine($"{change.Name} | {change.From} | {change.To}");
287+
}
288+
289+
result.Add(new BasicChange(
290+
"FollowerDatabase",
291+
$"Change{type}CachingPolicy",
292+
changePolicies.ToString(),
293+
changedPolicyScripts.Select(itm => itm.Script).ToList()));
294+
}
295+
296+
return result;
297+
}
178298
}
179299

180300
}

‎KustoSchemaTools/Changes/FollowerChange.cs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Text;
5+
using System.Threading.Tasks;
6+
7+
namespace KustoSchemaTools.Changes
8+
{
9+
public class BasicChange : IChange
10+
{
11+
12+
public string EntityType { get; set; }
13+
14+
public BasicChange(string entityType, string entity, string markdown, List<DatabaseScriptContainer> scripts)
15+
{
16+
EntityType = entityType;
17+
Entity = entity;
18+
Markdown = markdown;
19+
Scripts = scripts;
20+
}
21+
22+
public string Entity { get; set; }
23+
24+
public string Markdown { get; set; }
25+
26+
public List<DatabaseScriptContainer> Scripts { get; set; }
27+
28+
}
29+
}

‎KustoSchemaTools/KustoSchemaHandler.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,13 @@
22
using KustoSchemaTools.Helpers;
33
using KustoSchemaTools.Model;
44
using KustoSchemaTools.Parser;
5+
using KustoSchemaTools.Parser.KustoLoader;
56
using Microsoft.Extensions.Logging;
67
using System.Collections.Concurrent;
78
using System.Data;
9+
using System.Diagnostics.Metrics;
810
using System.Text;
11+
using System.Threading.Channels;
912

1013
namespace KustoSchemaTools
1114
{
@@ -65,6 +68,28 @@ public KustoSchemaHandler(ILogger<KustoSchemaHandler<T>> schemaHandlerLogger, Ya
6568

6669
Log.LogInformation($"Following scripts will be applied:\n{scriptSb}");
6770
}
71+
72+
foreach(var follower in yamlDb.Followers)
73+
{
74+
75+
Log.LogInformation($"Generating diff markdown for {Path.Combine(path, databaseName)} => {follower.Key}/{databaseName}");
76+
77+
78+
var followerClient = new KustoClient(follower.Key);
79+
var oldModel = FollowerLoader.LoadFollower(follower.Value.DatabaseName, followerClient);
80+
81+
var newModel = follower.Value;
82+
83+
var changes = DatabaseChanges.GenerateFollowerChanges(oldModel, newModel, Log);
84+
85+
sb.AppendLine($"# Changes for follower database {follower.Key}/{databaseName}");
86+
sb.AppendLine();
87+
foreach (var change in changes)
88+
{
89+
sb.AppendLine(change.Markdown);
90+
sb.AppendLine();
91+
}
92+
}
6893
return (sb.ToString(), isValid);
6994
}
7095

‎KustoSchemaTools/Model/Database.cs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
namespace KustoSchemaTools.Model
1+
using KustoSchemaTools.Changes;
2+
using System.Diagnostics.CodeAnalysis;
3+
4+
namespace KustoSchemaTools.Model
25
{
36
public class Database
47
{
@@ -31,6 +34,7 @@ public class Database
3134

3235
public Deletions Deletions { get; set; } = new Deletions();
3336

34-
}
37+
public Dictionary<string, FollowerDatabase> Followers { get; set; } = new Dictionary<string, FollowerDatabase>();
3538

39+
}
3640
}

‎KustoSchemaTools/Model/FollowerCache.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
namespace KustoSchemaTools.Model
2+
{
3+
public class FollowerCache
4+
{
5+
public string? DefaultHotCache { get; set; }
6+
public FollowerModificationKind ModificationKind { get; set; }
7+
public Dictionary<string, string> Tables { get; set; } = new Dictionary<string, string>();
8+
public Dictionary<string, string> MaterializedViews { get; set; } = new Dictionary<string, string>();
9+
}
10+
11+
}

‎KustoSchemaTools/Model/FollowerDatabase.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
namespace KustoSchemaTools.Model
2+
{
3+
public class FollowerDatabase
4+
{
5+
public string DatabaseName { get; set; }
6+
public FollowerCache Cache { get; set; } = new FollowerCache();
7+
// TODO: No logic to load data / roll out changes implemented yet!
8+
public FollowerPermissions Permissions { get; set; } = new FollowerPermissions();
9+
}
10+
11+
}

‎KustoSchemaTools/Model/FollowerModificationKind.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
namespace KustoSchemaTools.Model
2+
{
3+
public enum FollowerModificationKind
4+
{
5+
None,
6+
Union,
7+
Replace
8+
}
9+
10+
}

‎KustoSchemaTools/Model/FollowerPermissions.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
namespace KustoSchemaTools.Model
2+
{
3+
public class FollowerPermissions
4+
{
5+
public FollowerModificationKind ModificationKind { get; set; }
6+
public List<AADObject> Viewers { get; set; } = new List<AADObject>();
7+
public List<AADObject> Admins { get; set; } = new List<AADObject>();
8+
}
9+
10+
}

‎KustoSchemaTools/Parser/KustoLoader/FollowerLoader.cs‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
using KustoSchemaTools.Model;
2+
using Newtonsoft.Json;
3+
using Newtonsoft.Json.Linq;
4+
using System;
5+
using System.Collections.Generic;
6+
using System.Linq;
7+
using System.Text;
8+
using System.Threading.Tasks;
9+
10+
namespace KustoSchemaTools.Parser.KustoLoader
11+
{
12+
public class FollowerLoader
13+
{
14+
15+
const string FollowerMetadataQuery = @".show follower database {0}
16+
| extend TableMetadataOverridesJson=parse_json(TableMetadataOverrides)
17+
| mv-apply TableMetadataOverridesJson on(
18+
project Table=tostring(bag_keys(TableMetadataOverridesJson)[0]), TableMetadataOverridesJson, Fragments=split(TableMetadataOverridesJson, '""')
19+
| mv-apply Fragments on (
20+
project Timespan=totimespan(Fragments)
21+
| where isnotempty(Timespan)
22+
| limit 1
23+
)
24+
| summarize CachingPolicies=make_bag(bag_pack(Table,Timespan))
25+
)
26+
";
27+
28+
public static FollowerDatabase LoadFollower(string databaseName, KustoClient client)
29+
{
30+
var follower = new FollowerDatabase { DatabaseName = databaseName };
31+
var metdaData = client.Client.ExecuteQuery(string.Format(FollowerMetadataQuery,databaseName)).As<FollowerMetadata>().First();
32+
33+
switch (metdaData.AuthorizedPrincipalsModificationKind)
34+
{
35+
case "Union":
36+
follower.Permissions.ModificationKind = FollowerModificationKind.Union;
37+
break;
38+
case "Replace":
39+
follower.Permissions.ModificationKind = FollowerModificationKind.Replace;
40+
break;
41+
default:
42+
follower.Permissions.ModificationKind = FollowerModificationKind.None;
43+
break;
44+
}
45+
46+
switch (metdaData.CachingPoliciesModificationKind)
47+
{
48+
case "Union":
49+
follower.Cache.ModificationKind = FollowerModificationKind.Union;
50+
break;
51+
case "Replace":
52+
follower.Cache.ModificationKind = FollowerModificationKind.Replace;
53+
break;
54+
default:
55+
follower.Cache.ModificationKind = FollowerModificationKind.None;
56+
break;
57+
}
58+
59+
foreach (var kvp in metdaData.CachingPolicies)
60+
{
61+
var isMv = kvp.Key.StartsWith("_MV_");
62+
var key = isMv ? kvp.Key.Substring(4): kvp.Key;
63+
64+
var target = isMv? follower.Cache.MaterializedViews : follower.Cache.Tables;
65+
target.Add(key, kvp.Value.Days+"d");
66+
}
67+
68+
return follower;
69+
}
70+
}
71+
72+
public class FollowerMetadata
73+
{
74+
public string DatabaseName { get; set; }
75+
public string LeaderClusterMetadataPath { get; set; }
76+
public string CachingPolicyOverride { get; set; }
77+
public string AuthorizedPrincipalsOverride { get; set; }
78+
public string AuthorizedPrincipalsModificationKind { get; set; }
79+
public bool IsAutoPrefetchEnabled { get; set; }
80+
public string TableMetadataOverrides { get; set; }
81+
public string CachingPoliciesModificationKind { get; set; }
82+
public string ChildEntities { get; set; }
83+
public string OriginalDatabaseName { get; set; }
84+
public Dictionary<string,TimeSpan> CachingPolicies { get; set; }
85+
86+
}
87+
}

0 commit comments

Comments
 (0)