Skip to content

Commit 9ba9606

Browse files
author
Thomas Mahlberg
committed
Finished changes for caching policies
1 parent 42c3429 commit 9ba9606

File tree

5 files changed

+140
-25
lines changed

5 files changed

+140
-25
lines changed

‎KustoSchemaTools/Changes/DatabaseChanges.cs‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -180,8 +180,6 @@ private static List<IChange> GenerateScriptCompareChanges<T>(Database oldState,
180180

181181
public static List<IChange> GenerateFollowerChanges(FollowerDatabase oldState, FollowerDatabase newState, ILogger log)
182182
{
183-
184-
185183
List<IChange> result =
186184
[
187185
.. GenerateFollowerCachingChanges(oldState, newState, db => db.Tables, "Table", "table"),
@@ -192,15 +190,15 @@ .. GenerateFollowerCachingChanges(oldState, newState, db => db.MaterializedViews
192190
if (oldState.Permissions.ModificationKind != newState.Permissions.ModificationKind)
193191
{
194192
var kind = newState.Permissions.ModificationKind.ToString().ToLower();
195-
result.Add(new BasicChange("FollowerDatabase", "PermissionsModificationKind", $"From {oldState.Permissions.ModificationKind} to {newState.Permissions.ModificationKind}", new List<DatabaseScriptContainer>
193+
result.Add(new BasicChange("FollowerDatabase", "PermissionsModificationKind", $" Change Permission-Modification-Kind from {oldState.Permissions.ModificationKind} to {newState.Permissions.ModificationKind}", new List<DatabaseScriptContainer>
196194
{
197195
new DatabaseScriptContainer(new DatabaseScript($".alter follower database {newState.DatabaseName} principals-modification-kind = {kind}", 0), "FollowerChangePolicyModificationKind")
198196
}));
199197
}
200198
if (oldState.Cache.ModificationKind != newState.Cache.ModificationKind)
201199
{
202200
var kind = newState.Cache.ModificationKind.ToString().ToLower();
203-
result.Add(new BasicChange("FollowerDatabase", "ChangeModificationKind", $"From {oldState.Cache.ModificationKind} to {newState.Cache.ModificationKind}", new List<DatabaseScriptContainer>
201+
result.Add(new BasicChange("FollowerDatabase", "ChangeModificationKind", $"Change Caching-Modification-Kind from {oldState.Cache.ModificationKind} to {newState.Cache.ModificationKind}", new List<DatabaseScriptContainer>
204202
{
205203
new DatabaseScriptContainer(new DatabaseScript($".alter follower database {newState.DatabaseName} caching-policies-modification-kind = {kind}", 0), "FollowerChangePolicyModificationKind")
206204
}));
@@ -269,10 +267,10 @@ private static List<IChange> GenerateFollowerCachingChanges(FollowerDatabase old
269267
}
270268
if (changedPolicyScripts.Any())
271269
{
272-
result.Add(new Heading($"Changed {type} Caching Policies"));
273-
274270
var changePolicies = new StringBuilder();
271+
changePolicies.AppendLine($"## Changed {type} Caching Policies");
275272
changePolicies.AppendLine($"{type} | From | To");
273+
changePolicies.AppendLine("--|--|--");
276274
foreach (var change in changedPolicyScripts)
277275
{
278276
changePolicies.AppendLine($"{change.Name} | {change.From} | {change.To}");

‎KustoSchemaTools/KustoSchemaHandler.cs‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using KustoSchemaTools.Helpers;
33
using KustoSchemaTools.Model;
44
using KustoSchemaTools.Parser;
5+
using KustoSchemaTools.Parser.KustoLoader;
56
using Microsoft.Extensions.Logging;
67
using System.Collections.Concurrent;
78
using System.Data;
@@ -71,22 +72,24 @@ public KustoSchemaHandler(ILogger<KustoSchemaHandler<T>> schemaHandlerLogger, Ya
7172
foreach(var follower in yamlDb.Followers)
7273
{
7374

74-
Log.LogInformation($"Generating diff markdown for {Path.Combine(path, databaseName)} => {follower.Cluster}/{databaseName}");
75+
Log.LogInformation($"Generating diff markdown for {Path.Combine(path, databaseName)} => {follower.Key}/{databaseName}");
7576

7677

78+
var followerClient = new KustoClient(follower.Key);
79+
var oldModel = FollowerLoader.LoadFollower(follower.Value.DatabaseName, followerClient);
7780

81+
var newModel = follower.Value;
7882

83+
var changes = DatabaseChanges.GenerateFollowerChanges(oldModel, newModel, Log);
7984

80-
//var scriptSb = new StringBuilder();
81-
//foreach (var script in changes.SelectMany(itm => itm.Scripts).Where(itm => itm.IsValid == true).OrderBy(itm => itm.Order))
82-
//{
83-
// scriptSb.AppendLine(script.Text);
84-
//}
85-
86-
//Log.LogInformation($"Following scripts will be applied:\n{scriptSb}");
87-
85+
sb.AppendLine($"# Changes for follower database {follower.Key}/{databaseName}");
86+
sb.AppendLine();
87+
foreach (var change in changes)
88+
{
89+
sb.AppendLine(change.Markdown);
90+
sb.AppendLine();
91+
}
8892
}
89-
9093
return (sb.ToString(), isValid);
9194
}
9295

‎KustoSchemaTools/Model/FollowerDatabase.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@
22
{
33
public class FollowerDatabase
44
{
5-
public required string DatabaseName { get; set; }
6-
public FollowerCache Cache { get; set; } = new FollowerCache();
5+
public string DatabaseName { get; set; }
6+
public FollowerCache Cache { get; set; } = new FollowerCache();
7+
// TODO: No logic to load data / roll out changes implemented yet!
78
public FollowerPermissions Permissions { get; set; } = new FollowerPermissions();
89
}
910

‎KustoSchemaTools/Parser/KustoLoader/FollowerLoader.cs‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
using KustoSchemaTools.Model;
2+
using Newtonsoft.Json;
3+
using Newtonsoft.Json.Linq;
4+
using System;
5+
using System.Collections.Generic;
6+
using System.Linq;
7+
using System.Text;
8+
using System.Threading.Tasks;
9+
10+
namespace KustoSchemaTools.Parser.KustoLoader
11+
{
12+
public class FollowerLoader
13+
{
14+
15+
const string FollowerMetadataQuery = @".show follower database {0}
16+
| extend TableMetadataOverridesJson=parse_json(TableMetadataOverrides)
17+
| mv-apply TableMetadataOverridesJson on(
18+
project Table=tostring(bag_keys(TableMetadataOverridesJson)[0]), TableMetadataOverridesJson, Fragments=split(TableMetadataOverridesJson, '""')
19+
| mv-apply Fragments on (
20+
project Timespan=totimespan(Fragments)
21+
| where isnotempty(Timespan)
22+
| limit 1
23+
)
24+
| summarize CachingPolicies=make_bag(bag_pack(Table,Timespan))
25+
)
26+
";
27+
28+
public static FollowerDatabase LoadFollower(string databaseName, KustoClient client)
29+
{
30+
var follower = new FollowerDatabase { DatabaseName = databaseName };
31+
var metdaData = client.Client.ExecuteQuery(string.Format(FollowerMetadataQuery,databaseName)).As<FollowerMetadata>().First();
32+
33+
switch (metdaData.AuthorizedPrincipalsModificationKind)
34+
{
35+
case "Union":
36+
follower.Permissions.ModificationKind = FollowerModificationKind.Union;
37+
break;
38+
case "Replace":
39+
follower.Permissions.ModificationKind = FollowerModificationKind.Replace;
40+
break;
41+
default:
42+
follower.Permissions.ModificationKind = FollowerModificationKind.None;
43+
break;
44+
}
45+
46+
switch (metdaData.CachingPoliciesModificationKind)
47+
{
48+
case "Union":
49+
follower.Cache.ModificationKind = FollowerModificationKind.Union;
50+
break;
51+
case "Replace":
52+
follower.Cache.ModificationKind = FollowerModificationKind.Replace;
53+
break;
54+
default:
55+
follower.Cache.ModificationKind = FollowerModificationKind.None;
56+
break;
57+
}
58+
59+
foreach (var kvp in metdaData.CachingPolicies)
60+
{
61+
var isMv = kvp.Key.StartsWith("_MV_");
62+
var key = isMv ? kvp.Key.Substring(4): kvp.Key;
63+
64+
var target = isMv? follower.Cache.MaterializedViews : follower.Cache.Tables;
65+
target.Add(key, kvp.Value.Days+"d");
66+
}
67+
68+
return follower;
69+
}
70+
}
71+
72+
public class FollowerMetadata
73+
{
74+
public string DatabaseName { get; set; }
75+
public string LeaderClusterMetadataPath { get; set; }
76+
public string CachingPolicyOverride { get; set; }
77+
public string AuthorizedPrincipalsOverride { get; set; }
78+
public string AuthorizedPrincipalsModificationKind { get; set; }
79+
public bool IsAutoPrefetchEnabled { get; set; }
80+
public string TableMetadataOverrides { get; set; }
81+
public string CachingPoliciesModificationKind { get; set; }
82+
public string ChildEntities { get; set; }
83+
public string OriginalDatabaseName { get; set; }
84+
public Dictionary<string,TimeSpan> CachingPolicies { get; set; }
85+
86+
}
87+
}

‎KustoSchemaTools/Parser/KustoWriter/DefaultDatabaseWriter.cs‎

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using Kusto.Data;
22
using KustoSchemaTools.Changes;
33
using KustoSchemaTools.Model;
4+
using KustoSchemaTools.Parser.KustoLoader;
45
using KustoSchemaTools.Plugins;
56
using Microsoft.Extensions.Logging;
67
using System.Text;
@@ -19,14 +20,39 @@ public async Task WriteAsync(Database sourceDb, Database targetDb, KustoClient c
1920
Console.WriteLine($"{result.CommandType} ({result.OperationId}): {result.Result} => {result.Reason} ({result.CommandText})");
2021
Console.WriteLine("---------------------------------------------------------------------------");
2122
}
22-
var exs = results.Where(itm => itm.Result == "Failed").Select(itm => new Exception($"Execution failed for command \n{itm.CommandText} \n with reason\n{itm.Reason}")).ToList();
23-
if (exs.Count == 1)
24-
{
25-
throw exs[0];
26-
}
27-
if (exs.Count > 1)
23+
24+
foreach (var follower in targetDb.Followers)
2825
{
29-
throw new AggregateException(exs);
26+
var followerClient = new KustoClient(follower.Key);
27+
var source = FollowerLoader.LoadFollower(follower.Value.DatabaseName, followerClient);
28+
29+
var followerChanges = DatabaseChanges.GenerateFollowerChanges(source, follower.Value, logger);
30+
31+
var followerResults = await ApplyChangesToDatabase(follower.Value.DatabaseName, followerChanges, followerClient, logger);
32+
33+
Console.WriteLine();
34+
Console.WriteLine($"Follower: {follower.Key}");
35+
Console.WriteLine("---------------------------------------------------------------------------");
36+
Console.WriteLine();
37+
38+
foreach (var result in followerResults)
39+
{
40+
Console.WriteLine($"{result.CommandType} ({result.OperationId}): {result.Result} => {result.Reason} ({result.CommandText})");
41+
Console.WriteLine("---------------------------------------------------------------------------");
42+
}
43+
44+
Console.WriteLine();
45+
Console.WriteLine();
46+
47+
var exs = results.Where(itm => itm.Result == "Failed").Select(itm => new Exception($"Execution failed for command \n{itm.CommandText} \n with reason\n{itm.Reason}")).ToList();
48+
if (exs.Count == 1)
49+
{
50+
throw exs[0];
51+
}
52+
if (exs.Count > 1)
53+
{
54+
throw new AggregateException(exs);
55+
}
3056
}
3157
}
3258

0 commit comments

Comments
 (0)