Files
linqpad/DB Query Exceptions.linq
T

605 lines
21 KiB
C#

<Query Kind="Program">
<Connection>
<ID>580b3b8d-cc02-4a68-8fbd-d1ec374a25f2</ID>
<Persist>true</Persist>
<Server>D0001-sql-11.sdt.local</Server>
<Database>jazz dev master Prod</Database>
<ShowServer>true</ShowServer>
</Connection>
<Namespace>Newtonsoft.Json</Namespace>
<Namespace>Newtonsoft.Json.Converters</Namespace>
<Namespace>Newtonsoft.Json.Linq</Namespace>
<Namespace>Newtonsoft.Json.Schema</Namespace>
<Namespace>Newtonsoft.Json.Serialization</Namespace>
<Namespace>System</Namespace>
<Namespace>System.ComponentModel</Namespace>
<Namespace>System.Diagnostics</Namespace>
<Namespace>System.Diagnostics.Tracing</Namespace>
<Namespace>System.IO.Compression</Namespace>
<Namespace>System.Linq.Dynamic</Namespace>
</Query>
// This link provides a good explanation of the Dynamic Query used in the DataTableJoins
// https://ecs.syr.edu/faculty/fawcett/handouts/CoreTechnologies/CSharp/samples/CSharpSamples/LinqSamples/DynamicQuery/Dynamic%20Expressions.html
//
#region Options
public static int TopRows = 1000; //int.MaxValue;
public static int TableDumpDepth = int.MaxValue; //2;
public static bool ShowTableQuery = false;
public static bool ShowTablesToRemove = false;
public static bool ShowTableNames = false;
public static bool ShowTableList = false;
public static bool ShowMissingUpdate = true;
public static bool ShowMissingLegacy = false;
#endregion
private Connections connections = new Connections
{
//Legacy = new ServerInstance("E0001-SQL-11.sdt.local", "StrataSphere"),
//Update = new ServerInstance("D0011-SQL-01.sdt.local", "StrataSphere")
Legacy = new ServerInstance("D0011-SQL-01.sdt.local", "jazz tst Southern Illinois 20190918.1"),
Update = new ServerInstance("D0011-SQL-01.sdt.local", "jazz tst Southern Illinois 20200331.1")
//Legacy = new ServerInstance("D0001-SQL-11.sdt.local", "jazz tst master prod 20190708.1"),
//Update = new ServerInstance("D0001-SQL-11.sdt.local", "jazz tst master prod")
};
// Hints:
// 1) if you pass a null for the table list it will include all tables in the schema
// 2) if you pass a table name like "C%"' it will only include all tables that start with C
private List<TablesAndColumns> SchemasAndTablesToReview = new List<TablesAndColumns> {
new TablesAndColumns("cci", new[] {"%QVILineItemOpportunity%"}),
//new TablesAndColumns("clientdss", new [] { "FactPatientEncounterSummary" }),
//new TablesAndColumns("dss", new [] {"%CaseType%"}, new[] {"CaseTypeDRGMapping","CaseTypeFamilyMapping","FactCaseTypeMapping"}),
//new TablesAndColumns("fw", new []{ "DimAgeCohort"})
};
void Main()
{
var tables = GetTables().ToList();
// Using 'D|' at the beginning of a column name implies that the sort order is descending for that column
tables.ForEach(table =>
{
switch (table.SchemaName)
{
case "cci":
switch (table.TableName)
{
case "ChargeCodeConfig":
table.Init(
new[] { "ChargeCodeID", "CostDriver" },
keyColumns: new[] { "ChargeCodeID", "CostDriver" }
);
break;
case "ChargeCodeCostDetail":
table.Init(
new[] { "EntityID", "ChargeCodeID" },
keyColumns: new[] { "EntityID", "ChargeCodeID" }
);
break;
case "Configuration":
table.Init(new[] { "D|FiscalYearID" }, new[] { "InitiativeIsSimplifiedWorkflow" }, new[] { "FiscalYearId" });
break;
case "InitiativeWorkflowHistory":
table.Init(new[] { "FromStep", "ToStep" });
break;
case "VariationEncounterSavingInfo":
table.Init(new[] { "EncounterRecordNumber" });
break;
case "FactOpportunityMetric":
table.Init(new[] { "FiscalYearID", "FiscalMonthID" });
break;
case "FactOpportunitySavings":
table.Init(new[] { "FiscalyYearID", "FiscalMonthID" });
break;
case "Initiative":
table.Init(new[] { "Name" });
break;
case "SystemSetting":
table.Excludes(new[] { "IsEncrypted" });
break;
case "QVILineItemOpportunity":
table.KeyColumns(new[] {"BreakdownID", "QVILineItemID", "QualityVariationEventID"});
break;
case "VariationOpportunity":
table.Init(
new[] { "CaseTypeEntityId", "CostDriver" },
keyColumns: new[] { "CostDriver", "CastTypeEntityId" }
);
break;
};
break;
case "clientdss":
switch (table.TableName)
{
case "FactPatientEncounterSummary":
table.Init(
new[] { "EncounterRecordNumber" },
new[] { "MikeQAtest1ID", "CustomJesseValidatingAllOthers", "CustomNewCalculatedSystemFieldasdf" },
where: "OBJ.MSDRGCaseTypeFamilyID <> 0 OR OBJ.APRDRGCaseTypeFamilyID <> 0");
break;
};
break;
case "dss":
switch (table.TableName)
{
case "CaseTypeDRGMapping":
table.Init(new[] { "DRGID" }, keyColumns: new[] { "DRGID" });
break;
case "CaseTypeEntity":
table.Init(new[] { "EntityID", "CaseTypeEntityID" },
keyColumns: new[] { "EntityID", "CaseTypeEntityID" });
break;
case "CaseTypeFamily":
table.Init(new[] { "Name", "CaseTypeFamilyGlobalID", "CaseTypeFamilyID" },
keyColumns: new[] { "Name", "CaseTypeFamilyGlobalID" });
break;
case "CaseTypeFamilyMappingAPRDRG":
table.Init(new[] { "APRDRGCode", "CaseTypeFamilyVersionID", "CaseTypeFamilyID" },
keyColumns: new[] { "APRDRGCode", "CaseTypeFamilyVersionID", "CaseTypeFamilyID" });
break;
case "CaseTypeFamilyMappingCPT":
table.Init(new[] { "CPTCode", "CaseTypeFamilyVersionID", "CaseTypeFamilyID" },
keyColumns: new[] { "CPTCode", "CaseTypeFamilyVersionID", "CaseTypeFamilyID" });
break;
case "CaseTypeFamilyMappingICD10":
table.Init(new[] { "ICD10Code", "MSDRGCode", "CaseTypeFamilyVersionID", "CaseTypeFamilyID" },
keyColumns: new[] { "ICD10Code", "MSDRGCode", "CaseTypeFamilyVersionID", "CaseTypeFamilyID" });
break;
case "CaseTypeFamilyVersion":
table.Init(new[] { "CaseTypeFamilyVersionID" },
keyColumns: new[] { "CaseTypeFamilyVersionID" });
break;
case "DimCaseType":
table.Init(new[] { "CaseTypeFamilyID", "CaseTypeId", "Name" }, jsonColumns: new[] { "CaseTypeConfigJSON" });
break;
};
break;
};
});
if (ShowTableNames) tables.Select(t => t.SchemaTable).Dump(nameof(ShowTableNames));
if (ShowTableList) tables.Dump(nameof(ShowTableList));
DoTheComparisons(connections, tables);
}
// Define other methods and classes here
#region Gather Table Information
public string GetTableQuery()
{
var tableSchemas = string.Join(", ", SchemasAndTablesToReview.Select(sattr => $"'{sattr.SchemaName}'"));
var tableList = string.Join("\r\n", SchemasAndTablesToReview.ConvertAll(sattr =>
{
if (sattr.TableNames?.Any(cn => cn == "*") ?? true)
{
return $"\r\nWHEN c.TABLE_SCHEMA = '{sattr.SchemaName}' THEN 1";
}
else
{
var result = "";
if (sattr.TableNames.Any(tn => tn.Contains("%")))
{
result += sattr.TableNames.Where(tn => tn.Contains("%"))
.Aggregate("", (x, y) => $"{x}\r\nWHEN c.TABLE_SCHEMA = '{sattr.SchemaName}' AND c.TABLE_NAME LIKE '{y}' THEN 1");
}
if (sattr.TableNames.Any(tn => !tn.Contains("%")))
{
result += $"\r\nWHEN c.TABLE_SCHEMA = '{sattr.SchemaName}' AND c.TABLE_NAME IN ({string.Join(", ", sattr.TableNames.Where(tn => !tn.Contains("%")).Select(tn => $"'{tn}'"))}) THEN 1";
}
return result;
}
}));
return $@"SELECT c.TABLE_SCHEMA, c.TABLE_NAME, c.COLUMN_NAME, c.DATA_TYPE
, CAST(CASE WHEN ccu.CONSTRAINT_NAME IS NULL THEN 0 ELSE 1 END AS BIT) IsPrimaryKey
FROM INFORMATION_SCHEMA.COLUMNS AS c
LEFT OUTER JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS tc ON tc.TABLE_SCHEMA = c.TABLE_SCHEMA AND tc.TABLE_NAME = c.TABLE_NAME
LEFT OUTER JOIN INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE AS ccu ON ccu.TABLE_SCHEMA = tc.TABLE_SCHEMA AND ccu.TABLE_NAME = tc.TABLE_NAME AND ccu.CONSTRAINT_NAME = tc.CONSTRAINT_NAME AND ccu.COLUMN_NAME = c.COLUMN_NAME
WHERE c.TABLE_SCHEMA in ({tableSchemas})
AND (CASE {tableList}
ELSE 0
END = 1)
AND c.DATA_TYPE NOT IN ('datetime','uniqueidentifier', 'timestamp')
AND NOT (c.COLUMN_NAME = 'RowID' and CAST(CASE WHEN ccu.CONSTRAINT_NAME IS NULL THEN 0 ELSE 1 END AS BIT) = 1)
AND tc.CONSTRAINT_TYPE = 'Primary Key'
ORDER BY c.TABLE_NAME, c.ORDINAL_POSITION";
}
public IList<Table> GetTables()
{
var tableQuery = GetTableQuery();
if (ShowTableQuery) tableQuery.Dump(nameof(ShowTableQuery));
var tablesToRemove = SchemasAndTablesToReview.Where(sattr => sattr.ExcludeTables.Any()).SelectMany(sattr => sattr.ExcludeTables.Select(et => new { sattr.SchemaName, ExcludeTable = et }));
var tables = ExecuteQueryDynamic(tableQuery).ToList();
if (tablesToRemove.Any())
{
if (ShowTablesToRemove)
tablesToRemove.Dump(nameof(ShowTablesToRemove));
tables.RemoveAll(x => tablesToRemove.Any(sattr => sattr.SchemaName == x.TABLE_SCHEMA && sattr.ExcludeTable == x.TABLE_NAME));
}
return tables
.GroupBy(x => new { x.TABLE_SCHEMA, x.TABLE_NAME },
(g, d) => new Table(g.TABLE_SCHEMA, g.TABLE_NAME,
d.Select(y => new Column(y.COLUMN_NAME, y.DATA_TYPE, y.IsPrimaryKey)).ToList()))
.ToList();
}
public class Table
{
public string SchemaName { get; set; }
public string TableName { get; set; }
public IList<Column> Columns { get; set; } = new List<Column>();
public IList<OrderBy> OrderBy { get; set; } = new List<OrderBy>();
public IList<string> Exclude { get; set; } = new List<string>();
public IList<string> KeyColumn { get; set; } = new List<string>();
public IList<string> Json { get; set; } = new List<string>();
public string Where { get; set; } = "";
internal string SchemaTable => $"{SchemaName}.{TableName}";
public string SqlSelect => $"select TOP({TopRows}) \r\n\t{SelectColumns}\r\nfrom {SchemaTable} OBJ{Where}{OrderByStatement}";
internal string SelectColumns => !Columns?.Any() ?? true ? "*" : string.Join(",\r\n\t", Columns.Where(c => !Exclude.ToList().Contains(c.ColumnName)).Select(c => c.ColumnName));
internal string OrderByStatement => !OrderBy?.Any() ?? true ? "" : "\r\nOrder By " + string.Join(", ", OrderBy.Select(ob => $"{ob.ColumnName}{(ob.IsDescending ? " DESC" : "")}"));
public Table(string schemaName, string tableName)
{
SchemaName = schemaName;
TableName = tableName;
Columns = new List<Column>();
OrderBy = new List<OrderBy>();
}
public Table(string schemaName, string tableName, IEnumerable<Column> columns)
: this(schemaName, tableName)
{
Columns = columns.ToList();
}
public Table(string schemaName, string tableName, IEnumerable<Column> columns, IEnumerable<OrderBy> orderBy)
: this(schemaName, tableName, columns)
{
OrderBy = orderBy.ToList();
}
/// <sumarry>
///
/// </summary>
/// <param name="oderBys">
/// Using 'D|' at the beginning of a column name implies that the sort order is descending for that column
/// <param>
public void Init(IEnumerable<string> orderBys = null, IEnumerable<string> excludes = null, IEnumerable<string> keyColumns = null, IEnumerable<string> jsonColumns = null, string where = "")
{
Init(orderBys?.Distinct().Select(o => new OrderBy(o.Split('|').Last(), o.StartsWith("D|", StringComparison.CurrentCultureIgnoreCase))) ?? null,
excludes?.Distinct(), keyColumns?.Distinct(), jsonColumns?.Distinct(), where);
}
/// <sumarry>
///
/// </summary>
/// <param name="oderBys">
/// Using 'D|' at the beginning of a column name implies that the sort order is descending for that column
/// <param>
public void Init(IEnumerable<OrderBy> orderBys = null, IEnumerable<string> excludes = null, IEnumerable<string> keyColumns = null, IEnumerable<string> jsonColumns = null, string where = "")
{
if (orderBys != null)
OrderBys(orderBys.Where(x => Columns.Select(c => c.ColumnName).Contains(x.ColumnName)));
if (excludes != null)
Excludes(excludes.Where(x => Columns.Select(c => c.ColumnName).Contains(x)));
if (keyColumns != null)
KeyColumns(keyColumns.Where(x => Columns.Select(c => c.ColumnName).Contains(x)));
if (jsonColumns != null)
JsonColumns(jsonColumns.Where(x => Columns.Select(c => c.ColumnName).Contains(x)));
if (!string.IsNullOrEmpty(where))
where = $"\r\nwhere {where}";
}
/// <sumarry>
/// Using 'D|' at the beginning of a column name implies that the sort order is descending for that column
/// </summary>
public void OrderBys(IEnumerable<OrderBy> orderBys)
{
OrderBy.Clear();
((List<OrderBy>)OrderBy).AddRange(orderBys.Where(o => Columns.Select(c => c.ColumnName).Contains(o.ColumnName, StringComparer.CurrentCultureIgnoreCase)));
}
public void Excludes(IEnumerable<string> excludes)
{
Exclude.Clear();
((List<string>)Exclude).AddRange(excludes.Where(e => Columns.Select(c => c.ColumnName).Contains(e, StringComparer.CurrentCultureIgnoreCase)));
}
public void KeyColumns(IEnumerable<string> keyColumns)
{
KeyColumn.Clear();
((List<string>)KeyColumn).AddRange(keyColumns.Where(kc => Columns.Select(c => c.ColumnName).Contains(kc, StringComparer.CurrentCultureIgnoreCase)));
KeyColumn.Reverse().ToList().ForEach(kc => {
var col = Columns.Single(c => c.ColumnName == kc);
Columns.Remove(col);
Columns.Insert(0, col);
});
}
public void JsonColumns(IEnumerable<string> jsonColumns)
{
Json.Clear();
((List<string>)Json).AddRange(jsonColumns.Where(jc => Columns.Select(c => c.ColumnName).Contains(jc, StringComparer.CurrentCultureIgnoreCase)));
}
}
public class Column
{
public string ColumnName { get; set; }
public string DataType { get; set; }
public bool IsPrimaryKey { get; set; }
public Column(string columnName, string dataType = "", bool isPrimaryKey = false)
{
ColumnName = columnName;
DataType = dataType;
IsPrimaryKey = isPrimaryKey;
}
}
public class OrderBy
{
public string ColumnName { get; set; }
public bool IsDescending { get; set; } = false;
public OrderBy(string columnName, bool isDescending = false)
{
ColumnName = columnName;
IsDescending = isDescending;
}
}
public class TablesAndColumns
{
public string SchemaName { get; set; }
public IEnumerable<string> TableNames { get; set; }
public IEnumerable<string> ExcludeTables { get; set; }
public TablesAndColumns(string schemaName, IEnumerable<string> tables = null, IEnumerable<string> excludeTables = null)
{
SchemaName = schemaName;
TableNames = tables ?? new string[] { "*" };
ExcludeTables = excludeTables ?? new string[] { };
}
}
public class OrderByInformation
{
public string TableName { get; set; }
public IEnumerable<OrderBy> OrderBy { get; set; }
public OrderByInformation(string tableName, IEnumerable<string> columns)
{
TableName = tableName;
OrderBy = columns
.Select(c =>
new OrderBy(c.StartsWith("D|") ? c.Substring(2) : c,
c.StartsWith("D|")));
}
}
public class ColumnsToExclude
{
public string TableName { get; set; }
public IEnumerable<string> ColumnNames { get; set; }
public ColumnsToExclude(string tableName, IEnumerable<string> columns)
{
TableName = tableName;
ColumnNames = columns;
}
}
public class JsonColumns
{
public string TableName { get; set; }
public IEnumerable<string> ColumnNames { get; set; }
public JsonColumns(string tableName, IEnumerable<string> columns)
{
TableName = tableName;
ColumnNames = columns;
}
}
#endregion
#region Do the comparisons
public void DoTheComparisons(Connections connections, IEnumerable<Table> tables)
{
foreach (var table in tables)
{
var sql = table.SqlSelect;
var dataLegacy = GetData(connections.Legacy, sql, table.TableName).AsEnumerable().ToList();
var dataUpdate = GetData(connections.Update, sql, table.TableName).AsEnumerable().ToList();
var compare = new List<CompareSet>();
var jsonColumns = table.Json.ToList() ?? null;
var keyColumns = table.KeyColumn.ToList() ?? null;
var idx = 0;
dataLegacy
.ForEach(legacyRow =>
{
DataRow updateRow = default(DataRow);
if (table.KeyColumn.Any())
{
var keys = string.Join("|", table.KeyColumn.Select(kc => $"{legacyRow.Field<object>(kc)}"));
updateRow = dataUpdate
.FirstOrDefault(y => keys == string.Join("|", table.KeyColumn.Select(kc => $"{y.Field<object>(kc)}")));
}
else
{
if (dataUpdate.Count() > idx)
updateRow = dataUpdate[idx++];
}
if ((ShowMissingUpdate && updateRow == null) ||
(updateRow != null && JsonConvert.SerializeObject(legacyRow.ItemArray) != JsonConvert.SerializeObject(updateRow.ItemArray)))
compare.Add(new CompareSet(compare.Count, legacyRow, updateRow, jsonColumns, keyColumns));
});
if (ShowMissingLegacy && table.KeyColumn.Any())
{
dataUpdate
.ForEach(updateRow =>
{
DataRow legacyRow = default(DataRow);
if (table.KeyColumn.Any())
{
var keys = string.Join("|", table.KeyColumn.Select(kc => $"{updateRow.Field<object>(kc)}"));
legacyRow = dataLegacy
.FirstOrDefault(y => keys == string.Join("|", table.KeyColumn.Select(kc => $"{y.Field<object>(kc)}")));
}
if (legacyRow == null)
compare.Add(new CompareSet(compare.Count, legacyRow, updateRow, jsonColumns, keyColumns));
});
}
if (compare.Any())
{
compare.OrderBy(c => c.OrderBy)
.Dump($"{table.SchemaTable} ( {compare.Count()} / {dataLegacy.Count} / {dataUpdate.Count} rows )", TableDumpDepth);
}
else
{
"No differences found in data".Dump($"{table.SchemaTable} ( 0 / {dataLegacy.Count} / {dataUpdate.Count} rows )");
}
}
}
public class Connections
{
public ServerInstance Legacy { get; set; }
public ServerInstance Update { get; set; }
}
public class ServerInstance
{
public string Server { get; set; }
public string Database { get; set; }
public ServerInstance(string server, string database)
{
Server = server;
Database = database;
}
}
public DataTable GetData(ServerInstance serverInstance, string sql, string tableName)
{
var server = serverInstance.Server;
var database = serverInstance.Database;
var connStr = $"data source={server};initial catalog='{database}';persist security info=True;Integrated Security=SSPI;";
using (var conn = new SqlConnection(connStr))
{
var datatable = new DataTable();
var adapter = new SqlDataAdapter(sql, conn);
try
{
adapter.Fill(datatable);
}
catch (Exception)
{
sql.Dump(nameof(sql));
throw;
}
datatable.TableName = tableName;
return datatable;
}
}
public class CompareSet
{
internal int Row { get; set; }
internal IList<string> ColumnNames { get; set; }
internal IList<object> Legacy { get; set; }
internal IList<object> Update { get; set; }
internal int OrderBy { get; set; } = 0;
public IList<Row> Rows { get; set; }
internal IList<string> JsonColumns { get; set; }
internal IList<string> KeyColumns { get; set; }
public CompareSet(int row, DataRow right, DataRow left, IList<string> jsonColumns, IList<string> keyColumns)
{
Row = ++row;
if (right == null)
{
ColumnNames = left.Table.Columns.Cast<DataColumn>().Select(dc => dc.ColumnName).ToList();
}
else
{
ColumnNames = right.Table.Columns.Cast<DataColumn>().Select(dc => dc.ColumnName).ToList();
}
Legacy = right?.ItemArray.ToList();
Update = left?.ItemArray.ToList();
OrderBy = (right != null ? 0 : 2) + (left != null ? 0 : 1);
JsonColumns = jsonColumns ?? new List<string>();
KeyColumns = keyColumns ?? new List<string>();
Rows = ColumnNames.Select((c, i) => new Row(row, c, Legacy?[i], Update?[i], JsonColumns?.Contains(c, StringComparer.CurrentCultureIgnoreCase) ?? false)).ToList();
}
public CompareSet(int row, DataRow right, DataRow left)
: this(row, right, left, null, null)
{
}
object ToDump() => new
{
Row = $"{Row}",
Rows = Rows.Select(x => new
{
Column = (x.Left ?? "").Equals(x.Right ?? "")
? KeyColumns.Contains((string)x.Column)
? new XElement("LINQPad.HTML", new XElement("div", new XAttribute("style", "background-color:cyan;color:black"), x.Column))
: (object)x.Column
: new XElement("LINQPad.HTML", new XElement("div", new XAttribute("style", "background-color:lightyellow;color:black"), x.Column)),
Legacy = x.Left,
Update = x.Right
})
};
}
public class Row
{
public int RowId { get; set; }
public string Column;
public bool IsJson;
public object Left;
public object Right;
public Row(int row, string column, object left, object right)
{
RowId = row;
Column = column;
Left = left;
Right = right;
}
public Row(int row, string column, object left, object right, bool isJson)
: this(row, column, left, right)
{
IsJson = isJson;
if (!isJson) return;
if (left != null)
{
Left = JsonConvert.SerializeObject(JsonConvert.DeserializeObject(left.ToString(),
new JsonSerializerSettings
{
Error = delegate (object sender, Newtonsoft.Json.Serialization.ErrorEventArgs args)
{
args.ErrorContext.Handled = true;
}
}), Newtonsoft.Json.Formatting.Indented);
}
if (right != null)
{
Right = JsonConvert.SerializeObject(JsonConvert.DeserializeObject(right.ToString(),
new JsonSerializerSettings
{
Error = delegate (object sender, Newtonsoft.Json.Serialization.ErrorEventArgs args)
{
args.ErrorContext.Handled = true;
}
}), Newtonsoft.Json.Formatting.Indented);
}
}
}
#endregion