Files
linqpad/DB Query with Lambda Expressions.linq
T

460 lines
18 KiB
C#

<Query Kind="Program">
<Connection>
<ID>77b65bea-e766-4e3c-a1ed-c4c2534b6019</ID>
<Persist>true</Persist>
<Server>D0001-SQL-11.sdt.local</Server>
<Database>jazz dev master Prod</Database>
<ShowServer>true</ShowServer>
</Connection>
<Reference>&lt;ProgramFilesX64&gt;\Microsoft SDKs\Azure\.NET SDK\v2.9\bin\plugins\Diagnostics\Newtonsoft.Json.dll</Reference>
<Reference>&lt;RuntimeDirectory&gt;\System.Linq.Expressions.dll</Reference>
<Reference>&lt;RuntimeDirectory&gt;\System.Linq.Queryable.dll</Reference>
<NuGetReference>Amazon.Lambda.S3Events</NuGetReference>
<NuGetReference>AWSSDK.Core</NuGetReference>
<NuGetReference>AWSSDK.S3</NuGetReference>
<Namespace>Amazon</Namespace>
<Namespace>Amazon.Auth.AccessControlPolicy</Namespace>
<Namespace>Amazon.Auth.AccessControlPolicy.ActionIdentifiers</Namespace>
<Namespace>Amazon.Internal</Namespace>
<Namespace>Amazon.Lambda.S3Events</Namespace>
<Namespace>Amazon.MissingTypes</Namespace>
<Namespace>Amazon.Runtime</Namespace>
<Namespace>Amazon.Runtime.CredentialManagement</Namespace>
<Namespace>Amazon.Runtime.CredentialManagement.Internal</Namespace>
<Namespace>Amazon.Runtime.EventStreams</Namespace>
<Namespace>Amazon.Runtime.EventStreams.Internal</Namespace>
<Namespace>Amazon.Runtime.Internal</Namespace>
<Namespace>Amazon.Runtime.Internal.Auth</Namespace>
<Namespace>Amazon.Runtime.Internal.Settings</Namespace>
<Namespace>Amazon.Runtime.Internal.Transform</Namespace>
<Namespace>Amazon.Runtime.Internal.Util</Namespace>
<Namespace>Amazon.Runtime.SharedInterfaces</Namespace>
<Namespace>Amazon.Runtime.SharedInterfaces.Internal</Namespace>
<Namespace>Amazon.S3</Namespace>
<Namespace>Amazon.S3.Encryption</Namespace>
<Namespace>Amazon.S3.Encryption.Internal</Namespace>
<Namespace>Amazon.S3.Internal</Namespace>
<Namespace>Amazon.S3.IO</Namespace>
<Namespace>Amazon.S3.Model</Namespace>
<Namespace>Amazon.S3.Model.Internal.MarshallTransformations</Namespace>
<Namespace>Amazon.S3.Transfer</Namespace>
<Namespace>Amazon.S3.Util</Namespace>
<Namespace>Amazon.Util</Namespace>
<Namespace>Amazon.Util.Internal</Namespace>
<Namespace>Amazon.Util.Internal.PlatformServices</Namespace>
<Namespace>Newtonsoft.Json</Namespace>
<Namespace>Newtonsoft.Json.Converters</Namespace>
<Namespace>Newtonsoft.Json.Linq</Namespace>
<Namespace>Newtonsoft.Json.Schema</Namespace>
<Namespace>Newtonsoft.Json.Serialization</Namespace>
<Namespace>System</Namespace>
<Namespace>System.ComponentModel</Namespace>
<Namespace>System.Diagnostics</Namespace>
<Namespace>System.Diagnostics.Tracing</Namespace>
<Namespace>System.IO.Compression</Namespace>
<Namespace>System.Linq.Dynamic</Namespace>
<Namespace>ThirdParty.BouncyCastle.Asn1</Namespace>
<Namespace>ThirdParty.BouncyCastle.Asn1.Utilities</Namespace>
<Namespace>ThirdParty.BouncyCastle.Math</Namespace>
<Namespace>ThirdParty.BouncyCastle.OpenSsl</Namespace>
<Namespace>ThirdParty.BouncyCastle.Utilities.IO.Pem</Namespace>
<Namespace>ThirdParty.Ionic.Zlib</Namespace>
<Namespace>ThirdParty.Json.LitJson</Namespace>
<Namespace>ThirdParty.MD5</Namespace>
</Query>
private List<TablesAndColumns> SchemasAndTablesToReview = new List<TablesAndColumns>();
private List<OrderByInformation> OrderByStatements = new List<OrderByInformation>();
private List<ColumnsToExclude> ColumnsToBeExcluded = new List<ColumnsToExclude>();
private List<JsonColumns> JsonColumnsToBeParsed = new List<JsonColumns>();
private Dictionary<string, string> JoinClauses = new Dictionary<string, string>();
private static Dictionary<string, string> WhereClauses = new Dictionary<string, string>();
public static int TopRows = 1000;//int.MaxValue;
void Main()
{
SchemasAndTablesToReview.Clear();
SchemasAndTablesToReview.Add(new TablesAndColumns("cci", new[] {"VariationOpportunity"}));
WhereClauses.Clear();
WhereClauses.Add("clientdss.FactPatientEncounterSummary", "OBJ.MSDRGCaseTypeFamilyID <> 0 OR OBJ.APRDRGCaseTypeFamilyID <> 0");
OrderByStatements.AddRange(new[]
{
new OrderByInformation("cci.InitiativeWorkflowHistory", new [] {"FromStep","ToStep"}),
new OrderByInformation("cci.VariationEncounterSavingInfo", new []{"EncounterRecordNumber","PhysicianID"}),
new OrderByInformation("cci.FactOpportunityMetric", new []{"FiscalYearID","FiscalMonthID"}),
new OrderByInformation("cci.FactOpportunitySavings", new []{"FiscalyYearID","FiscalMonthID"}),
new OrderByInformation("cci.Initiative", new []{"Name"}),
new OrderByInformation("cci.VariationOpportunity", new []{"CaseTypeEntityId", "CostDriver"}),
new OrderByInformation("clientdss.FactPatientEncounterSummary", new []{"EncounterRecordNumber"}),
new OrderByInformation("dss.CaseTypeFamily", new []{"Name", "CaseTypeFamilyGlobalID", "CaseTypeFamilyID"}),
new OrderByInformation("dss.CaseTypeFamilyMappingCPT", new []{"CPTCode", "CaseTypeFamilyVersionID", "CaseTypeFamilyID"}),
new OrderByInformation("dss.DimCaseType", new []{"CaseTypeFamilyID", "CaseTypeId", "Name"})
});
JoinClauses.Clear();
JoinClauses.Add("cci.VariationOpportunity", "new(CaseTypeEntityId as firstKey, CostDriver as secondKey)");
ColumnsToBeExcluded.AddRange(new[]
{
new ColumnsToExclude("cci.SystemSetting", new [] {"IsEncrypted"}),
new ColumnsToExclude("cci.Configuration", new [] {"InitiativeIsSimplifiedWorkflow"}),
new ColumnsToExclude("clientdss.FactPatientEncounterSummary", new [] {"MikeQAtest1ID", "CustomJesseValidatingAllOthers", "CustomNewCalculatedSystemFieldasdf"})
});
JsonColumnsToBeParsed.AddRange(new[]
{
new JsonColumns("dss.DimCaseType", new [] {"CaseTypeConfigJSON"})
});
var tables = GetTables();
var connections = new Connections {
Legacy = new ServerInstance("D0001-SQL-11.sdt.local", "jazz tst master prod 20190708.1"),
Update = new ServerInstance("D0001-SQL-11.sdt.local","jazz tst master prod")
};
DoTheComparisons(connections, tables);
}
// Define other methods and classes here
#region Gather Table Information
public string GetTableQuery()
{
var tableSchemas = string.Join(", ", SchemasAndTablesToReview.Select(sattr => $"'{sattr.SchemaName}'"));
var tableList = string.Join("\r\n", SchemasAndTablesToReview.ConvertAll(sattr =>
{
if (sattr.TableNames?.Any(cn => cn == "*") ?? true)
{
return $"WHEN c.TABLE_SCHEMA = '{sattr.SchemaName}' THEN 1";
}
else
{
return $"WHEN c.TABLE_SCHEMA = '{sattr.SchemaName}' AND c.TABLE_NAME IN ({string.Join(", ", sattr.TableNames.Select(cn => $"'{cn}'"))}) THEN 1";
}
}));
return $@"SELECT c.TABLE_SCHEMA, c.TABLE_NAME, c.COLUMN_NAME, c.DATA_TYPE
, CAST(CASE WHEN ccu.CONSTRAINT_NAME IS NULL THEN 0 ELSE 1 END AS BIT) IsPrimaryKey
FROM INFORMATION_SCHEMA.COLUMNS AS c
LEFT OUTER JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS tc ON tc.TABLE_SCHEMA = c.TABLE_SCHEMA AND tc.TABLE_NAME = c.TABLE_NAME
LEFT OUTER JOIN INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE AS ccu ON ccu.TABLE_SCHEMA = tc.TABLE_SCHEMA AND ccu.TABLE_NAME = tc.TABLE_NAME AND ccu.CONSTRAINT_NAME = tc.CONSTRAINT_NAME AND ccu.COLUMN_NAME = c.COLUMN_NAME
WHERE c.TABLE_SCHEMA in ({tableSchemas})
AND (CASE
{tableList}
ELSE 0
END = 1)
AND c.DATA_TYPE NOT IN ('datetime','uniqueidentifier', 'timestamp')
AND NOT (c.COLUMN_NAME = 'RowID' and CAST(CASE WHEN ccu.CONSTRAINT_NAME IS NULL THEN 0 ELSE 1 END AS BIT) = 1)
AND tc.CONSTRAINT_TYPE = 'Primary Key'
ORDER BY c.TABLE_NAME, c.ORDINAL_POSITION";
}
public IList<Table> GetTables()
{
var tableQuery = GetTableQuery();
return ExecuteQueryDynamic(tableQuery)
.GroupBy(x => new { x.TABLE_SCHEMA, x.TABLE_NAME }, (g, d) => new Table
{
SchemaName = g.TABLE_SCHEMA,
TableName = g.TABLE_NAME,
Columns = d
.Select(y => new Column
{
ColumnName = y.COLUMN_NAME,
DataType = y.DATA_TYPE,
IsPrimaryKey = y.IsPrimaryKey
}),
OrderBy = OrderByStatements?.SingleOrDefault(obs => obs.TableName == $"{g.TABLE_SCHEMA}.{g.TABLE_NAME}")?.OrderBy,
Exclude = ColumnsToBeExcluded?.SingleOrDefault(ctbe => ctbe.TableName == $"{g.TABLE_SCHEMA}.{g.TABLE_NAME}")?.ColumnNames ?? new List<string>(),
Json = JsonColumnsToBeParsed?.SingleOrDefault(ctbe => ctbe.TableName == $"{g.TABLE_SCHEMA}.{g.TABLE_NAME}")?.ColumnNames ?? new List<string>(),
Where = WhereClauses.ContainsKey($"{g.TABLE_SCHEMA}.{g.TABLE_NAME}") ? $"\r\nwhere {WhereClauses[$"{g.TABLE_SCHEMA}.{g.TABLE_NAME}"]}" : ""
}).ToList();
}
public class Table
{
public string SchemaName { get; set; }
public string TableName { get; set; }
public IEnumerable<Column> Columns { get; set; }
public IEnumerable<OrderBy> OrderBy { get; set; }
public IEnumerable<string> Exclude { get; set; }
public IEnumerable<string> Json { get; set; }
public string Where { get; set; } = "";
internal string SchemaTable => $"{SchemaName}.{TableName}";
public string SqlSelect => $"select TOP({TopRows}) \r\n\t{SelectColumns}\r\nfrom {SchemaTable} OBJ{Where}{OrderByStatement}";
internal string SelectColumns => !Columns?.Any() ?? true ? "*" : string.Join(",\r\n\t", Columns.Where(c => !Exclude.ToList().Contains(c.ColumnName)).Select(c => c.ColumnName));
internal string OrderByStatement => !OrderBy?.Any() ?? true ? "" : "\r\nOrder By " + string.Join(", ", OrderBy.Select(ob => $"{ob.ColumnName}{(ob.IsDescending ? " DESC" : "")}"));
}
public class Column
{
public string ColumnName { get; set; }
public string DataType { get; set; }
public bool IsPrimaryKey { get; set; }
}
public class OrderBy
{
public string ColumnName { get; set; }
public bool IsDescending { get; set; } = false;
}
public class TablesAndColumns
{
public string SchemaName { get; set; }
public IEnumerable<string> TableNames { get; set; }
public TablesAndColumns(string schemaName, IEnumerable<string> tables)
{
SchemaName = schemaName;
TableNames = tables;
}
}
public class OrderByInformation
{
public string TableName { get; set; }
public IEnumerable<OrderBy> OrderBy { get; set; }
public OrderByInformation(string tableName, IEnumerable<string> columns)
{
TableName = tableName;
OrderBy = columns.Select(c => new OrderBy
{
ColumnName = c.StartsWith("D|") ? c.Substring(2) : c,
IsDescending = c.StartsWith("D|")
});
}
}
public class ColumnsToExclude
{
public string TableName { get; set; }
public IEnumerable<string> ColumnNames { get; set; }
public ColumnsToExclude(string tableName, IEnumerable<string> columns)
{
TableName = tableName;
ColumnNames = columns;
}
}
public class JsonColumns
{
public string TableName { get; set; }
public IEnumerable<string> ColumnNames { get; set; }
public JsonColumns(string tableName, IEnumerable<string> columns)
{
TableName = tableName;
ColumnNames = columns;
}
}
#endregion
#region Do the comparisons
public void DoTheComparisons(Connections connections, IEnumerable<Table> tables)
{
foreach (var table in tables)
{
var sql = table.SqlSelect;
var dataLegacy = GetData(connections.Legacy, sql, table.TableName);
var dataUpdate = GetData(connections.Update, sql, table.TableName);
var legacy = dataLegacy.Rows;
var update = dataUpdate.Rows;
ParameterExpression pe1 = Expression.Parameter(dataLegacy.Columns["CaseTypeEntityId"].DataType, "CaseTypeEntityId");
ParameterExpression pe2 = Expression.Parameter(dataLegacy.Columns["CostDriver"].DataType, "CostDriver");
IQueryable legacyTable = dataLegacy.AsEnumerable().AsQueryable();
IEnumerable updateTable = dataUpdate.AsEnumerable();
var query = legacyTable.Join(
updateTable,
"new(get_Item(4) as CostDriver, get_Item(34) as CaseTypeEntityId)",
"new(get_Item(4) as CostDriver, get_Item(34) as CaseTypeEntityId)",
"new(outer as legacy, inner as update)"
);
LambdaExpression le = System.Linq.Dynamic.DynamicExpression.ParseLambda(dataLegacy.AsEnumerable().AsQueryable().ElementType,
null, "new(CaseTypeEntityId as FirstKey, CostDriver as SecondKey)", "new(
// var test = update.AsQueryable().MyJoin(legacy.AsQueryable(), "new(CaseTypeEntityId as firstKey, CostDriver as secondKey)",
// "new(CaseTypeEntityId as firstKey, CostDriver as secondKey)",
// "new inner as legacy, outer as update)");
var idxCount = legacy.Count;
var compare = new List<CompareSet>();
var jsonColumns = tables.SingleOrDefault(t => t.SchemaTable == table.TableName)?.Json.ToList();
foreach (var row in Enumerable.Range(0, idxCount))
{
var r = legacy[row];
if (row < update.Count)
{
var u = update[row];
var rJson = JsonConvert.SerializeObject(r.ItemArray);
var uJson = JsonConvert.SerializeObject(u.ItemArray);
if (rJson != uJson)
{
if (jsonColumns == null)
{
compare.Add(new CompareSet(r, u));
}
else
{
compare.Add(new CompareSet(r, u, jsonColumns));
}
}
}
else
{
if (jsonColumns == null)
{
compare.Add(new CompareSet(r, null));
}
else
{
compare.Add(new CompareSet(r, null, jsonColumns));
}
}
}
if (compare.Any())
{
compare.Dump($"{table.TableName} ({compare.Count()}/{dataLegacy.Rows.Count} rows)", 2);
}
else
{
"No differences found in data".Dump($"{table.TableName} (0/{dataLegacy.Rows.Count} rows)");
}
}
}
/*
public static IQueryable MyJoin(this IQueryable outer, IEnumerable inner, string outerSelector, string innerSelector, string resultsSelector, params object[] values)
{
if (inner == null) throw new ArgumentNullException("inner");
if (outerSelector == null) throw new ArgumentNullException("outerSelector");
if (innerSelector == null) throw new ArgumentNullException("innerSelector");
if (resultsSelector == null) throw new ArgumentNullException("resultsSelctor");
LambdaExpression outerSelectorLambda = DynamicExpression.ParseLambda(outer.ElementType, null, outerSelector, values);
LambdaExpression innerSelectorLambda = DynamicExpression.ParseLambda(inner.AsQueryable().ElementType, null, innerSelector, values);
ParameterExpression[] parameters = new ParameterExpression[] {
Expression.Parameter(outer.ElementType, "outer"), Expression.Parameter(inner.AsQueryable().ElementType, "inner") };
LambdaExpression resultsSelectorLambda = DynamicExpression.ParseLambda(parameters, null, resultsSelector, values);
return outer.Provider.CreateQuery(
Expression.Call(
typeof(Queryable), "Join",
new Type[] { outer.ElementType, inner.AsQueryable().ElementType, outerSelectorLambda.Body.Type, resultsSelectorLambda.Body.Type },
outer.Expression, inner.AsQueryable().Expression, Expression.Quote(outerSelectorLambda), Expression.Quote(innerSelectorLambda), Expression.Quote(resultsSelectorLambda)));
}
public static IQueryable<T> Join<T>(this IQueryable<T> outer, IEnumerable<T> inner, string outerSelector, string innerSelector, string resultsSelector, params object[] values)
{
return (IQueryable<T>)Join((IQueryable)outer, (IEnumerable)inner, outerSelector, innerSelector, resultsSelector, values);
}
*/
public class Connections
{
public ServerInstance Legacy { get; set; }
public ServerInstance Update { get; set; }
}
public class ServerInstance
{
public string Server { get; set; }
public string Database { get; set; }
public ServerInstance(string server, string database)
{
Server = server;
Database = database;
}
}
public DataTable GetData(ServerInstance serverInstance, string sql, string tableName)
{
var server = serverInstance.Server;
var database = serverInstance.Database;
var connStr = $"data source={server};initial catalog='{database}';persist security info=True;Integrated Security=SSPI;";
using (var conn = new SqlConnection(connStr))
{
var datatable = new DataTable();
var adapter = new SqlDataAdapter(sql, conn);
try
{
adapter.Fill(datatable);
}
catch (Exception ex)
{
sql.Dump(nameof(sql));
throw;
}
datatable.TableName = tableName;
return datatable;
}
}
public class CompareSet
{
internal IList<string> ColumnNames { get; set; }
internal IList<object> Legacy { get; set; }
internal IList<object> Update { get; set; }
public IList<Row> Rows { get; set; }
internal IList<string> JsonColumns { get; set; }
public CompareSet(DataRow right, DataRow left)
{
ColumnNames = right.Table.Columns.Cast<DataColumn>().Select(dc => dc.ColumnName).ToList();
Legacy = right.ItemArray.ToList();
Update = left?.ItemArray.ToList();
Rows = ColumnNames.Select((c, i) => new Row(c, Legacy[i], Update?[i])).ToList();
}
public CompareSet(DataRow right, DataRow left, IList<string> jsonColumns)
{
ColumnNames = right.Table.Columns.Cast<DataColumn>().Select(dc => dc.ColumnName).ToList();
Legacy = right.ItemArray.ToList();
Update = left?.ItemArray.ToList();
JsonColumns = jsonColumns;
Rows = ColumnNames.Select((c, i) => new Row(c, Legacy[i], Update?[i], JsonColumns.Contains(c, StringComparer.CurrentCultureIgnoreCase))).ToList();
}
}
public class Row
{
public string Column;
public bool IsJson;
public object Left;
public object Right;
public Row(string column, object left, object right)
{
Column = column;
Left = left;
Right = right;
}
public Row(string column, object left, object right, bool isJson)
: this(column, left, right)
{
IsJson = isJson;
if (!isJson) return;
Left = JsonConvert.SerializeObject(JsonConvert.DeserializeObject(left.ToString(),
new JsonSerializerSettings
{
Error = delegate (object sender, Newtonsoft.Json.Serialization.ErrorEventArgs args)
{
args.ErrorContext.Handled = true;
}
}), Newtonsoft.Json.Formatting.Indented);
if (right == null) return;
Right = JsonConvert.SerializeObject(JsonConvert.DeserializeObject(right.ToString(),
new JsonSerializerSettings
{
Error = delegate (object sender, Newtonsoft.Json.Serialization.ErrorEventArgs args)
{
args.ErrorContext.Handled = true;
}
}), Newtonsoft.Json.Formatting.Indented);
}
object ToDump() => new
{
Column = Left.Equals(Right ?? "") ? (object)Column : new XElement("LINQPad.HTML", new XElement("div", new XAttribute("style", "background-color:lightyellow;color:black"), Column)),
Legacy = Left,
Update = Right
};
}
#endregion