505 lines
16 KiB
C#
505 lines
16 KiB
C#
<Query Kind="Program">
|
|
<Connection>
|
|
<ID>690bdbda-0918-4699-b08c-addb41e49702</ID>
|
|
<Persist>true</Persist>
|
|
<Server>E0001-sql-11.sdt.local</Server>
|
|
<Database>StrataSphere</Database>
|
|
<ShowServer>true</ShowServer>
|
|
</Connection>
|
|
<Namespace>Newtonsoft.Json</Namespace>
|
|
<Namespace>Newtonsoft.Json.Converters</Namespace>
|
|
<Namespace>Newtonsoft.Json.Linq</Namespace>
|
|
<Namespace>Newtonsoft.Json.Schema</Namespace>
|
|
<Namespace>Newtonsoft.Json.Serialization</Namespace>
|
|
<Namespace>System</Namespace>
|
|
<Namespace>System.ComponentModel</Namespace>
|
|
<Namespace>System.Diagnostics</Namespace>
|
|
<Namespace>System.Diagnostics.Tracing</Namespace>
|
|
<Namespace>System.IO.Compression</Namespace>
|
|
<Namespace>System.Linq.Dynamic</Namespace>
|
|
</Query>
|
|
|
|
// This link provides a good explanation of the Dynamic Query used in the DataTableJoins
|
|
// https://ecs.syr.edu/faculty/fawcett/handouts/CoreTechnologies/CSharp/samples/CSharpSamples/LinqSamples/DynamicQuery/Dynamic%20Expressions.html
|
|
//
|
|
|
|
private List<TablesAndColumns> SchemasAndTablesToReview = new List<TablesAndColumns>();
|
|
|
|
#region Options
|
|
|
|
public static int TopRows = 2000; //int.MaxValue;
|
|
public static bool ShowTableQuery = false;
|
|
public static bool ShowTableList = false;
|
|
public static int TableDumpDepth = int.MaxValue; //2;
|
|
|
|
#endregion
|
|
|
|
void Main()
|
|
{
|
|
SchemasAndTablesToReview.AddRange(new[]
|
|
{
|
|
new TablesAndColumns("stratasphere", null),
|
|
});
|
|
|
|
var tables = GetTables().ToList();
|
|
// Using 'D|' at the beginning of a column name implies that the sort order is descending for that column
|
|
tables.ForEach(table =>
|
|
{
|
|
switch (table.TableName)
|
|
{
|
|
case "Dashboard":
|
|
table.Init(
|
|
new[] { new OrderBy("Name"), new OrderBy("Module"), new OrderBy("MinJazzVersion") },
|
|
new[] { "Id", "DateCreated", "DateLastModified" },
|
|
new[] { "Name", "Module", "MinJazzVersion" }
|
|
);
|
|
break;
|
|
case "DashboardCardItem":
|
|
table.Init(
|
|
new[] { new OrderBy("Name") },
|
|
new[] { "ItemId", "DateCreated", "DateLastModified" },
|
|
new[] { "Name", "DashboardCardItemType" }
|
|
);
|
|
break;
|
|
case "Definition":
|
|
table.Init(
|
|
new[] { new OrderBy("Name"), new OrderBy("DimensionId"), new OrderBy("AttributeId") },
|
|
new[] { "Id", "DateCreated", "DateLastModified" },
|
|
new[] { "Name", "DimensionId", "AttributeId" }
|
|
);
|
|
break;
|
|
case "Download":
|
|
table.Init(
|
|
new[] { new OrderBy("UserName"), new OrderBy("DownloadType") },
|
|
new[] { "DownloadDateTime" },
|
|
new[] { "UserName", "DownloadType" },
|
|
null,
|
|
"OBJ.UserName <> 'System'"
|
|
);
|
|
break;
|
|
case "MDXScoreToCube":
|
|
table.Init(
|
|
new[] { new OrderBy("OrgPin"), new OrderBy("ScoreDimensionName"), new OrderBy("ScoreHierarchyName") },
|
|
new[] { "DateCreated", "DateLastModified" },
|
|
new[] { "OrgPin", "ScoreDimensionName", "ScoreHierarchyName" }
|
|
);
|
|
break;
|
|
case "Parameter":
|
|
table.Init(
|
|
new[] { new OrderBy("Name"), new OrderBy("ParameterType"), new OrderBy("DisplayOrder"), new OrderBy("DimensionName"), new OrderBy("AttributeName") },
|
|
new[] { "DateCreated", "DateLastModified" },
|
|
new[] { "Name", "ParameterType", "DimensionName", "AttributeName" },
|
|
new[] { "ParameterJson" }
|
|
);
|
|
break;
|
|
case "Report":
|
|
table.Init(
|
|
new[] { new OrderBy("Name"), new OrderBy("Module"), new OrderBy("MinJazzVersion") },
|
|
new[] { "ItemId", "DownloadCount", "DateCreated", "DateLastModified" },
|
|
new[] { "Name", "Module", "MinJazzVersion", "ReportType" },
|
|
new[] { "ReportSetupJson" }
|
|
);
|
|
break;
|
|
};
|
|
});
|
|
if (ShowTableList) tables.Dump(nameof(tables));
|
|
var connections = new Connections
|
|
{
|
|
Legacy = new ServerInstance("E0001-SQL-11.sdt.local", "StrataSphere"),
|
|
Update = new ServerInstance("D0011-SQL-01.sdt.local", "StrataSphere")
|
|
// Legacy = new ServerInstance("D0011-SQL-01.sdt.local", "jazz tst Southern Illinois 20190918.1"),
|
|
// Update = new ServerInstance("D0011-SQL-01.sdt.local", "jazz tst Southern Illinois 20200331.1")
|
|
// Legacy = new ServerInstance("D0001-SQL-11.sdt.local", "jazz tst master prod 20190708.1"),
|
|
// Update = new ServerInstance("D0001-SQL-11.sdt.local", "jazz tst master prod")
|
|
};
|
|
DoTheComparisons(connections, tables);
|
|
}
|
|
|
|
// Define other methods and classes here
|
|
|
|
#region Gather Table Information
|
|
|
|
public string GetTableQuery()
|
|
{
|
|
var tableSchemas = string.Join(", ", SchemasAndTablesToReview.Select(sattr => $"'{sattr.SchemaName}'"));
|
|
var tableList = string.Join("\r\n", SchemasAndTablesToReview.ConvertAll(sattr =>
|
|
{
|
|
if (sattr.TableNames?.Any(cn => cn == "*") ?? true)
|
|
{
|
|
return $"WHEN c.TABLE_SCHEMA = '{sattr.SchemaName}' THEN 1";
|
|
}
|
|
else
|
|
{
|
|
return $"WHEN c.TABLE_SCHEMA = '{sattr.SchemaName}' AND c.TABLE_NAME IN ({string.Join(", ", sattr.TableNames.Select(cn => $"'{cn}'"))}) THEN 1";
|
|
}
|
|
}));
|
|
return $@"SELECT c.TABLE_SCHEMA, c.TABLE_NAME, c.COLUMN_NAME, c.DATA_TYPE
|
|
, CAST(CASE WHEN ccu.CONSTRAINT_NAME IS NULL THEN 0 ELSE 1 END AS BIT) IsPrimaryKey
|
|
FROM INFORMATION_SCHEMA.COLUMNS AS c
|
|
LEFT OUTER JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS tc ON tc.TABLE_SCHEMA = c.TABLE_SCHEMA AND tc.TABLE_NAME = c.TABLE_NAME
|
|
LEFT OUTER JOIN INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE AS ccu ON ccu.TABLE_SCHEMA = tc.TABLE_SCHEMA AND ccu.TABLE_NAME = tc.TABLE_NAME AND ccu.CONSTRAINT_NAME = tc.CONSTRAINT_NAME AND ccu.COLUMN_NAME = c.COLUMN_NAME
|
|
WHERE c.TABLE_SCHEMA in ({tableSchemas})
|
|
AND (CASE
|
|
{tableList}
|
|
ELSE 0
|
|
END = 1)
|
|
AND c.DATA_TYPE NOT IN ('datetime','uniqueidentifier', 'timestamp')
|
|
AND NOT (c.COLUMN_NAME = 'RowID' and CAST(CASE WHEN ccu.CONSTRAINT_NAME IS NULL THEN 0 ELSE 1 END AS BIT) = 1)
|
|
AND tc.CONSTRAINT_TYPE = 'Primary Key'
|
|
ORDER BY c.TABLE_NAME, c.ORDINAL_POSITION";
|
|
}
|
|
|
|
public IList<Table> GetTables()
|
|
{
|
|
var tableQuery = GetTableQuery();
|
|
if (ShowTableQuery) tableQuery.Dump(nameof(tableQuery));
|
|
return ExecuteQueryDynamic(tableQuery)
|
|
.GroupBy(x => new { x.TABLE_SCHEMA, x.TABLE_NAME },
|
|
(g, d) => new Table(g.TABLE_SCHEMA, g.TABLE_NAME,
|
|
d.Select(y => new Column(y.COLUMN_NAME, y.DATA_TYPE, y.IsPrimaryKey)).ToList()))
|
|
.ToList();
|
|
}
|
|
|
|
public class Table
|
|
{
|
|
public string SchemaName { get; set; }
|
|
public string TableName { get; set; }
|
|
public IList<Column> Columns { get; set; } = new List<Column>();
|
|
public IList<OrderBy> OrderBy { get; set; } = new List<OrderBy>();
|
|
public IList<string> Exclude { get; set; } = new List<string>();
|
|
public IList<string> KeyColumn { get; set; } = new List<string>();
|
|
public IList<string> Json { get; set; } = new List<string>();
|
|
public string Where { get; set; } = "";
|
|
internal string SchemaTable => $"{SchemaName}.{TableName}";
|
|
public string SqlSelect => $"select TOP({TopRows}) \r\n\t{SelectColumns}\r\nfrom {SchemaTable} OBJ{Where}{OrderByStatement}";
|
|
internal string SelectColumns => !Columns?.Any() ?? true ? "*" : string.Join(",\r\n\t", Columns.Where(c => !Exclude.ToList().Contains(c.ColumnName)).Select(c => c.ColumnName));
|
|
internal string OrderByStatement => !OrderBy?.Any() ?? true ? "" : "\r\nOrder By " + string.Join(", ", OrderBy.Select(ob => $"{ob.ColumnName}{(ob.IsDescending ? " DESC" : "")}"));
|
|
|
|
public Table(string schemaName, string tableName)
|
|
{
|
|
SchemaName = schemaName;
|
|
TableName = tableName;
|
|
Columns = new List<Column>();
|
|
OrderBy = new List<OrderBy>();
|
|
}
|
|
|
|
public Table(string schemaName, string tableName, IEnumerable<Column> columns)
|
|
: this(schemaName, tableName)
|
|
{
|
|
Columns = columns.ToList();
|
|
}
|
|
|
|
public Table(string schemaName, string tableName, IEnumerable<Column> columns, IEnumerable<OrderBy> orderBy)
|
|
: this(schemaName, tableName, columns)
|
|
{
|
|
OrderBy = orderBy.ToList();
|
|
}
|
|
|
|
/// <sumarry>
|
|
///
|
|
/// </summary>
|
|
/// <param name="oderBys">
|
|
/// Using 'D|' at the beginning of a column name implies that the sort order is descending for that column
|
|
/// <param>
|
|
public void Init(IEnumerable<OrderBy> orderBys, IEnumerable<string> excludes = null, IEnumerable<string> keyColumns = null, IEnumerable<string> jsonColumns = null, string where = "")
|
|
{
|
|
if (orderBys != null)
|
|
OrderBys(orderBys);
|
|
if (excludes != null)
|
|
Excludes(excludes);
|
|
if (keyColumns != null)
|
|
KeyColumns(keyColumns);
|
|
if (jsonColumns != null)
|
|
JsonColumns(jsonColumns);
|
|
if (!string.IsNullOrEmpty(where))
|
|
where = $"\r\nwhere {where}";
|
|
}
|
|
|
|
/// <sumarry>
|
|
/// Using 'D|' at the beginning of a column name implies that the sort order is descending for that column
|
|
/// </summary>
|
|
public void OrderBys(IEnumerable<OrderBy> orderBys)
|
|
{
|
|
OrderBy.Clear();
|
|
((List<OrderBy>)OrderBy).AddRange(orderBys.Where(o => Columns.Select(c => c.ColumnName).Contains(o.ColumnName, StringComparer.CurrentCultureIgnoreCase)));
|
|
}
|
|
|
|
public void Excludes(IEnumerable<string> excludes)
|
|
{
|
|
Exclude.Clear();
|
|
((List<string>)Exclude).AddRange(excludes.Where(e => Columns.Select(c => c.ColumnName).Contains(e, StringComparer.CurrentCultureIgnoreCase)));
|
|
}
|
|
|
|
public void KeyColumns(IEnumerable<string> keyColumns)
|
|
{
|
|
KeyColumn.Clear();
|
|
((List<string>)KeyColumn).AddRange(keyColumns.Where(kc => Columns.Select(c => c.ColumnName).Contains(kc, StringComparer.CurrentCultureIgnoreCase)));
|
|
}
|
|
|
|
public void JsonColumns(IEnumerable<string> jsonColumns)
|
|
{
|
|
Json.Clear();
|
|
((List<string>)Json).AddRange(jsonColumns.Where(jc => Columns.Select(c => c.ColumnName).Contains(jc, StringComparer.CurrentCultureIgnoreCase)));
|
|
}
|
|
}
|
|
|
|
public class Column
|
|
{
|
|
public string ColumnName { get; set; }
|
|
public string DataType { get; set; }
|
|
public bool IsPrimaryKey { get; set; }
|
|
public Column(string columnName, string dataType = "", bool isPrimaryKey = false)
|
|
{
|
|
ColumnName = columnName;
|
|
DataType = dataType;
|
|
IsPrimaryKey = isPrimaryKey;
|
|
}
|
|
}
|
|
|
|
public class OrderBy
|
|
{
|
|
public string ColumnName { get; set; }
|
|
public bool IsDescending { get; set; } = false;
|
|
|
|
public OrderBy(string columnName, bool isDescending = false)
|
|
{
|
|
ColumnName = columnName;
|
|
IsDescending = isDescending;
|
|
}
|
|
|
|
}
|
|
|
|
public class TablesAndColumns
|
|
{
|
|
public string SchemaName { get; set; }
|
|
public IEnumerable<string> TableNames { get; set; }
|
|
public TablesAndColumns(string schemaName, IEnumerable<string> tables)
|
|
{
|
|
SchemaName = schemaName;
|
|
TableNames = tables;
|
|
}
|
|
}
|
|
|
|
public class OrderByInformation
|
|
{
|
|
public string TableName { get; set; }
|
|
public IEnumerable<OrderBy> OrderBy { get; set; }
|
|
public OrderByInformation(string tableName, IEnumerable<string> columns)
|
|
{
|
|
TableName = tableName;
|
|
OrderBy = columns
|
|
.Select(c =>
|
|
new OrderBy(c.StartsWith("D|") ? c.Substring(2) : c,
|
|
c.StartsWith("D|")));
|
|
}
|
|
}
|
|
|
|
public class ColumnsToExclude
|
|
{
|
|
public string TableName { get; set; }
|
|
public IEnumerable<string> ColumnNames { get; set; }
|
|
public ColumnsToExclude(string tableName, IEnumerable<string> columns)
|
|
{
|
|
TableName = tableName;
|
|
ColumnNames = columns;
|
|
}
|
|
}
|
|
|
|
public class JsonColumns
|
|
{
|
|
public string TableName { get; set; }
|
|
public IEnumerable<string> ColumnNames { get; set; }
|
|
public JsonColumns(string tableName, IEnumerable<string> columns)
|
|
{
|
|
TableName = tableName;
|
|
ColumnNames = columns;
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Do the comparisons
|
|
|
|
public void DoTheComparisons(Connections connections, IEnumerable<Table> tables)
|
|
{
|
|
foreach (var table in tables)
|
|
{
|
|
var sql = table.SqlSelect;
|
|
var dataLegacy = GetData(connections.Legacy, sql, table.TableName);
|
|
var dataUpdate = GetData(connections.Update, sql, table.TableName);
|
|
|
|
var compare = new List<CompareSet>();
|
|
var jsonColumns = table.Json.ToList() ?? null;
|
|
dataLegacy.AsEnumerable().ToList()
|
|
.ForEach(legacyRow =>
|
|
{
|
|
DataRow updateRow = default(DataRow);
|
|
if (table.KeyColumn.Any())
|
|
{
|
|
var keys = string.Join("|", table.KeyColumn.Select(kc => $"{legacyRow.Field<object>(kc)}"));
|
|
updateRow = dataUpdate.AsEnumerable()
|
|
.FirstOrDefault(y => keys == string.Join("|", table.KeyColumn.Select(kc => $"{y.Field<object>(kc)}")));
|
|
}
|
|
if (updateRow == null || JsonConvert.SerializeObject(legacyRow.ItemArray) != JsonConvert.SerializeObject(updateRow.ItemArray))
|
|
compare.Add(new CompareSet(compare.Count, legacyRow, updateRow, jsonColumns));
|
|
});
|
|
dataUpdate.AsEnumerable().ToList()
|
|
.ForEach(updateRow =>
|
|
{
|
|
DataRow legacyRow = default(DataRow);
|
|
if (table.KeyColumn.Any())
|
|
{
|
|
var keys = string.Join("|", table.KeyColumn.Select(kc => $"{updateRow.Field<object>(kc)}"));
|
|
legacyRow = dataLegacy.AsEnumerable()
|
|
.FirstOrDefault(y => keys == string.Join("|", table.KeyColumn.Select(kc => $"{y.Field<object>(kc)}")));
|
|
}
|
|
if (legacyRow == null)
|
|
compare.Add(new CompareSet(compare.Count, legacyRow, updateRow, jsonColumns));
|
|
});
|
|
if (compare.Any())
|
|
{
|
|
compare.OrderBy(c => c.OrderBy)
|
|
.Dump($"{table.SchemaTable} ( {compare.Count()} / {dataLegacy.Rows.Count} / {dataUpdate.Rows.Count} rows )", TableDumpDepth);
|
|
}
|
|
else
|
|
{
|
|
"No differences found in data".Dump($"{table.SchemaTable} ( 0 / {dataLegacy.Rows.Count} / {dataUpdate.Rows.Count} rows )");
|
|
}
|
|
}
|
|
}
|
|
|
|
public class Connections
|
|
{
|
|
public ServerInstance Legacy { get; set; }
|
|
public ServerInstance Update { get; set; }
|
|
}
|
|
|
|
public class ServerInstance
|
|
{
|
|
public string Server { get; set; }
|
|
public string Database { get; set; }
|
|
|
|
public ServerInstance(string server, string database)
|
|
{
|
|
Server = server;
|
|
Database = database;
|
|
}
|
|
}
|
|
|
|
public DataTable GetData(ServerInstance serverInstance, string sql, string tableName)
|
|
{
|
|
var server = serverInstance.Server;
|
|
var database = serverInstance.Database;
|
|
var connStr = $"data source={server};initial catalog='{database}';persist security info=True;Integrated Security=SSPI;";
|
|
using (var conn = new SqlConnection(connStr))
|
|
{
|
|
var datatable = new DataTable();
|
|
var adapter = new SqlDataAdapter(sql, conn);
|
|
try
|
|
{
|
|
adapter.Fill(datatable);
|
|
}
|
|
catch (Exception)
|
|
{
|
|
sql.Dump(nameof(sql));
|
|
throw;
|
|
}
|
|
datatable.TableName = tableName;
|
|
return datatable;
|
|
}
|
|
}
|
|
|
|
public class CompareSet
|
|
{
|
|
internal int Row { get; set; }
|
|
internal IList<string> ColumnNames { get; set; }
|
|
internal IList<object> Legacy { get; set; }
|
|
internal IList<object> Update { get; set; }
|
|
internal int OrderBy { get; set; } = 0;
|
|
public IList<Row> Rows { get; set; }
|
|
internal IList<string> JsonColumns { get; set; }
|
|
public CompareSet(int row, DataRow right, DataRow left, IList<string> jsonColumns)
|
|
{
|
|
Row = ++row;
|
|
if (right == null)
|
|
{
|
|
ColumnNames = left.Table.Columns.Cast<DataColumn>().Select(dc => dc.ColumnName).ToList();
|
|
}
|
|
else
|
|
{
|
|
ColumnNames = right.Table.Columns.Cast<DataColumn>().Select(dc => dc.ColumnName).ToList();
|
|
}
|
|
Legacy = right?.ItemArray.ToList();
|
|
Update = left?.ItemArray.ToList();
|
|
OrderBy = (right != null ? 0 : 2) + (left != null ? 0 : 1);
|
|
JsonColumns = jsonColumns;
|
|
Rows = ColumnNames.Select((c, i) => new Row(row, c, Legacy?[i], Update?[i], JsonColumns?.Contains(c, StringComparer.CurrentCultureIgnoreCase) ?? false)).ToList();
|
|
}
|
|
|
|
public CompareSet(int row, DataRow right, DataRow left)
|
|
: this(row, right, left, null)
|
|
{
|
|
}
|
|
|
|
object ToDump() => new
|
|
{
|
|
Row = $"{Row}",
|
|
Rows = Rows.Select(x => new
|
|
{
|
|
Column = (x.Left ?? "").Equals(x.Right ?? "") ? (object)x.Column : new XElement("LINQPad.HTML", new XElement("div", new XAttribute("style", "background-color:lightyellow;color:black"), x.Column)),
|
|
Legacy = x.Left,
|
|
Update = x.Right
|
|
})
|
|
};
|
|
|
|
}
|
|
|
|
public class Row
|
|
{
|
|
public int RowId { get; set; }
|
|
public string Column;
|
|
public bool IsJson;
|
|
public object Left;
|
|
public object Right;
|
|
|
|
public Row(int row, string column, object left, object right)
|
|
{
|
|
RowId = row;
|
|
Column = column;
|
|
Left = left;
|
|
Right = right;
|
|
}
|
|
|
|
public Row(int row, string column, object left, object right, bool isJson)
|
|
: this(row, column, left, right)
|
|
{
|
|
|
|
IsJson = isJson;
|
|
if (!isJson) return;
|
|
if (left != null)
|
|
{
|
|
Left = JsonConvert.SerializeObject(JsonConvert.DeserializeObject(left.ToString(),
|
|
new JsonSerializerSettings
|
|
{
|
|
Error = delegate (object sender, Newtonsoft.Json.Serialization.ErrorEventArgs args)
|
|
{
|
|
args.ErrorContext.Handled = true;
|
|
}
|
|
}), Newtonsoft.Json.Formatting.Indented);
|
|
}
|
|
if (right != null)
|
|
{
|
|
Right = JsonConvert.SerializeObject(JsonConvert.DeserializeObject(right.ToString(),
|
|
new JsonSerializerSettings
|
|
{
|
|
Error = delegate (object sender, Newtonsoft.Json.Serialization.ErrorEventArgs args)
|
|
{
|
|
args.ErrorContext.Handled = true;
|
|
}
|
|
}), Newtonsoft.Json.Formatting.Indented);
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
#endregion |