linqpad/CrossDBQuery.linq

<Query Kind="Program">
  <Reference>&lt;ProgramFilesX64&gt;\Microsoft SDKs\Azure\.NET SDK\v2.9\bin\plugins\Diagnostics\Newtonsoft.Json.dll</Reference>
  <Namespace>Newtonsoft.Json</Namespace>
  <Namespace>Newtonsoft.Json.Converters</Namespace>
  <Namespace>Newtonsoft.Json.Linq</Namespace>
  <Namespace>Newtonsoft.Json.Schema</Namespace>
  <Namespace>Newtonsoft.Json.Serialization</Namespace>
  <Namespace>System</Namespace>
  <Namespace>System.Diagnostics</Namespace>
  <Namespace>System.Diagnostics.Tracing</Namespace>
  <Namespace>System.Dynamic</Namespace>
  <Namespace>System.IO.Compression</Namespace>
  <Namespace>System.Threading.Tasks</Namespace>
  <Namespace>System.Data.Common</Namespace>
</Query>

#region *** Enumerations ***

public enum QueryRegions
{
	Production,
	Training,
	Oak,
	Dev
}

#endregion

#region *** Options ***

public static QueryRegions queryRegion = QueryRegions.Dev;
public static string crossDBQueryPath = @"C:\Users\tlamb\Documents\CrossDBQueries"; // This is the path to find the queries
public static bool checkForCILicensing = true; // This flag will add a column if the client db is licensed for CI
public static bool dumpOptions = false; // This flag will dumpt this set of options
public static bool dumpSmcSql = false; // This flag will dump the smc sql to the result
public static bool dumpDBList = false; // This flag will dump the databases to the result
public static bool dumpQueries = false; // This flag will dump the queries to the result
public static bool dumpDBExclusions = false; // This flag will dump the db and server exclusions
public static bool dumpDBTargets = false; // This flag will dump the db and server targets
public static bool dumpAppVersions = false; // This flag will dump the app versions used in the queryRegion
public static bool dumpAppVersionTargets = false; // This flag will dump the app versions targets
public static bool excludeExceptions = true;
public static bool queryCompress = false;
public static bool flatten = true; // This will flatten all of the results into a single list
public static bool showStopWatch = false;
public static bool noQueries = false; // This will stop the CrossDBQuery without running the queries

#endregion

public static Parms parms = new Parms
{
	name = "FY2020"
};

#region *** Database Selection ***

public static string[] appVersions = { // This is a list of appVersions to filter for in the database list
	//"2020.31"
};
public static string[] serverExclusions = { // This is a list of servers to exclude from the database list
	@"paonsql19.sdt.local\instance1",
	@"paonsql17.sdt.local\instance1",
	@"p0555-sql-01.sdt.local\instance1",
	"P0000-sql-01.sdt.local",
	"P9900-SQL-03.sdt.local",
	"P9900-sql-05.sdt.local"
};
public static string[] dbExclusions = { // This is a list of database physical names to exclude from the database list
};
public static string[] targetOrgPins = { // This is a list of org pins to target in the database list
//	"0430", "0434", "0555", "0777"
};
public static string[] targetServers = { // This is a list of servers to target in the database list
//	"P9900-SQL-11.sdt.local"
//	"D0018-SQL-01.sdt.local"
};
public static string[] targetDBs = { // This is a list of database physicial names to target in the database list
//	"jazz prd Gillette Childrens 20150417.1"
//	"jazz prd Southern Illinois 20140408.1"
//	"jazz tst Southern Illinois 20190918.1"
//	"jazz dev master Prod"
};

#endregion *** Options ***

#region *** SMC Settings and queries ***

public static string smcSql => $@"SELECT di.ServerName, di.DatabaseGUID, di.DatabaseName, di.PhysicalName, di.OrgPin, di.AppVersion, di.Description
{(checkForCILicensing ? "   ,(select COUNT(*) from [dbo].[viewLicenseLink] l where l.NAME = 'ContinuousImprovement' and l.DatabaseGUID = di.DatabaseGUID and l.IsLicensed = 1) CI_LICENSED" : "")}
FROM [dbo].[viewDatabaseInfo] di
WHERE 1=1"
+ (queryRegion == QueryRegions.Dev
? @"
  AND di.PhysicalName like 'jazz%'"
: $@"
  AND di.PhysicalName like 'jazz {(queryRegion != QueryRegions.Production ? "tst" : "prd")}%'
  AND di.ServerName like 'p%'");
public static string smcServer { get; set; }
public static string smcDB { get; set; }

#endregion

#region *** Program variables and settings ***

public static int maxThreads = 8;
public static int timeoutInMinutes = 2;
public static IList<CrossDBQueryResult> crossDBQueryResults = new List<CrossDBQueryResult>();
public static IOrderedEnumerable<Database> databases => GetDatabases();
public static Dictionary<string, IEnumerable<string>> queries = System.IO.Directory.Exists(crossDBQueryPath)
	? GetUserQueries(crossDBQueryPath)
	: new Dictionary<string, IEnumerable<string>>();

#endregion

void Main()
{
	if (dumpOptions)
	{
		OptionsDump();
	}
	if (dumpDBExclusions)
	{
		dbExclusions.Dump(nameof(dbExclusions));
		serverExclusions.Dump(nameof(serverExclusions));
	}
	if (dumpDBTargets)
	{
		targetOrgPins.Dump(nameof(targetOrgPins));
		targetDBs.Dump(nameof(targetDBs));
		targetServers.Dump(nameof(targetServers));
	}
	if (dumpAppVersionTargets)
	{
		appVersions.Dump(nameof(appVersions));
	}
	if (dumpDBList)
	{
		databases.OrderBy(d => d.Server).ThenBy(d => d.PhysicalName).Dump($"{queryRegion} {nameof(databases)}");
	}
	if (dumpAppVersions)
	{
		databases
		.GroupBy(db => db.AppVersion, (grp, data) => data.Select(x => x.AppVersion).First())
		.OrderByDescending(db => db)
		.Dump($"{queryRegion} {nameof(appVersions)}");
	}
	if (noQueries) return;
	var results = ExecuteCrossDBQueries()
	.OrderBy(dbqr => dbqr.Results.Any() ? 0 : 1)
	.ThenBy(dbqr => dbqr.Database.DatabaseName);
	if (flatten)
	{
		results
		.SelectMany(dbqr => dbqr.Results.SelectMany(r => r.Results).ToList())
		.Where(dbqr => dbqr.CI_LICENSED == "1")
		.GroupBy(dbqr => dbqr.DatabaseGUID, (g,d) => d.First())
		.Distinct()
		.Dump(nameof(crossDBQueryResults));
	}
	else
	{
		results.Dump(nameof(crossDBQueryResults));
	}
}

public void OptionsDump()
{
	var options = new
	{
		queryRegion,
		crossDBQueryPath,
		dumpOptions,
		dumpSmcSql,
		dumpDBList,
		dumpQueries,
		dumpDBExclusions,
		dumpDBTargets,
		dumpAppVersions,
		dumpAppVersionTargets,
		excludeExceptions,
		queryCompress,
		flatten,
		showStopWatch,
		noQueries
	};
	options.Dump(nameof(options));
}

public class Parms
{
	public string name { get; set; }
}

#region *** Other classes ***

public class CrossDBQueryResultLite
{
	public Database Database { get; set; }
	public List<SqlResults> Results { get; set; }
}
public class CrossDBQueryResult : CrossDBQueryResultLite
{
	public List<string> Exceptions { get; set; }
	public CrossDBQueryResult()
	{
		Results = new List<SqlResults>();
		Exceptions = new List<string>();
	}

	public CrossDBQueryResult(Database database, string name, IEnumerable<dynamic> enumerable)
	: this()
	{
		Database = database;
		Results.Add(new SqlResults(name, enumerable));
	}

	public CrossDBQueryResult(Database database, Exception exception)
	: this()
	{
		Database = database;
		Exceptions.Add(exception.ExceptionMessage());
	}

	object ToDump() => excludeExceptions
	? (object)new
	{
		Database,
		Results
	}
: new
{
	Database,
	Results,
	Exceptions
};
}

public class SqlResults
{
	public string Name { get; set; }
	public IEnumerable<dynamic> Results { get; set; }
	public SqlResults(string name, IEnumerable<dynamic> results)
	{
		Name = name;
		Results = results;
	}
}

public static class ExceptionExtension
{
	public static string ExceptionMessage(this Exception exception)
	{
		return $@"{exception.Message}{(exception.InnerException != null
		? $@"
		{"".PadLeft(40, '-')}
		{exception.InnerException.ExceptionMessage()}"
		: "")}";
	}
}

public class Database
{
	public string Server { get; set; }
	public Guid DatabaseGuid { get; set; }
	public string DatabaseName { get; set; }
	public string PhysicalName { get; set; }
	public string OrgPin { get; set; }
	public string AppVersion { get; set; }
	public string Description { get; set; }
	public Dictionary<string, string> Fields { get; set; }

	public Database()
	{

	}

	public Database(DataRow dr)
	{
		Server = dr.Field<string>("ServerName");
		DatabaseGuid = dr.Field<Guid>("DatabaseGuid");
		DatabaseName = dr.Field<string>("DatabaseName");
		PhysicalName = dr.Field<string>("PhysicalName");
		OrgPin = dr.Field<string>("OrgPin");
		AppVersion = dr.Field<string>("AppVersion");
		Description = dr.Field<string>("Description");
		Fields = new Dictionary<string, string>();
		foreach (DataColumn column in dr.Table.Columns)
		{
			var index = dr.Table.Columns.IndexOf(column);
			Fields.Add(column.ColumnName, dr[index].ToString());
		}
	}

	object ToDump() => (object)new
	{
		Server,
		DatabaseGuid,
		DatabaseName,
		PhysicalName,
		OrgPin,
		AppVersion,
		Description
	};
}

public class ThreadWithState
{
	private string connectionString => BuildConnectionString(_server, _databaseName);
	private Database _database;
	private string _server => _database?.Server ?? string.Empty;
	private string _databaseName => _database?.PhysicalName ?? string.Empty;
	private string sqlName;
	private IEnumerable<string> _sqlQueries;

	public ThreadWithState(Database database, KeyValuePair<string, IEnumerable<string>> sqlqueries)
	{
		_database = database;
		sqlName = sqlqueries.Key;
		_sqlQueries = sqlqueries.Value;
	}

	public void ThreadProc()
	{
		var crossDB = crossDBQueryResults.FirstOrDefault(x => x.Database == _database);
		IEnumerable<dynamic> ThreadResults;
		foreach (var sqlQuery in _sqlQueries)
		{
			try
			{
				ThreadResults = SelectDynamic(_database, connectionString, sqlQuery);
				var result = ThreadResults.First();
				if (crossDB != null)
				{
					crossDB.Results.Add(new SqlResults(sqlName, ThreadResults));
				}
				else
				{
					crossDB = new CrossDBQueryResult(_database, sqlName, ThreadResults);
					crossDBQueryResults.Add(crossDB);
				}
			}
			catch (Exception ex)
			{
				if (crossDB != null)
				{
					crossDB.Exceptions.Add(new Exception(sqlQuery, ex).ExceptionMessage());
				}
				else
				{
					crossDBQueryResults.Add(new CrossDBQueryResult(_database, new Exception(sqlQuery, ex)));
				}
			}
		}
	}
}

#endregion

#region *** Other methods here ***

public static IOrderedEnumerable<Database> GetDatabases()
{
	switch (queryRegion)
	{
		case QueryRegions.Production:
			smcServer = "ioak-sql-smc-01";
			smcDB = "smc prod";
			break;
		case QueryRegions.Training:
			smcServer = "ioak-sql-smc-01";
			smcDB = "smc training";
			break;
		case QueryRegions.Oak:
			smcServer = "ioak-sql-smc-01.sdt.local";
			smcDB = "smc qa";
			break;
		default:
			smcServer = "DDEN-SQL-SMC-01.sdt.local";
			smcDB = "smc dev";
			break;
	}
	if (dumpSmcSql) smcSql.Dump(nameof(smcSql));
	List<Database> databases = new List<UserQuery.Database>();
	var connStr = BuildConnectionString(smcServer, smcDB);
	using (var conn = new SqlConnection(connStr))
	{
		var command = new SqlCommand(smcSql, conn);
		command.CommandType = CommandType.Text;
		var database = new DataSet();
		conn.Open();
		using (DbDataReader reader = command.ExecuteReader())
		{
			while (true)
			{
				var table = new DataTable();
				database.Tables.Add(table);
				table.Load(reader);
				if (reader.IsClosed || !reader.NextResult()) break;
			}
		}
		conn.Close();
		databases = database.Tables[0].AsEnumerable().Select(dr => new Database(dr)).ToList();
		databases = databases.Where(dr => !dbExclusions.Contains(dr.PhysicalName) && !serverExclusions.Contains(dr.Server)).ToList();
		if (targetOrgPins.Any())
			databases = databases.Where(dr => targetOrgPins.Contains(dr.OrgPin)).ToList();
		if (targetServers.Any())
			databases = databases.Where(dr => targetServers.Contains(dr.Server)).ToList();
		if (targetDBs.Any())
			databases = databases.Where(dr => targetDBs.Contains(dr.PhysicalName)).ToList();
		if (appVersions.Any())
			databases = databases.Where(dr => appVersions.Contains(dr.AppVersion)).ToList();
	}
	return databases
	.OrderBy(d => d.Server).ThenBy(d => d.PhysicalName);
}

public static Dictionary<string, IEnumerable<string>> GetUserQueries(string path = null)
{
	var grp = 0;
	var queries = Directory.GetFiles(path ?? Directory.GetCurrentDirectory(), "*.sql", SearchOption.TopDirectoryOnly)
	.ToDictionary(d => Path.GetFileNameWithoutExtension(d), f => File.ReadAllLines(f)
	.Select(r =>
	{
		if (r.Equals("go", StringComparison.CurrentCultureIgnoreCase))
		{
			r = "";
			return new { grp = grp++, r };
		}
		else
		{
			return new { grp, r };
		}
	}).GroupBy(r => r.grp, (x, lines) => string.Join(queryCompress ? " " : "\r\n", lines.Select(l => l.r).Where(l => !string.IsNullOrEmpty(l)))));

	if (dumpQueries)
		queries.Dump(nameof(queries));
	return queries;
}

public static IEnumerable<dynamic> SelectDynamic(Database db, string connStr, string sql)
{
	using (var conn = new SqlConnection(connStr))
	{
		var command = new SqlCommand(sql, conn);
		command.CommandType = CommandType.Text;
		command.CommandTimeout = (int)TimeSpan.FromMinutes(timeoutInMinutes).TotalSeconds;
		var database = new DataSet();
		conn.Open();
		using (DbDataReader reader = command.ExecuteReader())
		{
			while (true)
			{
				var names = Enumerable.Range(0, reader.FieldCount).Select(reader.GetName).ToList();
				foreach (IDataRecord record in reader as IEnumerable)
				{
					var expando = new ExpandoObject() as IDictionary<string, object>;
					if (flatten)
					{
						foreach (var field in db.Fields)
						{
							expando[field.Key] = field.Value;
						}
					}
					foreach (var name in names)
						expando[name] = record[name];

					yield return expando;
				}
				if (reader.IsClosed || !reader.NextResult()) break;
			}
		}
		conn.Close();
	}
}

public IList<CrossDBQueryResult> ExecuteCrossDBQueries()
{
	RunCrossDBQuery(databases, queries);
	return crossDBQueryResults;
}

public void RunCrossDBQuery(IEnumerable<Database> databases, Dictionary<string, IEnumerable<string>> sqlList)
{
	var stopWatch = DateTime.Now;
	var query = databases.AsParallel()
	.SelectMany(database => sqlList.Where(sql => sql.Value.Any()).AsParallel()
	.Select(sql => new ThreadWithState(database, sql)));
	foreach (var q in query)
	{
		q.ThreadProc();
	}
	var timer = DateTime.Now.Subtract(stopWatch).TotalSeconds;
	if (showStopWatch) timer.Dump(nameof(stopWatch));
}

public static string BuildConnectionString(string server, string database, string username = null, string password = null)
{
	return username == null
	? $"data source={server};initial catalog='{database}';persist security info=True;Integrated Security=SSPI;"
	: $"data source={server};initial catalog='{database}';User ID={username};pwd={password}";
}

#endregion