chore: deploy initial linqpad queries

2026-05-12 11:33:21 -05:00
parent 0a7a550d46
commit 96d0067f2d
120 changed files with 22939 additions and 0 deletions
@@ -0,0 +1,142 @@
+<Query Kind="Program">
+  <Reference>&lt;ProgramFilesX64&gt;\Microsoft SDKs\Azure\.NET SDK\v2.9\bin\plugins\Diagnostics\Newtonsoft.Json.dll</Reference>
+  <NuGetReference>Amazon.Lambda.S3Events</NuGetReference>
+  <NuGetReference>AWSSDK.Core</NuGetReference>
+  <NuGetReference>AWSSDK.Glue</NuGetReference>
+  <NuGetReference>AWSSDK.S3</NuGetReference>
+  <Namespace>Amazon</Namespace>
+  <Namespace>Amazon.Auth.AccessControlPolicy</Namespace>
+  <Namespace>Amazon.Auth.AccessControlPolicy.ActionIdentifiers</Namespace>
+  <Namespace>Amazon.Internal</Namespace>
+  <Namespace>Amazon.Lambda.S3Events</Namespace>
+  <Namespace>Amazon.MissingTypes</Namespace>
+  <Namespace>Amazon.Runtime</Namespace>
+  <Namespace>Amazon.Runtime.CredentialManagement</Namespace>
+  <Namespace>Amazon.Runtime.CredentialManagement.Internal</Namespace>
+  <Namespace>Amazon.Runtime.EventStreams</Namespace>
+  <Namespace>Amazon.Runtime.EventStreams.Internal</Namespace>
+  <Namespace>Amazon.Runtime.Internal</Namespace>
+  <Namespace>Amazon.Runtime.Internal.Auth</Namespace>
+  <Namespace>Amazon.Runtime.Internal.Settings</Namespace>
+  <Namespace>Amazon.Runtime.Internal.Transform</Namespace>
+  <Namespace>Amazon.Runtime.Internal.Util</Namespace>
+  <Namespace>Amazon.Runtime.SharedInterfaces</Namespace>
+  <Namespace>Amazon.Runtime.SharedInterfaces.Internal</Namespace>
+  <Namespace>Amazon.S3</Namespace>
+  <Namespace>Amazon.S3.Encryption</Namespace>
+  <Namespace>Amazon.S3.Encryption.Internal</Namespace>
+  <Namespace>Amazon.S3.Internal</Namespace>
+  <Namespace>Amazon.S3.IO</Namespace>
+  <Namespace>Amazon.S3.Model</Namespace>
+  <Namespace>Amazon.S3.Model.Internal.MarshallTransformations</Namespace>
+  <Namespace>Amazon.S3.Transfer</Namespace>
+  <Namespace>Amazon.S3.Util</Namespace>
+  <Namespace>Amazon.Util</Namespace>
+  <Namespace>Amazon.Util.Internal</Namespace>
+  <Namespace>Amazon.Util.Internal.PlatformServices</Namespace>
+  <Namespace>Newtonsoft.Json</Namespace>
+  <Namespace>Newtonsoft.Json.Converters</Namespace>
+  <Namespace>Newtonsoft.Json.Linq</Namespace>
+  <Namespace>Newtonsoft.Json.Schema</Namespace>
+  <Namespace>Newtonsoft.Json.Serialization</Namespace>
+  <Namespace>System</Namespace>
+  <Namespace>System.Diagnostics</Namespace>
+  <Namespace>System.Diagnostics.Tracing</Namespace>
+  <Namespace>System.IO.Compression</Namespace>
+  <Namespace>ThirdParty.BouncyCastle.Asn1</Namespace>
+  <Namespace>ThirdParty.BouncyCastle.Asn1.Utilities</Namespace>
+  <Namespace>ThirdParty.BouncyCastle.Math</Namespace>
+  <Namespace>ThirdParty.BouncyCastle.OpenSsl</Namespace>
+  <Namespace>ThirdParty.BouncyCastle.Utilities.IO.Pem</Namespace>
+  <Namespace>ThirdParty.Ionic.Zlib</Namespace>
+  <Namespace>ThirdParty.Json.LitJson</Namespace>
+  <Namespace>ThirdParty.MD5</Namespace>
+  <Namespace>Amazon.Glue</Namespace>
+  <Namespace>Amazon.Glue.Model</Namespace>
+  <Namespace>System.Threading.Tasks</Namespace>
+</Query>
+
+//#define TRACE
+
+void Main()
+{
+	var accessKeyID = Util.GetPassword("aws.sdt-tlamb.accessid");
+	var secretKey = Util.GetPassword("aws.sdt-tlamb.secretid");
+	var bucketName = "sdt-glue-hackweek";
+
+	var s3 = new AmazonS3Client(accessKeyID, secretKey, RegionEndpoint.USEast1);
+
+	var s3Objects = bucketName.AWSListBucket(s3).S3Objects
+	.Where(so => so.Key.StartsWith("wfu/"));
+	s3Objects.Dump("S3 Objects");
+
+	//var crawlerName = "client1_input_icd10dx";
+
+	var glue = new AmazonGlueClient(accessKeyID, secretKey, RegionEndpoint.USEast1);
+	var dbs = glue.GetDatabases(new GetDatabasesRequest()).DatabaseList.Select(dl => dl.Name);
+	var myDBName = "glue-hackweek-thom";
+	if (dbs.Contains(myDBName))
+	{
+		glue.DeleteDatabase(new DeleteDatabaseRequest { Name = myDBName });
+	}
+	glue.CreateDatabase(new CreateDatabaseRequest { DatabaseInput = new DatabaseInput { Name = myDBName, Description = "Thom's Glue Hackweek DB" } });
+	var crawlers = glue.ListCrawlers(new ListCrawlersRequest()).CrawlerNames;
+	var tasks = new List<Task<CreateCrawlerResponse>>();
+	crawlers.Where(c => c.StartsWith("wfu_")).ToList()
+	.ForEach(c =>
+	{
+		try
+		{
+			glue.DeleteCrawler(new DeleteCrawlerRequest { Name = c });
+		}
+		catch (EntityNotFoundException ex)
+		{
+			ex.Message.Dump();
+		}
+	});
+	crawlers = glue.ListCrawlers(new ListCrawlersRequest()).CrawlerNames;
+
+	var tags = new[] { new { key = "environment", value = "hackweek" } }.ToDictionary(dl => dl.key, dl => dl.value);
+	var newCrawlers = s3Objects.ToList()
+	.ConvertAll(o => new CreateCrawlerRequest
+	{
+		Classifiers = new[] { "client1-input-psv" }.ToList(),
+		Configuration = @"{""Version"":1.0,""CrawlerOutput"":{""Partitions"":{""AddOrUpdateBehavior"":""InheritFromTable""}}}",
+		DatabaseName = myDBName,
+		Name = string.Join("_", o.Key.Split('/').TakeAllButLast()),
+		Role = "hackday-glue-role-team2",
+		SchemaChangePolicy = new SchemaChangePolicy
+		{
+			DeleteBehavior = new DeleteBehavior("LOG"),
+			UpdateBehavior = new UpdateBehavior("UPDATE_IN_DATABASE")
+		},
+		TablePrefix = string.Join("_", o.Key.Split('/').TakeAllButLast().TakeAllButLast()) + "_",
+		Tags = tags,
+		Targets = new CrawlerTargets
+		{
+			S3Targets = new List<S3Target> {
+				new S3Target { Path = $"s3://{o.BucketName}/{string.Join("/", o.Key.Split('/').TakeAllButLast())}" }
+				}
+		}
+	}).OrderBy(o => o.Name).ToList();
+	newCrawlers.Dump("New Crawlers");
+	newCrawlers.ForEach(c =>
+	{
+		try
+		{
+			tasks.Add(glue.CreateCrawlerAsync(c, new CancellationToken()));
+		}
+		catch (AmazonGlueException ex)
+		{
+			ex.Dump($"Glue Exception {c.Name}");
+		}
+	});
+	while (tasks.Any(t => !t.IsCompleted))
+	{
+	}
+	var runtasks = new List<Task<StartCrawlerResponse>>();
+	newCrawlers.ForEach(c => runtasks.Add(glue.StartCrawlerAsync(new StartCrawlerRequest { Name = c.Name })));
+	runtasks.Dump("Run Tasks");
+}
+
+// Define other methods and classes here