linqpad/HackWeek AWS/HackWeek AWS Glue Bucket to Crawlers.linq

<Query Kind="Program">
  <Reference>&lt;ProgramFilesX64&gt;\Microsoft SDKs\Azure\.NET SDK\v2.9\bin\plugins\Diagnostics\Newtonsoft.Json.dll</Reference>
  <NuGetReference>Amazon.Lambda.S3Events</NuGetReference>
  <NuGetReference>AWSSDK.Core</NuGetReference>
  <NuGetReference>AWSSDK.Glue</NuGetReference>
  <NuGetReference>AWSSDK.S3</NuGetReference>
  <Namespace>Amazon</Namespace>
  <Namespace>Amazon.Auth.AccessControlPolicy</Namespace>
  <Namespace>Amazon.Auth.AccessControlPolicy.ActionIdentifiers</Namespace>
  <Namespace>Amazon.Internal</Namespace>
  <Namespace>Amazon.Lambda.S3Events</Namespace>
  <Namespace>Amazon.MissingTypes</Namespace>
  <Namespace>Amazon.Runtime</Namespace>
  <Namespace>Amazon.Runtime.CredentialManagement</Namespace>
  <Namespace>Amazon.Runtime.CredentialManagement.Internal</Namespace>
  <Namespace>Amazon.Runtime.EventStreams</Namespace>
  <Namespace>Amazon.Runtime.EventStreams.Internal</Namespace>
  <Namespace>Amazon.Runtime.Internal</Namespace>
  <Namespace>Amazon.Runtime.Internal.Auth</Namespace>
  <Namespace>Amazon.Runtime.Internal.Settings</Namespace>
  <Namespace>Amazon.Runtime.Internal.Transform</Namespace>
  <Namespace>Amazon.Runtime.Internal.Util</Namespace>
  <Namespace>Amazon.Runtime.SharedInterfaces</Namespace>
  <Namespace>Amazon.Runtime.SharedInterfaces.Internal</Namespace>
  <Namespace>Amazon.S3</Namespace>
  <Namespace>Amazon.S3.Encryption</Namespace>
  <Namespace>Amazon.S3.Encryption.Internal</Namespace>
  <Namespace>Amazon.S3.Internal</Namespace>
  <Namespace>Amazon.S3.IO</Namespace>
  <Namespace>Amazon.S3.Model</Namespace>
  <Namespace>Amazon.S3.Model.Internal.MarshallTransformations</Namespace>
  <Namespace>Amazon.S3.Transfer</Namespace>
  <Namespace>Amazon.S3.Util</Namespace>
  <Namespace>Amazon.Util</Namespace>
  <Namespace>Amazon.Util.Internal</Namespace>
  <Namespace>Amazon.Util.Internal.PlatformServices</Namespace>
  <Namespace>Newtonsoft.Json</Namespace>
  <Namespace>Newtonsoft.Json.Converters</Namespace>
  <Namespace>Newtonsoft.Json.Linq</Namespace>
  <Namespace>Newtonsoft.Json.Schema</Namespace>
  <Namespace>Newtonsoft.Json.Serialization</Namespace>
  <Namespace>System</Namespace>
  <Namespace>System.Diagnostics</Namespace>
  <Namespace>System.Diagnostics.Tracing</Namespace>
  <Namespace>System.IO.Compression</Namespace>
  <Namespace>ThirdParty.BouncyCastle.Asn1</Namespace>
  <Namespace>ThirdParty.BouncyCastle.Asn1.Utilities</Namespace>
  <Namespace>ThirdParty.BouncyCastle.Math</Namespace>
  <Namespace>ThirdParty.BouncyCastle.OpenSsl</Namespace>
  <Namespace>ThirdParty.BouncyCastle.Utilities.IO.Pem</Namespace>
  <Namespace>ThirdParty.Ionic.Zlib</Namespace>
  <Namespace>ThirdParty.Json.LitJson</Namespace>
  <Namespace>ThirdParty.MD5</Namespace>
  <Namespace>Amazon.Glue</Namespace>
  <Namespace>Amazon.Glue.Model</Namespace>
  <Namespace>System.Threading.Tasks</Namespace>
</Query>

//#define TRACE

void Main()
{
	var accessKeyID = Util.GetPassword("aws.sdt-tlamb.accessid");
	var secretKey = Util.GetPassword("aws.sdt-tlamb.secretid");
	var bucketName = "sdt-glue-hackweek";

	var s3 = new AmazonS3Client(accessKeyID, secretKey, RegionEndpoint.USEast1);

	var s3Objects = bucketName.AWSListBucket(s3).S3Objects
	.Where(so => so.Key.StartsWith("wfu/"));
	s3Objects.Dump("S3 Objects");

	//var crawlerName = "client1_input_icd10dx";

	var glue = new AmazonGlueClient(accessKeyID, secretKey, RegionEndpoint.USEast1);
	var dbs = glue.GetDatabases(new GetDatabasesRequest()).DatabaseList.Select(dl => dl.Name);
	var myDBName = "glue-hackweek-thom";
	if (dbs.Contains(myDBName))
	{
		glue.DeleteDatabase(new DeleteDatabaseRequest { Name = myDBName });
	}
	glue.CreateDatabase(new CreateDatabaseRequest { DatabaseInput = new DatabaseInput { Name = myDBName, Description = "Thom's Glue Hackweek DB" } });
	var crawlers = glue.ListCrawlers(new ListCrawlersRequest()).CrawlerNames;
	var tasks = new List<Task<CreateCrawlerResponse>>();
	crawlers.Where(c => c.StartsWith("wfu_")).ToList()
	.ForEach(c =>
	{
		try
		{
			glue.DeleteCrawler(new DeleteCrawlerRequest { Name = c });
		}
		catch (EntityNotFoundException ex)
		{
			ex.Message.Dump();
		}
	});
	crawlers = glue.ListCrawlers(new ListCrawlersRequest()).CrawlerNames;

	var tags = new[] { new { key = "environment", value = "hackweek" } }.ToDictionary(dl => dl.key, dl => dl.value);
	var newCrawlers = s3Objects.ToList()
	.ConvertAll(o => new CreateCrawlerRequest
	{
		Classifiers = new[] { "client1-input-psv" }.ToList(),
		Configuration = @"{""Version"":1.0,""CrawlerOutput"":{""Partitions"":{""AddOrUpdateBehavior"":""InheritFromTable""}}}",
		DatabaseName = myDBName,
		Name = string.Join("_", o.Key.Split('/').TakeAllButLast()),
		Role = "hackday-glue-role-team2",
		SchemaChangePolicy = new SchemaChangePolicy
		{
			DeleteBehavior = new DeleteBehavior("LOG"),
			UpdateBehavior = new UpdateBehavior("UPDATE_IN_DATABASE")
		},
		TablePrefix = string.Join("_", o.Key.Split('/').TakeAllButLast().TakeAllButLast()) + "_",
		Tags = tags,
		Targets = new CrawlerTargets
		{
			S3Targets = new List<S3Target> {
				new S3Target { Path = $"s3://{o.BucketName}/{string.Join("/", o.Key.Split('/').TakeAllButLast())}" }
				}
		}
	}).OrderBy(o => o.Name).ToList();
	newCrawlers.Dump("New Crawlers");
	newCrawlers.ForEach(c =>
	{
		try
		{
			tasks.Add(glue.CreateCrawlerAsync(c, new CancellationToken()));
		}
		catch (AmazonGlueException ex)
		{
			ex.Dump($"Glue Exception {c.Name}");
		}
	});
	while (tasks.Any(t => !t.IsCompleted))
	{
	}
	var runtasks = new List<Task<StartCrawlerResponse>>();
	newCrawlers.ForEach(c => runtasks.Add(glue.StartCrawlerAsync(new StartCrawlerRequest { Name = c.Name })));
	runtasks.Dump("Run Tasks");
}

// Define other methods and classes here