linqpad/HackWeek AWS/HackWeek AWS Glue List Crawlers.linq

<Query Kind="Program">
  <Reference>&lt;ProgramFilesX64&gt;\Microsoft SDKs\Azure\.NET SDK\v2.9\bin\plugins\Diagnostics\Newtonsoft.Json.dll</Reference>
  <NuGetReference>Amazon.Lambda.S3Events</NuGetReference>
  <NuGetReference>AWSSDK.Core</NuGetReference>
  <NuGetReference>AWSSDK.Glue</NuGetReference>
  <NuGetReference>AWSSDK.S3</NuGetReference>
  <Namespace>Amazon</Namespace>
  <Namespace>Amazon.Auth.AccessControlPolicy</Namespace>
  <Namespace>Amazon.Auth.AccessControlPolicy.ActionIdentifiers</Namespace>
  <Namespace>Amazon.Internal</Namespace>
  <Namespace>Amazon.Lambda.S3Events</Namespace>
  <Namespace>Amazon.MissingTypes</Namespace>
  <Namespace>Amazon.Runtime</Namespace>
  <Namespace>Amazon.Runtime.CredentialManagement</Namespace>
  <Namespace>Amazon.Runtime.CredentialManagement.Internal</Namespace>
  <Namespace>Amazon.Runtime.EventStreams</Namespace>
  <Namespace>Amazon.Runtime.EventStreams.Internal</Namespace>
  <Namespace>Amazon.Runtime.Internal</Namespace>
  <Namespace>Amazon.Runtime.Internal.Auth</Namespace>
  <Namespace>Amazon.Runtime.Internal.Settings</Namespace>
  <Namespace>Amazon.Runtime.Internal.Transform</Namespace>
  <Namespace>Amazon.Runtime.Internal.Util</Namespace>
  <Namespace>Amazon.Runtime.SharedInterfaces</Namespace>
  <Namespace>Amazon.Runtime.SharedInterfaces.Internal</Namespace>
  <Namespace>Amazon.S3</Namespace>
  <Namespace>Amazon.S3.Encryption</Namespace>
  <Namespace>Amazon.S3.Encryption.Internal</Namespace>
  <Namespace>Amazon.S3.Internal</Namespace>
  <Namespace>Amazon.S3.IO</Namespace>
  <Namespace>Amazon.S3.Model</Namespace>
  <Namespace>Amazon.S3.Model.Internal.MarshallTransformations</Namespace>
  <Namespace>Amazon.S3.Transfer</Namespace>
  <Namespace>Amazon.S3.Util</Namespace>
  <Namespace>Amazon.Util</Namespace>
  <Namespace>Amazon.Util.Internal</Namespace>
  <Namespace>Amazon.Util.Internal.PlatformServices</Namespace>
  <Namespace>Newtonsoft.Json</Namespace>
  <Namespace>Newtonsoft.Json.Converters</Namespace>
  <Namespace>Newtonsoft.Json.Linq</Namespace>
  <Namespace>Newtonsoft.Json.Schema</Namespace>
  <Namespace>Newtonsoft.Json.Serialization</Namespace>
  <Namespace>System</Namespace>
  <Namespace>System.Diagnostics</Namespace>
  <Namespace>System.Diagnostics.Tracing</Namespace>
  <Namespace>System.IO.Compression</Namespace>
  <Namespace>ThirdParty.BouncyCastle.Asn1</Namespace>
  <Namespace>ThirdParty.BouncyCastle.Asn1.Utilities</Namespace>
  <Namespace>ThirdParty.BouncyCastle.Math</Namespace>
  <Namespace>ThirdParty.BouncyCastle.OpenSsl</Namespace>
  <Namespace>ThirdParty.BouncyCastle.Utilities.IO.Pem</Namespace>
  <Namespace>ThirdParty.Ionic.Zlib</Namespace>
  <Namespace>ThirdParty.Json.LitJson</Namespace>
  <Namespace>ThirdParty.MD5</Namespace>
  <Namespace>Amazon.Glue</Namespace>
  <Namespace>Amazon.Glue.Model</Namespace>
  <Namespace>System.Threading.Tasks</Namespace>
</Query>

void Main()
{
	var accessKeyID = Util.GetPassword("aws.sdt-tlamb.accessid");
	var secretKey = Util.GetPassword("aws.sdt-tlamb.secretid");
	var bucketName = "sdt-glue-hackweek";
	var crawlerName = "client1_input_icd10dx";

	var client = new AmazonS3Client(accessKeyID, secretKey, RegionEndpoint.USEast1);
	var glueclient = new AmazonGlueClient(accessKeyID, secretKey, RegionEndpoint.USEast1);

	//UploadICDFiles(client, bucketName);
	//ListBucket(client, bucketName).Dump("S3 Buckets");
	ListCrawlers(glueclient).Crawlers.Select(c => new
	{
		c.Name,
		c.DatabaseName,
		c.LastCrawl.Status,
		c.LastCrawl.ErrorMessage,
		c.Description,
		c.Targets.S3Targets
	}).Dump("Crawlers");
	/*
		//RunCrawler(glueclient, crawlerName).Dump();
		glueclient.GetDatabases(new GetDatabasesRequest{}).DatabaseList.Dump("Database");
		glueclient.GetCrawler(new GetCrawlerRequest { Name = crawlerName}).Crawler.Dump("Crawler");
		glueclient.GetTables(new GetTablesRequest{DatabaseName = "glue-hackweek-icd"}).TableList.Dump("glue hackweek icd tables");
	*/
}

// Define other methods and classes here

void UploadICDFiles(AmazonS3Client client, string bucketName)
{
	var regex = new Regex("wfu_icd(10|9)_(dx|px)_(\\d*)", RegexOptions.IgnoreCase | RegexOptions.Singleline);
	var sourcePath = @"S:\Development\Cube Killers\Deidentified DI Import Files\Wake - Deidentified";
	var sourceFiles = "wfu_icd*.psv";
	var files = System.IO.Directory.GetFiles(sourcePath, sourceFiles).OrderBy(f => f);
	files
	//.Where(f => f.Contains("10_dx"))
	//.Take(1)
	.ToList().ForEach(f =>
	{
		var fileName = Path.GetFileNameWithoutExtension(f);
		var subdir = regex.Replace(fileName, "icd/icd$1/$2");
		var key = regex.Replace(fileName, "icd$1$2");
		TransferToBucketAsync(client, bucketName, subdir, key, f).Wait();//.Dump();
	});
}

ListObjectsResponse ListBucket(AmazonS3Client client, string bucketName)
{
	var request = new Amazon.S3.Model.ListObjectsRequest();
	return client.ListObjects(bucketName);
}

ListObjectsResponse ListBucket(string accessKeyID, string secretKey, string bucketName)
{
	using (var client = new AmazonS3Client(accessKeyID, secretKey, RegionEndpoint.USEast1))
	{
		return ListBucket(client, bucketName);
	}
}

static async Task TransferToBucketAsync(AmazonS3Client client, string bucketName, string subdirName, string keyName, string fileName)
{
	new { keyName, fileName }.Dump();
	var utility = new TransferUtility(client);
	var request = new TransferUtilityUploadRequest
	{
		BucketName = $"{bucketName}{(!string.IsNullOrEmpty(subdirName) ? @"/" + subdirName : "")}",
		Key = keyName,
		ContentType = "text/plain"
	};
	request.FilePath = fileName;
	await utility.UploadAsync(request);
}

static async Task<PutObjectResponse> UploadToBucketAsync(AmazonS3Client client, string bucketName, string keyName, string fileName)
{
	new { keyName, fileName }.Dump();
	var request = new Amazon.S3.Model.PutObjectRequest
	{
		BucketName = bucketName,
		Key = keyName,
		ContentBody = File.ReadAllText(fileName),
		ContentType = "text/plain"
	};
	request.FilePath = fileName;
	return await client.PutObjectAsync(request);
}

PutObjectResponse UploadToBucket(AmazonS3Client client, string bucketName, string keyName, string fileName)
{
	new { keyName, fileName }.Dump();
	var result = UploadToBucketAsync(client, bucketName, keyName, fileName);
	result.Wait();
	return result.Result;
}

PutObjectResponse UploadToBucket(string accessKeyID, string secretKey, string bucketName, string keyName, string fileName)
{
	using (var client = new AmazonS3Client(accessKeyID, secretKey, RegionEndpoint.USEast1))
	{
		return UploadToBucket(client, bucketName, keyName, fileName);
	}
}

GetCrawlersResponse ListCrawlers(AmazonGlueClient client)
{
	return client.GetCrawlers(new GetCrawlersRequest());
}

GetCrawlersResponse ListCrawlers(string accessKeyID, string secretKey)
{

	using (var client = new AmazonGlueClient(accessKeyID, secretKey, RegionEndpoint.USEast1))
	{
		var crawlerRequest = new GetCrawlersRequest();
		return client.GetCrawlers(crawlerRequest);
	}
}

StartCrawlerResponse RunCrawler(AmazonGlueClient client, string crawlerName)
{
	var crawler = new StartCrawlerRequest
	{
		Name = crawlerName
	};
	return client.StartCrawler(crawler);
}

StartCrawlerResponse RunCrawler(string accessKeyID, string secretKey, string crawlerName)
{

	using (var client = new AmazonGlueClient(accessKeyID, secretKey, RegionEndpoint.USEast1))
	{
		var crawler = new StartCrawlerRequest
		{
			Name = crawlerName
		};
		return client.StartCrawler(crawler);
	}
}