<ProgramFilesX64>\Microsoft SDKs\Azure\.NET SDK\v2.9\bin\plugins\Diagnostics\Newtonsoft.Json.dll
Amazon.Lambda.S3Events
AWSSDK.Core
AWSSDK.Glue
AWSSDK.S3
Amazon
Amazon.Auth.AccessControlPolicy
Amazon.Auth.AccessControlPolicy.ActionIdentifiers
Amazon.Internal
Amazon.Lambda.S3Events
Amazon.MissingTypes
Amazon.Runtime
Amazon.Runtime.CredentialManagement
Amazon.Runtime.CredentialManagement.Internal
Amazon.Runtime.EventStreams
Amazon.Runtime.EventStreams.Internal
Amazon.Runtime.Internal
Amazon.Runtime.Internal.Auth
Amazon.Runtime.Internal.Settings
Amazon.Runtime.Internal.Transform
Amazon.Runtime.Internal.Util
Amazon.Runtime.SharedInterfaces
Amazon.Runtime.SharedInterfaces.Internal
Amazon.S3
Amazon.S3.Encryption
Amazon.S3.Encryption.Internal
Amazon.S3.Internal
Amazon.S3.IO
Amazon.S3.Model
Amazon.S3.Model.Internal.MarshallTransformations
Amazon.S3.Transfer
Amazon.S3.Util
Amazon.Util
Amazon.Util.Internal
Amazon.Util.Internal.PlatformServices
Newtonsoft.Json
Newtonsoft.Json.Converters
Newtonsoft.Json.Linq
Newtonsoft.Json.Schema
Newtonsoft.Json.Serialization
System
System.Diagnostics
System.Diagnostics.Tracing
System.IO.Compression
ThirdParty.BouncyCastle.Asn1
ThirdParty.BouncyCastle.Asn1.Utilities
ThirdParty.BouncyCastle.Math
ThirdParty.BouncyCastle.OpenSsl
ThirdParty.BouncyCastle.Utilities.IO.Pem
ThirdParty.Ionic.Zlib
ThirdParty.Json.LitJson
ThirdParty.MD5
Amazon.Glue
Amazon.Glue.Model
System.Threading.Tasks
void Main()
{
var accessKeyID = Util.GetPassword("aws.sdt-tlamb.accessid");
var secretKey = Util.GetPassword("aws.sdt-tlamb.secretid");
var bucketName = "sdt-glue-hackweek";
var crawlerName = "client1_input_icd10dx";
var client = new AmazonS3Client(accessKeyID, secretKey, RegionEndpoint.USEast1);
var glueclient = new AmazonGlueClient(accessKeyID, secretKey, RegionEndpoint.USEast1);
//UploadICDFiles(client, bucketName);
//ListBucket(client, bucketName).Dump("S3 Buckets");
ListCrawlers(glueclient).Crawlers.Select(c => new
{
c.Name,
c.DatabaseName,
c.LastCrawl.Status,
c.LastCrawl.ErrorMessage,
c.Description,
c.Targets.S3Targets
}).Dump("Crawlers");
/*
//RunCrawler(glueclient, crawlerName).Dump();
glueclient.GetDatabases(new GetDatabasesRequest{}).DatabaseList.Dump("Database");
glueclient.GetCrawler(new GetCrawlerRequest { Name = crawlerName}).Crawler.Dump("Crawler");
glueclient.GetTables(new GetTablesRequest{DatabaseName = "glue-hackweek-icd"}).TableList.Dump("glue hackweek icd tables");
*/
}
// Define other methods and classes here
void UploadICDFiles(AmazonS3Client client, string bucketName)
{
var regex = new Regex("wfu_icd(10|9)_(dx|px)_(\\d*)", RegexOptions.IgnoreCase | RegexOptions.Singleline);
var sourcePath = @"S:\Development\Cube Killers\Deidentified DI Import Files\Wake - Deidentified";
var sourceFiles = "wfu_icd*.psv";
var files = System.IO.Directory.GetFiles(sourcePath, sourceFiles).OrderBy(f => f);
files
//.Where(f => f.Contains("10_dx"))
//.Take(1)
.ToList().ForEach(f =>
{
var fileName = Path.GetFileNameWithoutExtension(f);
var subdir = regex.Replace(fileName, "icd/icd$1/$2");
var key = regex.Replace(fileName, "icd$1$2");
TransferToBucketAsync(client, bucketName, subdir, key, f).Wait();//.Dump();
});
}
ListObjectsResponse ListBucket(AmazonS3Client client, string bucketName)
{
var request = new Amazon.S3.Model.ListObjectsRequest();
return client.ListObjects(bucketName);
}
ListObjectsResponse ListBucket(string accessKeyID, string secretKey, string bucketName)
{
using (var client = new AmazonS3Client(accessKeyID, secretKey, RegionEndpoint.USEast1))
{
return ListBucket(client, bucketName);
}
}
static async Task TransferToBucketAsync(AmazonS3Client client, string bucketName, string subdirName, string keyName, string fileName)
{
new { keyName, fileName }.Dump();
var utility = new TransferUtility(client);
var request = new TransferUtilityUploadRequest
{
BucketName = $"{bucketName}{(!string.IsNullOrEmpty(subdirName) ? @"/" + subdirName : "")}",
Key = keyName,
ContentType = "text/plain"
};
request.FilePath = fileName;
await utility.UploadAsync(request);
}
static async Task UploadToBucketAsync(AmazonS3Client client, string bucketName, string keyName, string fileName)
{
new { keyName, fileName }.Dump();
var request = new Amazon.S3.Model.PutObjectRequest
{
BucketName = bucketName,
Key = keyName,
ContentBody = File.ReadAllText(fileName),
ContentType = "text/plain"
};
request.FilePath = fileName;
return await client.PutObjectAsync(request);
}
PutObjectResponse UploadToBucket(AmazonS3Client client, string bucketName, string keyName, string fileName)
{
new { keyName, fileName }.Dump();
var result = UploadToBucketAsync(client, bucketName, keyName, fileName);
result.Wait();
return result.Result;
}
PutObjectResponse UploadToBucket(string accessKeyID, string secretKey, string bucketName, string keyName, string fileName)
{
using (var client = new AmazonS3Client(accessKeyID, secretKey, RegionEndpoint.USEast1))
{
return UploadToBucket(client, bucketName, keyName, fileName);
}
}
GetCrawlersResponse ListCrawlers(AmazonGlueClient client)
{
return client.GetCrawlers(new GetCrawlersRequest());
}
GetCrawlersResponse ListCrawlers(string accessKeyID, string secretKey)
{
using (var client = new AmazonGlueClient(accessKeyID, secretKey, RegionEndpoint.USEast1))
{
var crawlerRequest = new GetCrawlersRequest();
return client.GetCrawlers(crawlerRequest);
}
}
StartCrawlerResponse RunCrawler(AmazonGlueClient client, string crawlerName)
{
var crawler = new StartCrawlerRequest
{
Name = crawlerName
};
return client.StartCrawler(crawler);
}
StartCrawlerResponse RunCrawler(string accessKeyID, string secretKey, string crawlerName)
{
using (var client = new AmazonGlueClient(accessKeyID, secretKey, RegionEndpoint.USEast1))
{
var crawler = new StartCrawlerRequest
{
Name = crawlerName
};
return client.StartCrawler(crawler);
}
}