chore: deploy initial linqpad queries
This commit is contained in:
Thom Lamb
@@ -0,0 +1,200 @@
|
||||
<Query Kind="Program">
|
||||
<Reference><ProgramFilesX64>\Microsoft SDKs\Azure\.NET SDK\v2.9\bin\plugins\Diagnostics\Newtonsoft.Json.dll</Reference>
|
||||
<NuGetReference>Amazon.Lambda.S3Events</NuGetReference>
|
||||
<NuGetReference>AWSSDK.Core</NuGetReference>
|
||||
<NuGetReference>AWSSDK.Glue</NuGetReference>
|
||||
<NuGetReference>AWSSDK.S3</NuGetReference>
|
||||
<Namespace>Amazon</Namespace>
|
||||
<Namespace>Amazon.Auth.AccessControlPolicy</Namespace>
|
||||
<Namespace>Amazon.Auth.AccessControlPolicy.ActionIdentifiers</Namespace>
|
||||
<Namespace>Amazon.Internal</Namespace>
|
||||
<Namespace>Amazon.Lambda.S3Events</Namespace>
|
||||
<Namespace>Amazon.MissingTypes</Namespace>
|
||||
<Namespace>Amazon.Runtime</Namespace>
|
||||
<Namespace>Amazon.Runtime.CredentialManagement</Namespace>
|
||||
<Namespace>Amazon.Runtime.CredentialManagement.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.EventStreams</Namespace>
|
||||
<Namespace>Amazon.Runtime.EventStreams.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Auth</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Settings</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Transform</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Util</Namespace>
|
||||
<Namespace>Amazon.Runtime.SharedInterfaces</Namespace>
|
||||
<Namespace>Amazon.Runtime.SharedInterfaces.Internal</Namespace>
|
||||
<Namespace>Amazon.S3</Namespace>
|
||||
<Namespace>Amazon.S3.Encryption</Namespace>
|
||||
<Namespace>Amazon.S3.Encryption.Internal</Namespace>
|
||||
<Namespace>Amazon.S3.Internal</Namespace>
|
||||
<Namespace>Amazon.S3.IO</Namespace>
|
||||
<Namespace>Amazon.S3.Model</Namespace>
|
||||
<Namespace>Amazon.S3.Model.Internal.MarshallTransformations</Namespace>
|
||||
<Namespace>Amazon.S3.Transfer</Namespace>
|
||||
<Namespace>Amazon.S3.Util</Namespace>
|
||||
<Namespace>Amazon.Util</Namespace>
|
||||
<Namespace>Amazon.Util.Internal</Namespace>
|
||||
<Namespace>Amazon.Util.Internal.PlatformServices</Namespace>
|
||||
<Namespace>Newtonsoft.Json</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Converters</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Linq</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Schema</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Serialization</Namespace>
|
||||
<Namespace>System</Namespace>
|
||||
<Namespace>System.Diagnostics</Namespace>
|
||||
<Namespace>System.Diagnostics.Tracing</Namespace>
|
||||
<Namespace>System.IO.Compression</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Asn1</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Asn1.Utilities</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Math</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.OpenSsl</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Utilities.IO.Pem</Namespace>
|
||||
<Namespace>ThirdParty.Ionic.Zlib</Namespace>
|
||||
<Namespace>ThirdParty.Json.LitJson</Namespace>
|
||||
<Namespace>ThirdParty.MD5</Namespace>
|
||||
<Namespace>Amazon.Glue</Namespace>
|
||||
<Namespace>Amazon.Glue.Model</Namespace>
|
||||
<Namespace>System.Threading.Tasks</Namespace>
|
||||
</Query>
|
||||
|
||||
void Main()
|
||||
{
|
||||
var accessKeyID = Util.GetPassword("aws.sdt-tlamb.accessid");
|
||||
var secretKey = Util.GetPassword("aws.sdt-tlamb.secretid");
|
||||
var bucketName = "sdt-glue-hackweek";
|
||||
var crawlerName = "client1_input_icd10dx";
|
||||
|
||||
var client = new AmazonS3Client(accessKeyID, secretKey, RegionEndpoint.USEast1);
|
||||
var glueclient = new AmazonGlueClient(accessKeyID, secretKey, RegionEndpoint.USEast1);
|
||||
|
||||
//UploadICDFiles(client, bucketName);
|
||||
//ListBucket(client, bucketName).Dump("S3 Buckets");
|
||||
ListCrawlers(glueclient).Crawlers.Select(c => new
|
||||
{
|
||||
c.Name,
|
||||
c.DatabaseName,
|
||||
c.LastCrawl.Status,
|
||||
c.LastCrawl.ErrorMessage,
|
||||
c.Description,
|
||||
c.Targets.S3Targets
|
||||
}).Dump("Crawlers");
|
||||
/*
|
||||
//RunCrawler(glueclient, crawlerName).Dump();
|
||||
glueclient.GetDatabases(new GetDatabasesRequest{}).DatabaseList.Dump("Database");
|
||||
glueclient.GetCrawler(new GetCrawlerRequest { Name = crawlerName}).Crawler.Dump("Crawler");
|
||||
glueclient.GetTables(new GetTablesRequest{DatabaseName = "glue-hackweek-icd"}).TableList.Dump("glue hackweek icd tables");
|
||||
*/
|
||||
}
|
||||
|
||||
// Define other methods and classes here
|
||||
|
||||
void UploadICDFiles(AmazonS3Client client, string bucketName)
|
||||
{
|
||||
var regex = new Regex("wfu_icd(10|9)_(dx|px)_(\\d*)", RegexOptions.IgnoreCase | RegexOptions.Singleline);
|
||||
var sourcePath = @"S:\Development\Cube Killers\Deidentified DI Import Files\Wake - Deidentified";
|
||||
var sourceFiles = "wfu_icd*.psv";
|
||||
var files = System.IO.Directory.GetFiles(sourcePath, sourceFiles).OrderBy(f => f);
|
||||
files
|
||||
//.Where(f => f.Contains("10_dx"))
|
||||
//.Take(1)
|
||||
.ToList().ForEach(f =>
|
||||
{
|
||||
var fileName = Path.GetFileNameWithoutExtension(f);
|
||||
var subdir = regex.Replace(fileName, "icd/icd$1/$2");
|
||||
var key = regex.Replace(fileName, "icd$1$2");
|
||||
TransferToBucketAsync(client, bucketName, subdir, key, f).Wait();//.Dump();
|
||||
});
|
||||
}
|
||||
|
||||
ListObjectsResponse ListBucket(AmazonS3Client client, string bucketName)
|
||||
{
|
||||
var request = new Amazon.S3.Model.ListObjectsRequest();
|
||||
return client.ListObjects(bucketName);
|
||||
}
|
||||
|
||||
ListObjectsResponse ListBucket(string accessKeyID, string secretKey, string bucketName)
|
||||
{
|
||||
using (var client = new AmazonS3Client(accessKeyID, secretKey, RegionEndpoint.USEast1))
|
||||
{
|
||||
return ListBucket(client, bucketName);
|
||||
}
|
||||
}
|
||||
|
||||
static async Task TransferToBucketAsync(AmazonS3Client client, string bucketName, string subdirName, string keyName, string fileName)
|
||||
{
|
||||
new { keyName, fileName }.Dump();
|
||||
var utility = new TransferUtility(client);
|
||||
var request = new TransferUtilityUploadRequest
|
||||
{
|
||||
BucketName = $"{bucketName}{(!string.IsNullOrEmpty(subdirName) ? @"/" + subdirName : "")}",
|
||||
Key = keyName,
|
||||
ContentType = "text/plain"
|
||||
};
|
||||
request.FilePath = fileName;
|
||||
await utility.UploadAsync(request);
|
||||
}
|
||||
|
||||
static async Task<PutObjectResponse> UploadToBucketAsync(AmazonS3Client client, string bucketName, string keyName, string fileName)
|
||||
{
|
||||
new { keyName, fileName }.Dump();
|
||||
var request = new Amazon.S3.Model.PutObjectRequest
|
||||
{
|
||||
BucketName = bucketName,
|
||||
Key = keyName,
|
||||
ContentBody = File.ReadAllText(fileName),
|
||||
ContentType = "text/plain"
|
||||
};
|
||||
request.FilePath = fileName;
|
||||
return await client.PutObjectAsync(request);
|
||||
}
|
||||
PutObjectResponse UploadToBucket(AmazonS3Client client, string bucketName, string keyName, string fileName)
|
||||
{
|
||||
new { keyName, fileName }.Dump();
|
||||
var result = UploadToBucketAsync(client, bucketName, keyName, fileName);
|
||||
result.Wait();
|
||||
return result.Result;
|
||||
}
|
||||
|
||||
PutObjectResponse UploadToBucket(string accessKeyID, string secretKey, string bucketName, string keyName, string fileName)
|
||||
{
|
||||
using (var client = new AmazonS3Client(accessKeyID, secretKey, RegionEndpoint.USEast1))
|
||||
{
|
||||
return UploadToBucket(client, bucketName, keyName, fileName);
|
||||
}
|
||||
}
|
||||
|
||||
GetCrawlersResponse ListCrawlers(AmazonGlueClient client)
|
||||
{
|
||||
return client.GetCrawlers(new GetCrawlersRequest());
|
||||
}
|
||||
|
||||
GetCrawlersResponse ListCrawlers(string accessKeyID, string secretKey)
|
||||
{
|
||||
|
||||
using (var client = new AmazonGlueClient(accessKeyID, secretKey, RegionEndpoint.USEast1))
|
||||
{
|
||||
var crawlerRequest = new GetCrawlersRequest();
|
||||
return client.GetCrawlers(crawlerRequest);
|
||||
}
|
||||
}
|
||||
|
||||
StartCrawlerResponse RunCrawler(AmazonGlueClient client, string crawlerName)
|
||||
{
|
||||
var crawler = new StartCrawlerRequest
|
||||
{
|
||||
Name = crawlerName
|
||||
};
|
||||
return client.StartCrawler(crawler);
|
||||
}
|
||||
|
||||
StartCrawlerResponse RunCrawler(string accessKeyID, string secretKey, string crawlerName)
|
||||
{
|
||||
|
||||
using (var client = new AmazonGlueClient(accessKeyID, secretKey, RegionEndpoint.USEast1))
|
||||
{
|
||||
var crawler = new StartCrawlerRequest
|
||||
{
|
||||
Name = crawlerName
|
||||
};
|
||||
return client.StartCrawler(crawler);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
<Query Kind="Program">
|
||||
<Reference><ProgramFilesX64>\Microsoft SDKs\Azure\.NET SDK\v2.9\bin\plugins\Diagnostics\Newtonsoft.Json.dll</Reference>
|
||||
<NuGetReference>Amazon.Lambda.S3Events</NuGetReference>
|
||||
<NuGetReference>AWSSDK.Core</NuGetReference>
|
||||
<NuGetReference>AWSSDK.Glue</NuGetReference>
|
||||
<NuGetReference>AWSSDK.S3</NuGetReference>
|
||||
<Namespace>Amazon</Namespace>
|
||||
<Namespace>Amazon.Auth.AccessControlPolicy</Namespace>
|
||||
<Namespace>Amazon.Auth.AccessControlPolicy.ActionIdentifiers</Namespace>
|
||||
<Namespace>Amazon.Internal</Namespace>
|
||||
<Namespace>Amazon.Lambda.S3Events</Namespace>
|
||||
<Namespace>Amazon.MissingTypes</Namespace>
|
||||
<Namespace>Amazon.Runtime</Namespace>
|
||||
<Namespace>Amazon.Runtime.CredentialManagement</Namespace>
|
||||
<Namespace>Amazon.Runtime.CredentialManagement.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.EventStreams</Namespace>
|
||||
<Namespace>Amazon.Runtime.EventStreams.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Auth</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Settings</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Transform</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Util</Namespace>
|
||||
<Namespace>Amazon.Runtime.SharedInterfaces</Namespace>
|
||||
<Namespace>Amazon.Runtime.SharedInterfaces.Internal</Namespace>
|
||||
<Namespace>Amazon.S3</Namespace>
|
||||
<Namespace>Amazon.S3.Encryption</Namespace>
|
||||
<Namespace>Amazon.S3.Encryption.Internal</Namespace>
|
||||
<Namespace>Amazon.S3.Internal</Namespace>
|
||||
<Namespace>Amazon.S3.IO</Namespace>
|
||||
<Namespace>Amazon.S3.Model</Namespace>
|
||||
<Namespace>Amazon.S3.Model.Internal.MarshallTransformations</Namespace>
|
||||
<Namespace>Amazon.S3.Transfer</Namespace>
|
||||
<Namespace>Amazon.S3.Util</Namespace>
|
||||
<Namespace>Amazon.Util</Namespace>
|
||||
<Namespace>Amazon.Util.Internal</Namespace>
|
||||
<Namespace>Amazon.Util.Internal.PlatformServices</Namespace>
|
||||
<Namespace>Newtonsoft.Json</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Converters</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Linq</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Schema</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Serialization</Namespace>
|
||||
<Namespace>System</Namespace>
|
||||
<Namespace>System.Diagnostics</Namespace>
|
||||
<Namespace>System.Diagnostics.Tracing</Namespace>
|
||||
<Namespace>System.IO.Compression</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Asn1</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Asn1.Utilities</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Math</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.OpenSsl</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Utilities.IO.Pem</Namespace>
|
||||
<Namespace>ThirdParty.Ionic.Zlib</Namespace>
|
||||
<Namespace>ThirdParty.Json.LitJson</Namespace>
|
||||
<Namespace>ThirdParty.MD5</Namespace>
|
||||
<Namespace>Amazon.Glue</Namespace>
|
||||
<Namespace>Amazon.Glue.Model</Namespace>
|
||||
<Namespace>System.Threading.Tasks</Namespace>
|
||||
</Query>
|
||||
|
||||
//#define TRACE
|
||||
|
||||
void Main()
|
||||
{
|
||||
var accessKeyID = Util.GetPassword("aws.sdt-tlamb.accessid");
|
||||
var secretKey = Util.GetPassword("aws.sdt-tlamb.secretid");
|
||||
var bucketName = "sdt-glue-hackweek";
|
||||
|
||||
var s3 = new AmazonS3Client(accessKeyID, secretKey, RegionEndpoint.USEast1);
|
||||
|
||||
var s3Objects = bucketName.AWSListBucket(s3).S3Objects
|
||||
.Where(so => so.Key.StartsWith("wfu/"));
|
||||
s3Objects.Dump("S3 Objects");
|
||||
|
||||
//var crawlerName = "client1_input_icd10dx";
|
||||
|
||||
var glue = new AmazonGlueClient(accessKeyID, secretKey, RegionEndpoint.USEast1);
|
||||
var dbs = glue.GetDatabases(new GetDatabasesRequest()).DatabaseList.Select(dl => dl.Name);
|
||||
var myDBName = "glue-hackweek-thom";
|
||||
if (dbs.Contains(myDBName))
|
||||
{
|
||||
glue.DeleteDatabase(new DeleteDatabaseRequest { Name = myDBName });
|
||||
}
|
||||
glue.CreateDatabase(new CreateDatabaseRequest { DatabaseInput = new DatabaseInput { Name = myDBName, Description = "Thom's Glue Hackweek DB" } });
|
||||
var crawlers = glue.ListCrawlers(new ListCrawlersRequest()).CrawlerNames;
|
||||
var tasks = new List<Task<CreateCrawlerResponse>>();
|
||||
crawlers.Where(c => c.StartsWith("wfu_")).ToList()
|
||||
.ForEach(c =>
|
||||
{
|
||||
try
|
||||
{
|
||||
glue.DeleteCrawler(new DeleteCrawlerRequest { Name = c });
|
||||
}
|
||||
catch (EntityNotFoundException ex)
|
||||
{
|
||||
ex.Message.Dump();
|
||||
}
|
||||
});
|
||||
crawlers = glue.ListCrawlers(new ListCrawlersRequest()).CrawlerNames;
|
||||
|
||||
var tags = new[] { new { key = "environment", value = "hackweek" } }.ToDictionary(dl => dl.key, dl => dl.value);
|
||||
var newCrawlers = s3Objects.ToList()
|
||||
.ConvertAll(o => new CreateCrawlerRequest
|
||||
{
|
||||
Classifiers = new[] { "client1-input-psv" }.ToList(),
|
||||
Configuration = @"{""Version"":1.0,""CrawlerOutput"":{""Partitions"":{""AddOrUpdateBehavior"":""InheritFromTable""}}}",
|
||||
DatabaseName = myDBName,
|
||||
Name = string.Join("_", o.Key.Split('/').TakeAllButLast()),
|
||||
Role = "hackday-glue-role-team2",
|
||||
SchemaChangePolicy = new SchemaChangePolicy
|
||||
{
|
||||
DeleteBehavior = new DeleteBehavior("LOG"),
|
||||
UpdateBehavior = new UpdateBehavior("UPDATE_IN_DATABASE")
|
||||
},
|
||||
TablePrefix = string.Join("_", o.Key.Split('/').TakeAllButLast().TakeAllButLast()) + "_",
|
||||
Tags = tags,
|
||||
Targets = new CrawlerTargets
|
||||
{
|
||||
S3Targets = new List<S3Target> {
|
||||
new S3Target { Path = $"s3://{o.BucketName}/{string.Join("/", o.Key.Split('/').TakeAllButLast())}" }
|
||||
}
|
||||
}
|
||||
}).OrderBy(o => o.Name).ToList();
|
||||
newCrawlers.Dump("New Crawlers");
|
||||
newCrawlers.ForEach(c =>
|
||||
{
|
||||
try
|
||||
{
|
||||
tasks.Add(glue.CreateCrawlerAsync(c, new CancellationToken()));
|
||||
}
|
||||
catch (AmazonGlueException ex)
|
||||
{
|
||||
ex.Dump($"Glue Exception {c.Name}");
|
||||
}
|
||||
});
|
||||
while (tasks.Any(t => !t.IsCompleted))
|
||||
{
|
||||
}
|
||||
var runtasks = new List<Task<StartCrawlerResponse>>();
|
||||
newCrawlers.ForEach(c => runtasks.Add(glue.StartCrawlerAsync(new StartCrawlerRequest { Name = c.Name })));
|
||||
runtasks.Dump("Run Tasks");
|
||||
}
|
||||
|
||||
// Define other methods and classes here
|
||||
@@ -0,0 +1,77 @@
|
||||
<Query Kind="Program">
|
||||
<Reference><ProgramFilesX64>\Microsoft SDKs\Azure\.NET SDK\v2.9\bin\plugins\Diagnostics\Newtonsoft.Json.dll</Reference>
|
||||
<NuGetReference>Amazon.Lambda.S3Events</NuGetReference>
|
||||
<NuGetReference>AWSSDK.Core</NuGetReference>
|
||||
<NuGetReference>AWSSDK.Glue</NuGetReference>
|
||||
<NuGetReference>AWSSDK.S3</NuGetReference>
|
||||
<Namespace>Amazon</Namespace>
|
||||
<Namespace>Amazon.Auth.AccessControlPolicy</Namespace>
|
||||
<Namespace>Amazon.Auth.AccessControlPolicy.ActionIdentifiers</Namespace>
|
||||
<Namespace>Amazon.Internal</Namespace>
|
||||
<Namespace>Amazon.Lambda.S3Events</Namespace>
|
||||
<Namespace>Amazon.MissingTypes</Namespace>
|
||||
<Namespace>Amazon.Runtime</Namespace>
|
||||
<Namespace>Amazon.Runtime.CredentialManagement</Namespace>
|
||||
<Namespace>Amazon.Runtime.CredentialManagement.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.EventStreams</Namespace>
|
||||
<Namespace>Amazon.Runtime.EventStreams.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Auth</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Settings</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Transform</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Util</Namespace>
|
||||
<Namespace>Amazon.Runtime.SharedInterfaces</Namespace>
|
||||
<Namespace>Amazon.Runtime.SharedInterfaces.Internal</Namespace>
|
||||
<Namespace>Amazon.S3</Namespace>
|
||||
<Namespace>Amazon.S3.Encryption</Namespace>
|
||||
<Namespace>Amazon.S3.Encryption.Internal</Namespace>
|
||||
<Namespace>Amazon.S3.Internal</Namespace>
|
||||
<Namespace>Amazon.S3.IO</Namespace>
|
||||
<Namespace>Amazon.S3.Model</Namespace>
|
||||
<Namespace>Amazon.S3.Model.Internal.MarshallTransformations</Namespace>
|
||||
<Namespace>Amazon.S3.Transfer</Namespace>
|
||||
<Namespace>Amazon.S3.Util</Namespace>
|
||||
<Namespace>Amazon.Util</Namespace>
|
||||
<Namespace>Amazon.Util.Internal</Namespace>
|
||||
<Namespace>Amazon.Util.Internal.PlatformServices</Namespace>
|
||||
<Namespace>Newtonsoft.Json</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Converters</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Linq</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Schema</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Serialization</Namespace>
|
||||
<Namespace>System</Namespace>
|
||||
<Namespace>System.Diagnostics</Namespace>
|
||||
<Namespace>System.Diagnostics.Tracing</Namespace>
|
||||
<Namespace>System.IO.Compression</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Asn1</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Asn1.Utilities</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Math</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.OpenSsl</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Utilities.IO.Pem</Namespace>
|
||||
<Namespace>ThirdParty.Ionic.Zlib</Namespace>
|
||||
<Namespace>ThirdParty.Json.LitJson</Namespace>
|
||||
<Namespace>ThirdParty.MD5</Namespace>
|
||||
<Namespace>Amazon.Glue</Namespace>
|
||||
<Namespace>Amazon.Glue.Model</Namespace>
|
||||
<Namespace>System.Threading.Tasks</Namespace>
|
||||
</Query>
|
||||
|
||||
void Main()
|
||||
{
|
||||
var accessKeyID = Util.GetPassword("aws.sdt-tlamb.accessid");
|
||||
var secretKey = Util.GetPassword("aws.sdt-tlamb.secretid");
|
||||
var crawlerName = "client1_input_icd10dx";
|
||||
|
||||
var client = new AmazonGlueClient(accessKeyID, secretKey, RegionEndpoint.USEast1);
|
||||
var regex = new Regex("(icd_?)(10)");
|
||||
|
||||
var crawler = client.GetCrawler(new GetCrawlerRequest { Name = crawlerName }).Crawler;
|
||||
var newCrawler = JsonConvert.DeserializeObject<CreateCrawlerRequest>(regex.Replace(JsonConvert.SerializeObject(crawler), "${1}9"));
|
||||
//newCrawler.LastCrawl = null;
|
||||
crawler.Dump("Crawler");
|
||||
newCrawler.Dump("New Crawler");
|
||||
var result = client.CreateCrawler(newCrawler);
|
||||
result.Dump();
|
||||
}
|
||||
|
||||
// Define other methods and classes here
|
||||
@@ -0,0 +1,70 @@
|
||||
<Query Kind="Program">
|
||||
<Reference><ProgramFilesX64>\Microsoft SDKs\Azure\.NET SDK\v2.9\bin\plugins\Diagnostics\Newtonsoft.Json.dll</Reference>
|
||||
<NuGetReference>Amazon.Lambda.S3Events</NuGetReference>
|
||||
<NuGetReference>AWSSDK.Core</NuGetReference>
|
||||
<NuGetReference>AWSSDK.Glue</NuGetReference>
|
||||
<NuGetReference>AWSSDK.S3</NuGetReference>
|
||||
<Namespace>Amazon</Namespace>
|
||||
<Namespace>Amazon.Auth.AccessControlPolicy</Namespace>
|
||||
<Namespace>Amazon.Auth.AccessControlPolicy.ActionIdentifiers</Namespace>
|
||||
<Namespace>Amazon.Internal</Namespace>
|
||||
<Namespace>Amazon.Lambda.S3Events</Namespace>
|
||||
<Namespace>Amazon.MissingTypes</Namespace>
|
||||
<Namespace>Amazon.Runtime</Namespace>
|
||||
<Namespace>Amazon.Runtime.CredentialManagement</Namespace>
|
||||
<Namespace>Amazon.Runtime.CredentialManagement.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.EventStreams</Namespace>
|
||||
<Namespace>Amazon.Runtime.EventStreams.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Auth</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Settings</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Transform</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Util</Namespace>
|
||||
<Namespace>Amazon.Runtime.SharedInterfaces</Namespace>
|
||||
<Namespace>Amazon.Runtime.SharedInterfaces.Internal</Namespace>
|
||||
<Namespace>Amazon.S3</Namespace>
|
||||
<Namespace>Amazon.S3.Encryption</Namespace>
|
||||
<Namespace>Amazon.S3.Encryption.Internal</Namespace>
|
||||
<Namespace>Amazon.S3.Internal</Namespace>
|
||||
<Namespace>Amazon.S3.IO</Namespace>
|
||||
<Namespace>Amazon.S3.Model</Namespace>
|
||||
<Namespace>Amazon.S3.Model.Internal.MarshallTransformations</Namespace>
|
||||
<Namespace>Amazon.S3.Transfer</Namespace>
|
||||
<Namespace>Amazon.S3.Util</Namespace>
|
||||
<Namespace>Amazon.Util</Namespace>
|
||||
<Namespace>Amazon.Util.Internal</Namespace>
|
||||
<Namespace>Amazon.Util.Internal.PlatformServices</Namespace>
|
||||
<Namespace>Newtonsoft.Json</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Converters</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Linq</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Schema</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Serialization</Namespace>
|
||||
<Namespace>System</Namespace>
|
||||
<Namespace>System.Diagnostics</Namespace>
|
||||
<Namespace>System.Diagnostics.Tracing</Namespace>
|
||||
<Namespace>System.IO.Compression</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Asn1</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Asn1.Utilities</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Math</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.OpenSsl</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Utilities.IO.Pem</Namespace>
|
||||
<Namespace>ThirdParty.Ionic.Zlib</Namespace>
|
||||
<Namespace>ThirdParty.Json.LitJson</Namespace>
|
||||
<Namespace>ThirdParty.MD5</Namespace>
|
||||
<Namespace>Amazon.Glue</Namespace>
|
||||
<Namespace>Amazon.Glue.Model</Namespace>
|
||||
<Namespace>System.Threading.Tasks</Namespace>
|
||||
</Query>
|
||||
|
||||
void Main()
|
||||
{
|
||||
var accessKeyID = Util.GetPassword("aws.sdt-tlamb.accessid");
|
||||
var secretKey = Util.GetPassword("aws.sdt-tlamb.secretid");
|
||||
var bucketName = "sdt-glue-hackweek";
|
||||
|
||||
var client = new AmazonS3Client(accessKeyID, secretKey, RegionEndpoint.USEast1);
|
||||
|
||||
bucketName.AWSListBucket(client).S3Objects.Dump("S3 Objects");
|
||||
}
|
||||
|
||||
// Define other methods and classes here
|
||||
@@ -0,0 +1,201 @@
|
||||
<Query Kind="Program">
|
||||
<Reference><ProgramFilesX64>\Microsoft SDKs\Azure\.NET SDK\v2.9\bin\plugins\Diagnostics\Newtonsoft.Json.dll</Reference>
|
||||
<NuGetReference>Amazon.Lambda.S3Events</NuGetReference>
|
||||
<NuGetReference>AWSSDK.Core</NuGetReference>
|
||||
<NuGetReference>AWSSDK.Glue</NuGetReference>
|
||||
<NuGetReference>AWSSDK.S3</NuGetReference>
|
||||
<Namespace>Amazon</Namespace>
|
||||
<Namespace>Amazon.Auth.AccessControlPolicy</Namespace>
|
||||
<Namespace>Amazon.Auth.AccessControlPolicy.ActionIdentifiers</Namespace>
|
||||
<Namespace>Amazon.Internal</Namespace>
|
||||
<Namespace>Amazon.Lambda.S3Events</Namespace>
|
||||
<Namespace>Amazon.MissingTypes</Namespace>
|
||||
<Namespace>Amazon.Runtime</Namespace>
|
||||
<Namespace>Amazon.Runtime.CredentialManagement</Namespace>
|
||||
<Namespace>Amazon.Runtime.CredentialManagement.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.EventStreams</Namespace>
|
||||
<Namespace>Amazon.Runtime.EventStreams.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Auth</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Settings</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Transform</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Util</Namespace>
|
||||
<Namespace>Amazon.Runtime.SharedInterfaces</Namespace>
|
||||
<Namespace>Amazon.Runtime.SharedInterfaces.Internal</Namespace>
|
||||
<Namespace>Amazon.S3</Namespace>
|
||||
<Namespace>Amazon.S3.Encryption</Namespace>
|
||||
<Namespace>Amazon.S3.Encryption.Internal</Namespace>
|
||||
<Namespace>Amazon.S3.Internal</Namespace>
|
||||
<Namespace>Amazon.S3.IO</Namespace>
|
||||
<Namespace>Amazon.S3.Model</Namespace>
|
||||
<Namespace>Amazon.S3.Model.Internal.MarshallTransformations</Namespace>
|
||||
<Namespace>Amazon.S3.Transfer</Namespace>
|
||||
<Namespace>Amazon.S3.Util</Namespace>
|
||||
<Namespace>Amazon.Util</Namespace>
|
||||
<Namespace>Amazon.Util.Internal</Namespace>
|
||||
<Namespace>Amazon.Util.Internal.PlatformServices</Namespace>
|
||||
<Namespace>Newtonsoft.Json</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Converters</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Linq</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Schema</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Serialization</Namespace>
|
||||
<Namespace>System</Namespace>
|
||||
<Namespace>System.Diagnostics</Namespace>
|
||||
<Namespace>System.Diagnostics.Tracing</Namespace>
|
||||
<Namespace>System.IO.Compression</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Asn1</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Asn1.Utilities</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Math</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.OpenSsl</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Utilities.IO.Pem</Namespace>
|
||||
<Namespace>ThirdParty.Ionic.Zlib</Namespace>
|
||||
<Namespace>ThirdParty.Json.LitJson</Namespace>
|
||||
<Namespace>ThirdParty.MD5</Namespace>
|
||||
<Namespace>Amazon.Glue</Namespace>
|
||||
<Namespace>Amazon.Glue.Model</Namespace>
|
||||
<Namespace>System.Threading.Tasks</Namespace>
|
||||
</Query>
|
||||
|
||||
void Main()
|
||||
{
|
||||
var accessKeyID = Util.GetPassword("aws.sdt-tlamb.accessid");
|
||||
var secretKey = Util.GetPassword("aws.sdt-tlamb.secretid");
|
||||
var bucketName = "sdt-glue-hackweek";
|
||||
var crawlerName = "client1_input_icd10dx";
|
||||
|
||||
var client = new AmazonS3Client(accessKeyID, secretKey, RegionEndpoint.USEast1);
|
||||
var glueclient = new AmazonGlueClient(accessKeyID, secretKey, RegionEndpoint.USEast1);
|
||||
|
||||
//UploadICDFiles(client, bucketName);
|
||||
//ListBucket(client, bucketName).Dump("S3 Buckets");
|
||||
ListCrawlers(glueclient).Crawlers.Select(c => new
|
||||
{
|
||||
c.Name,
|
||||
c.DatabaseName,
|
||||
c.LastCrawl.Status,
|
||||
c.LastCrawl.ErrorMessage,
|
||||
c.Description,
|
||||
c.Targets.S3Targets
|
||||
}).Dump("Crawlers");
|
||||
/*
|
||||
//RunCrawler(glueclient, crawlerName).Dump();
|
||||
glueclient.GetDatabases(new GetDatabasesRequest{}).DatabaseList.Dump("Database");
|
||||
glueclient.GetCrawler(new GetCrawlerRequest { Name = crawlerName}).Crawler.Dump("Crawler");
|
||||
glueclient.GetTables(new GetTablesRequest{DatabaseName = "glue-hackweek-icd"}).TableList.Dump("glue hackweek icd tables");
|
||||
*/
|
||||
}
|
||||
|
||||
// Define other methods and classes here
|
||||
|
||||
void UploadICDFiles(AmazonS3Client client, string bucketName)
|
||||
{
|
||||
var regex = new Regex("wfu_icd(10|9)_(dx|px)_(\\d*)", RegexOptions.IgnoreCase | RegexOptions.Singleline);
|
||||
var sourcePath = @"S:\Development\Cube Killers\Deidentified DI Import Files\Wake - Deidentified";
|
||||
var sourceFiles = "wfu_icd*.psv";
|
||||
var files = System.IO.Directory.GetFiles(sourcePath, sourceFiles).OrderBy(f => f);
|
||||
files
|
||||
//.Where(f => f.Contains("10_dx"))
|
||||
//.Take(1)
|
||||
.ToList().ForEach(f =>
|
||||
{
|
||||
var fileName = Path.GetFileNameWithoutExtension(f);
|
||||
var subdir = regex.Replace(fileName, "icd/icd$1/$2");
|
||||
var key = regex.Replace(fileName, "icd$1$2");
|
||||
TransferToBucketAsync(client, bucketName, subdir, key, f).Wait();//.Dump();
|
||||
});
|
||||
}
|
||||
|
||||
ListObjectsResponse ListBucket(AmazonS3Client client, string bucketName)
|
||||
{
|
||||
var request = new Amazon.S3.Model.ListObjectsRequest();
|
||||
return client.ListObjects(bucketName);
|
||||
}
|
||||
|
||||
ListObjectsResponse ListBucket(string accessKeyID, string secretKey, string bucketName)
|
||||
{
|
||||
using (var client = new AmazonS3Client(accessKeyID, secretKey, RegionEndpoint.USEast1))
|
||||
{
|
||||
return ListBucket(client, bucketName);
|
||||
}
|
||||
}
|
||||
|
||||
static async Task TransferToBucketAsync(AmazonS3Client client, string bucketName, string subdirName, string keyName, string fileName)
|
||||
{
|
||||
new { keyName, fileName }.Dump();
|
||||
var utility = new TransferUtility(client);
|
||||
var request = new TransferUtilityUploadRequest
|
||||
{
|
||||
BucketName = $"{bucketName}{(!string.IsNullOrEmpty(subdirName) ? @"/" + subdirName : "")}",
|
||||
Key = keyName,
|
||||
ContentType = "text/plain"
|
||||
};
|
||||
request.FilePath = fileName;
|
||||
await utility.UploadAsync(request);
|
||||
}
|
||||
|
||||
static async Task<PutObjectResponse> UploadToBucketAsync(AmazonS3Client client, string bucketName, string keyName, string fileName)
|
||||
{
|
||||
new { keyName, fileName }.Dump();
|
||||
var request = new Amazon.S3.Model.PutObjectRequest
|
||||
{
|
||||
BucketName = bucketName,
|
||||
Key = keyName,
|
||||
ContentBody = File.ReadAllText(fileName),
|
||||
ContentType = "text/plain"
|
||||
};
|
||||
request.FilePath = fileName;
|
||||
return await client.PutObjectAsync(request);
|
||||
}
|
||||
|
||||
PutObjectResponse UploadToBucket(AmazonS3Client client, string bucketName, string keyName, string fileName)
|
||||
{
|
||||
new { keyName, fileName }.Dump();
|
||||
var result = UploadToBucketAsync(client, bucketName, keyName, fileName);
|
||||
result.Wait();
|
||||
return result.Result;
|
||||
}
|
||||
|
||||
PutObjectResponse UploadToBucket(string accessKeyID, string secretKey, string bucketName, string keyName, string fileName)
|
||||
{
|
||||
using (var client = new AmazonS3Client(accessKeyID, secretKey, RegionEndpoint.USEast1))
|
||||
{
|
||||
return UploadToBucket(client, bucketName, keyName, fileName);
|
||||
}
|
||||
}
|
||||
|
||||
GetCrawlersResponse ListCrawlers(AmazonGlueClient client)
|
||||
{
|
||||
return client.GetCrawlers(new GetCrawlersRequest());
|
||||
}
|
||||
|
||||
GetCrawlersResponse ListCrawlers(string accessKeyID, string secretKey)
|
||||
{
|
||||
|
||||
using (var client = new AmazonGlueClient(accessKeyID, secretKey, RegionEndpoint.USEast1))
|
||||
{
|
||||
var crawlerRequest = new GetCrawlersRequest();
|
||||
return client.GetCrawlers(crawlerRequest);
|
||||
}
|
||||
}
|
||||
|
||||
StartCrawlerResponse RunCrawler(AmazonGlueClient client, string crawlerName)
|
||||
{
|
||||
var crawler = new StartCrawlerRequest
|
||||
{
|
||||
Name = crawlerName
|
||||
};
|
||||
return client.StartCrawler(crawler);
|
||||
}
|
||||
|
||||
StartCrawlerResponse RunCrawler(string accessKeyID, string secretKey, string crawlerName)
|
||||
{
|
||||
|
||||
using (var client = new AmazonGlueClient(accessKeyID, secretKey, RegionEndpoint.USEast1))
|
||||
{
|
||||
var crawler = new StartCrawlerRequest
|
||||
{
|
||||
Name = crawlerName
|
||||
};
|
||||
return client.StartCrawler(crawler);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
<Query Kind="Program">
|
||||
<Reference><ProgramFilesX64>\Microsoft SDKs\Azure\.NET SDK\v2.9\bin\plugins\Diagnostics\Newtonsoft.Json.dll</Reference>
|
||||
<NuGetReference>Amazon.Lambda.S3Events</NuGetReference>
|
||||
<NuGetReference>AWSSDK.Core</NuGetReference>
|
||||
<NuGetReference>AWSSDK.Glue</NuGetReference>
|
||||
<NuGetReference>AWSSDK.S3</NuGetReference>
|
||||
<Namespace>Amazon</Namespace>
|
||||
<Namespace>Amazon.Auth.AccessControlPolicy</Namespace>
|
||||
<Namespace>Amazon.Auth.AccessControlPolicy.ActionIdentifiers</Namespace>
|
||||
<Namespace>Amazon.Internal</Namespace>
|
||||
<Namespace>Amazon.Lambda.S3Events</Namespace>
|
||||
<Namespace>Amazon.MissingTypes</Namespace>
|
||||
<Namespace>Amazon.Runtime</Namespace>
|
||||
<Namespace>Amazon.Runtime.CredentialManagement</Namespace>
|
||||
<Namespace>Amazon.Runtime.CredentialManagement.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.EventStreams</Namespace>
|
||||
<Namespace>Amazon.Runtime.EventStreams.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Auth</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Settings</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Transform</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Util</Namespace>
|
||||
<Namespace>Amazon.Runtime.SharedInterfaces</Namespace>
|
||||
<Namespace>Amazon.Runtime.SharedInterfaces.Internal</Namespace>
|
||||
<Namespace>Amazon.S3</Namespace>
|
||||
<Namespace>Amazon.S3.Encryption</Namespace>
|
||||
<Namespace>Amazon.S3.Encryption.Internal</Namespace>
|
||||
<Namespace>Amazon.S3.Internal</Namespace>
|
||||
<Namespace>Amazon.S3.IO</Namespace>
|
||||
<Namespace>Amazon.S3.Model</Namespace>
|
||||
<Namespace>Amazon.S3.Model.Internal.MarshallTransformations</Namespace>
|
||||
<Namespace>Amazon.S3.Transfer</Namespace>
|
||||
<Namespace>Amazon.S3.Util</Namespace>
|
||||
<Namespace>Amazon.Util</Namespace>
|
||||
<Namespace>Amazon.Util.Internal</Namespace>
|
||||
<Namespace>Amazon.Util.Internal.PlatformServices</Namespace>
|
||||
<Namespace>Newtonsoft.Json</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Converters</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Linq</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Schema</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Serialization</Namespace>
|
||||
<Namespace>System</Namespace>
|
||||
<Namespace>System.Diagnostics</Namespace>
|
||||
<Namespace>System.Diagnostics.Tracing</Namespace>
|
||||
<Namespace>System.IO.Compression</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Asn1</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Asn1.Utilities</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Math</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.OpenSsl</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Utilities.IO.Pem</Namespace>
|
||||
<Namespace>ThirdParty.Ionic.Zlib</Namespace>
|
||||
<Namespace>ThirdParty.Json.LitJson</Namespace>
|
||||
<Namespace>ThirdParty.MD5</Namespace>
|
||||
<Namespace>Amazon.Glue</Namespace>
|
||||
<Namespace>Amazon.Glue.Model</Namespace>
|
||||
<Namespace>System.Threading.Tasks</Namespace>
|
||||
</Query>
|
||||
|
||||
void Main()
|
||||
{
|
||||
var accessKeyID = Util.GetPassword("aws.sdt-tlamb.accessid");
|
||||
var secretKey = Util.GetPassword("aws.sdt-tlamb.secretid");
|
||||
var crawlerNames = new[] { "client1_input_icd10dx", "client1_input_icd10px"};
|
||||
|
||||
var client = new AmazonGlueClient(accessKeyID, secretKey, RegionEndpoint.USEast1);
|
||||
|
||||
crawlerNames.ToList().ForEach(crawlerName => crawlerName.RunCrawler(client).Dump($"{crawlerName}"));
|
||||
}
|
||||
|
||||
// Define other methods and classes here
|
||||
@@ -0,0 +1,87 @@
|
||||
<Query Kind="Program">
|
||||
<Reference><ProgramFilesX64>\Microsoft SDKs\Azure\.NET SDK\v2.9\bin\plugins\Diagnostics\Newtonsoft.Json.dll</Reference>
|
||||
<NuGetReference>Amazon.Lambda.S3Events</NuGetReference>
|
||||
<NuGetReference>AWSSDK.Core</NuGetReference>
|
||||
<NuGetReference>AWSSDK.S3</NuGetReference>
|
||||
<Namespace>Amazon</Namespace>
|
||||
<Namespace>Amazon.Auth.AccessControlPolicy</Namespace>
|
||||
<Namespace>Amazon.Auth.AccessControlPolicy.ActionIdentifiers</Namespace>
|
||||
<Namespace>Amazon.Internal</Namespace>
|
||||
<Namespace>Amazon.Lambda.S3Events</Namespace>
|
||||
<Namespace>Amazon.MissingTypes</Namespace>
|
||||
<Namespace>Amazon.Runtime</Namespace>
|
||||
<Namespace>Amazon.Runtime.CredentialManagement</Namespace>
|
||||
<Namespace>Amazon.Runtime.CredentialManagement.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.EventStreams</Namespace>
|
||||
<Namespace>Amazon.Runtime.EventStreams.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Auth</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Settings</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Transform</Namespace>
|
||||
<Namespace>Amazon.Runtime.Internal.Util</Namespace>
|
||||
<Namespace>Amazon.Runtime.SharedInterfaces</Namespace>
|
||||
<Namespace>Amazon.Runtime.SharedInterfaces.Internal</Namespace>
|
||||
<Namespace>Amazon.S3</Namespace>
|
||||
<Namespace>Amazon.S3.Encryption</Namespace>
|
||||
<Namespace>Amazon.S3.Encryption.Internal</Namespace>
|
||||
<Namespace>Amazon.S3.Internal</Namespace>
|
||||
<Namespace>Amazon.S3.IO</Namespace>
|
||||
<Namespace>Amazon.S3.Model</Namespace>
|
||||
<Namespace>Amazon.S3.Model.Internal.MarshallTransformations</Namespace>
|
||||
<Namespace>Amazon.S3.Transfer</Namespace>
|
||||
<Namespace>Amazon.S3.Util</Namespace>
|
||||
<Namespace>Amazon.Util</Namespace>
|
||||
<Namespace>Amazon.Util.Internal</Namespace>
|
||||
<Namespace>Amazon.Util.Internal.PlatformServices</Namespace>
|
||||
<Namespace>Newtonsoft.Json</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Converters</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Linq</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Schema</Namespace>
|
||||
<Namespace>Newtonsoft.Json.Serialization</Namespace>
|
||||
<Namespace>System</Namespace>
|
||||
<Namespace>System.Diagnostics</Namespace>
|
||||
<Namespace>System.Diagnostics.Tracing</Namespace>
|
||||
<Namespace>System.IO.Compression</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Asn1</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Asn1.Utilities</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Math</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.OpenSsl</Namespace>
|
||||
<Namespace>ThirdParty.BouncyCastle.Utilities.IO.Pem</Namespace>
|
||||
<Namespace>ThirdParty.Ionic.Zlib</Namespace>
|
||||
<Namespace>ThirdParty.Json.LitJson</Namespace>
|
||||
<Namespace>ThirdParty.MD5</Namespace>
|
||||
</Query>
|
||||
|
||||
void Main()
|
||||
{
|
||||
var sourcePath = @"S:\Development\Cube Killers\Deidentified DI Import Files\Wake - Deidentified";
|
||||
var files = Directory.GetFiles(sourcePath, "wfu*.*");
|
||||
var uploads = files.Select(f =>
|
||||
{
|
||||
var s3Name = Regex.Replace(Path.GetFileNameWithoutExtension(f).Replace("_", "/"), @"^(.*)/[\d]{8}$", "${1}");
|
||||
var subdir = string.Join("/", s3Name.Split('/').TakeAllButLast());
|
||||
var keyName = s3Name.Split('/').Last();
|
||||
return new
|
||||
{
|
||||
fullName = f,
|
||||
fileName = Path.GetFileNameWithoutExtension(f),
|
||||
s3Name,
|
||||
subdir,
|
||||
keyName
|
||||
};
|
||||
});
|
||||
|
||||
uploads.Dump("Uploads");
|
||||
var accessKeyID = Util.GetPassword("aws.sdt-tlamb.accessid");
|
||||
var secretKey = Util.GetPassword("aws.sdt-tlamb.secretid");
|
||||
var bucketName = "sdt-glue-hackweek";
|
||||
|
||||
var client = new AmazonS3Client(accessKeyID, secretKey, RegionEndpoint.USEast1);
|
||||
var oldFiles = bucketName.AWSListBucket(client).S3Objects.Where(f => f.Key.StartsWith("wfu/")).ToList();
|
||||
oldFiles.ForEach(f => client.DeleteObject(bucketName, f.Key));
|
||||
bucketName.AWSListBucket(client).S3Objects.Dump("Before");
|
||||
uploads.ToList().ForEach(u => u.fullName.AWSUploadAsync(client, bucketName, u.keyName, u.s3Name));
|
||||
bucketName.AWSListBucket(client).S3Objects.Dump("After");
|
||||
}
|
||||
|
||||
// Define other methods and classes here
|
||||
Reference in New Issue
Block a user