<ProgramFilesX64>\Microsoft SDKs\Azure\.NET SDK\v2.9\bin\plugins\Diagnostics\Newtonsoft.Json.dll Amazon.Lambda.S3Events AWSSDK.Core AWSSDK.Glue AWSSDK.S3 Amazon Amazon.Auth.AccessControlPolicy Amazon.Auth.AccessControlPolicy.ActionIdentifiers Amazon.Internal Amazon.Lambda.S3Events Amazon.MissingTypes Amazon.Runtime Amazon.Runtime.CredentialManagement Amazon.Runtime.CredentialManagement.Internal Amazon.Runtime.EventStreams Amazon.Runtime.EventStreams.Internal Amazon.Runtime.Internal Amazon.Runtime.Internal.Auth Amazon.Runtime.Internal.Settings Amazon.Runtime.Internal.Transform Amazon.Runtime.Internal.Util Amazon.Runtime.SharedInterfaces Amazon.Runtime.SharedInterfaces.Internal Amazon.S3 Amazon.S3.Encryption Amazon.S3.Encryption.Internal Amazon.S3.Internal Amazon.S3.IO Amazon.S3.Model Amazon.S3.Model.Internal.MarshallTransformations Amazon.S3.Transfer Amazon.S3.Util Amazon.Util Amazon.Util.Internal Amazon.Util.Internal.PlatformServices Newtonsoft.Json Newtonsoft.Json.Converters Newtonsoft.Json.Linq Newtonsoft.Json.Schema Newtonsoft.Json.Serialization System System.Diagnostics System.Diagnostics.Tracing System.IO.Compression ThirdParty.BouncyCastle.Asn1 ThirdParty.BouncyCastle.Asn1.Utilities ThirdParty.BouncyCastle.Math ThirdParty.BouncyCastle.OpenSsl ThirdParty.BouncyCastle.Utilities.IO.Pem ThirdParty.Ionic.Zlib ThirdParty.Json.LitJson ThirdParty.MD5 Amazon.Glue Amazon.Glue.Model System.Threading.Tasks //#define TRACE void Main() { var accessKeyID = Util.GetPassword("aws.sdt-tlamb.accessid"); var secretKey = Util.GetPassword("aws.sdt-tlamb.secretid"); var bucketName = "sdt-glue-hackweek"; var s3 = new AmazonS3Client(accessKeyID, secretKey, RegionEndpoint.USEast1); var s3Objects = bucketName.AWSListBucket(s3).S3Objects .Where(so => so.Key.StartsWith("wfu/")); s3Objects.Dump("S3 Objects"); //var crawlerName = "client1_input_icd10dx"; var glue = new AmazonGlueClient(accessKeyID, secretKey, RegionEndpoint.USEast1); var dbs = glue.GetDatabases(new GetDatabasesRequest()).DatabaseList.Select(dl => dl.Name); var myDBName = "glue-hackweek-thom"; if (dbs.Contains(myDBName)) { glue.DeleteDatabase(new DeleteDatabaseRequest { Name = myDBName }); } glue.CreateDatabase(new CreateDatabaseRequest { DatabaseInput = new DatabaseInput { Name = myDBName, Description = "Thom's Glue Hackweek DB" } }); var crawlers = glue.ListCrawlers(new ListCrawlersRequest()).CrawlerNames; var tasks = new List>(); crawlers.Where(c => c.StartsWith("wfu_")).ToList() .ForEach(c => { try { glue.DeleteCrawler(new DeleteCrawlerRequest { Name = c }); } catch (EntityNotFoundException ex) { ex.Message.Dump(); } }); crawlers = glue.ListCrawlers(new ListCrawlersRequest()).CrawlerNames; var tags = new[] { new { key = "environment", value = "hackweek" } }.ToDictionary(dl => dl.key, dl => dl.value); var newCrawlers = s3Objects.ToList() .ConvertAll(o => new CreateCrawlerRequest { Classifiers = new[] { "client1-input-psv" }.ToList(), Configuration = @"{""Version"":1.0,""CrawlerOutput"":{""Partitions"":{""AddOrUpdateBehavior"":""InheritFromTable""}}}", DatabaseName = myDBName, Name = string.Join("_", o.Key.Split('/').TakeAllButLast()), Role = "hackday-glue-role-team2", SchemaChangePolicy = new SchemaChangePolicy { DeleteBehavior = new DeleteBehavior("LOG"), UpdateBehavior = new UpdateBehavior("UPDATE_IN_DATABASE") }, TablePrefix = string.Join("_", o.Key.Split('/').TakeAllButLast().TakeAllButLast()) + "_", Tags = tags, Targets = new CrawlerTargets { S3Targets = new List { new S3Target { Path = $"s3://{o.BucketName}/{string.Join("/", o.Key.Split('/').TakeAllButLast())}" } } } }).OrderBy(o => o.Name).ToList(); newCrawlers.Dump("New Crawlers"); newCrawlers.ForEach(c => { try { tasks.Add(glue.CreateCrawlerAsync(c, new CancellationToken())); } catch (AmazonGlueException ex) { ex.Dump($"Glue Exception {c.Name}"); } }); while (tasks.Any(t => !t.IsCompleted)) { } var runtasks = new List>(); newCrawlers.ForEach(c => runtasks.Add(glue.StartCrawlerAsync(new StartCrawlerRequest { Name = c.Name }))); runtasks.Dump("Run Tasks"); } // Define other methods and classes here