feat: initial commit
This commit is contained in:
Thom Lamb
@@ -0,0 +1,63 @@
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FluentAssertions;
|
||||
using NUnit.Framework;
|
||||
using Strata.Analytics.Biz.Extracts.Models;
|
||||
using Strata.Analytics.Biz.Extracts.Services;
|
||||
using Strata.Analytics.Test.Unit.Biz.TestData;
|
||||
using Strata.CoreLib.Claims.Extensions;
|
||||
using Strata.Testing.Categories;
|
||||
|
||||
namespace Strata.Analytics.Test.Unit.Biz.Extracts
|
||||
{
|
||||
[NUnit.Framework.Category(TestCategory.Unit)]
|
||||
[Parallelizable(ParallelScope.Self)]
|
||||
public class ExtractCreateTest
|
||||
{
|
||||
|
||||
[Test]
|
||||
public async Task Extract_CurrentUserAssignedAsEditor_GivenUserIsNotStrata()
|
||||
{
|
||||
// arrange
|
||||
var claimsPrincipal = IClaimsPrincipalAccessorMock.GetClaimsPrincipalAccessor();
|
||||
var dbContext = AnalyticsDbContextMock.GetAnalyticsDbContext(claimsPrincipal);
|
||||
var extractService = new ExtractService(dbContext, claimsPrincipal, AutoMapperMock.GetAutoMapper());
|
||||
|
||||
// act
|
||||
var extract = await extractService.CreateAsync(new ExtractCreateInfo()
|
||||
{
|
||||
Name = "Extract1",
|
||||
Content = "{\"dataSourceId\":1,\"rows\":[],\"columnGroups\":[{\"groupId\":\"ABCD1\",\"values\":[],\"dimensions\":[],\"sortOrder\":0}],\"filters\":[],\"queryConfig\":{\"rows\":[],\"values\":[],\"filterGroups\":[],\"withTotals\":false,\"rowLimit\":25}}"
|
||||
}, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
extract.CreatedByUserGuid.Should().Be(claimsPrincipal.GetUserGuid());
|
||||
extract.CurrentUserRole.Should().Be(ExtractRole.Editor);
|
||||
dbContext.ExtractRoleAssignments.Should().HaveCount(1);
|
||||
dbContext.ExtractRoleAssignments.FirstOrDefault().IdentityGuid.Should().Be(claimsPrincipal.GetUserGuid());
|
||||
dbContext.ExtractRoleAssignments.FirstOrDefault().Role.Should().Be(ExtractRole.Editor);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task Extract_CurrentUserAssignedAsEditorButNoRoleAssignmentsCreated_GivenUserIsStrata()
|
||||
{
|
||||
// arrange
|
||||
var claimsPrincipal = IClaimsPrincipalAccessorMock.GetClaimsPrincipalAccessor(5, true);
|
||||
var dbContext = AnalyticsDbContextMock.GetAnalyticsDbContext(claimsPrincipal);
|
||||
var extractService = new ExtractService(dbContext, claimsPrincipal, AutoMapperMock.GetAutoMapper());
|
||||
|
||||
// act
|
||||
var extract = await extractService.CreateAsync(new ExtractCreateInfo()
|
||||
{
|
||||
Name = "Extract1",
|
||||
Content = "{\"dataSourceId\":1,\"rows\":[],\"columnGroups\":[{\"groupId\":\"ABCD1\",\"values\":[],\"dimensions\":[],\"sortOrder\":0}],\"filters\":[],\"queryConfig\":{\"rows\":[],\"values\":[],\"filterGroups\":[],\"withTotals\":false,\"rowLimit\":25}}"
|
||||
}, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
extract.CreatedByUserGuid.Should().Be(claimsPrincipal.GetUserGuid());
|
||||
extract.CurrentUserRole.Should().Be(ExtractRole.Editor);
|
||||
dbContext.ExtractRoleAssignments.Should().HaveCount(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutoMapper;
|
||||
using NUnit.Framework;
|
||||
using Strata.Analytics.Biz.DbContexts;
|
||||
using Strata.Analytics.Biz.Extracts.Models;
|
||||
using Strata.Analytics.Biz.Extracts.Services;
|
||||
using Strata.Analytics.Test.Unit.Biz.TestData;
|
||||
using Strata.CoreLib.Claims;
|
||||
using Strata.CoreLib.Claims.Extensions;
|
||||
using Strata.Testing.Categories;
|
||||
|
||||
namespace Strata.Analytics.Test.Unit.Biz.Extracts
|
||||
{
|
||||
[Category(TestCategory.Unit)]
|
||||
[Parallelizable(ParallelScope.Self)]
|
||||
public class ExtractCrudTests
|
||||
{
|
||||
private IClaimsPrincipalAccessor _claimsPrincipalAccessor;
|
||||
private AnalyticsDbContext _dbContext;
|
||||
private IMapper _mapper;
|
||||
private IExtractService _extractService;
|
||||
|
||||
[OneTimeSetUp]
|
||||
public void OneTimeSetup()
|
||||
{
|
||||
_claimsPrincipalAccessor = IClaimsPrincipalAccessorMock.GetClaimsPrincipalAccessor(5, true);
|
||||
_dbContext = AnalyticsDbContextMock.GetAnalyticsDbContext(_claimsPrincipalAccessor);
|
||||
_mapper = AutoMapperMock.GetAutoMapper();
|
||||
_extractService = new ExtractService(_dbContext, _claimsPrincipalAccessor, _mapper);
|
||||
_dbContext.Extracts.AddRange(ExtractTestData.Extracts());
|
||||
_dbContext.ExtractExportHistory.AddRange(ExtractTestData.ExtractExportHistories());
|
||||
_dbContext.SaveChanges();
|
||||
}
|
||||
|
||||
[OneTimeTearDown]
|
||||
public async Task OneTimeTearDown()
|
||||
{
|
||||
await _dbContext.Database.EnsureDeletedAsync();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task CanCreate()
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
// arrange
|
||||
var saveInfo = new ExtractCreateInfo()
|
||||
{
|
||||
Name = "My First Unit Test Extract!",
|
||||
Description = "Be Happy For Me!!!",
|
||||
ExtractType = ExtractType.DataSource,
|
||||
Content = "{\"dataSourceId\":1,\"rows\":[],\"columnGroups\":[{\"groupId\":\"ABCD1\",\"values\":[],\"dimensions\":[],\"sortOrder\":0}],\"filters\":[],\"queryConfig\":{\"rows\":[],\"values\":[],\"filterGroups\":[],\"withTotals\":false,\"rowLimit\":25}}"
|
||||
};
|
||||
|
||||
// act
|
||||
var result = await _extractService.CreateAsync(saveInfo, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
Assert.IsNotNull(result);
|
||||
Assert.That(result.ExtractId, Is.GreaterThan(ExtractTestData.Extracts().Max(h => h.ExtractId)));
|
||||
Assert.That(result.ExtractType, Is.EqualTo(ExtractType.DataSource));
|
||||
Assert.That(result.Content, Is.EqualTo(saveInfo.Content));
|
||||
Assert.That(result.CreatedByUserGuid, Is.EqualTo(_claimsPrincipalAccessor.GetCurrentClaimsPrincipal().GetUserGuid()));
|
||||
Assert.That(result.UpdatedByUserGuid, Is.EqualTo(_claimsPrincipalAccessor.GetCurrentClaimsPrincipal().GetUserGuid()));
|
||||
Assert.IsNotNull(result.CreatedDate);
|
||||
Assert.IsNotNull(result.UpdatedDate);
|
||||
}
|
||||
|
||||
|
||||
[Test]
|
||||
public async Task CanGetById()
|
||||
{
|
||||
// act
|
||||
var result = await _extractService.GetByIdAsync(1, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
Assert.IsNotNull(result);
|
||||
}
|
||||
|
||||
|
||||
[Test]
|
||||
public async Task CanUpdate()
|
||||
{
|
||||
// arrange
|
||||
var extractId = _dbContext.Extracts.First().ExtractId;
|
||||
|
||||
var extract = await _extractService.GetByIdAsync(extractId, CancellationToken.None);
|
||||
|
||||
var saveInfo = new ExtractSaveInfo();
|
||||
_mapper.Map(extract, saveInfo);
|
||||
|
||||
saveInfo.Name = "My First RENAMED Extract!";
|
||||
saveInfo.Description = "Fear Me!!!";
|
||||
saveInfo.RawSql = "My New and IMPROVED Query...";
|
||||
_dbContext.ChangeTracker.Clear();
|
||||
|
||||
// act
|
||||
var result = await _extractService.UpdateAsync(extract.ExtractId, saveInfo, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
Assert.IsNotNull(result);
|
||||
Assert.That(result.ExtractId, Is.EqualTo(extract.ExtractId));
|
||||
Assert.That(result.Name, Is.EqualTo(saveInfo.Name));
|
||||
Assert.That(result.Description, Is.EqualTo(saveInfo.Description));
|
||||
Assert.That(result.RawSql, Is.EqualTo(saveInfo.RawSql));
|
||||
Assert.That(result.CreatedByUserGuid, Is.EqualTo(extract.CreatedByUserGuid));
|
||||
Assert.That(result.UpdatedByUserGuid, Is.EqualTo(_claimsPrincipalAccessor.GetCurrentClaimsPrincipal().GetUserGuid()));
|
||||
Assert.That(result.CreatedDate, Is.EqualTo(extract.CreatedDate));
|
||||
Assert.That(result.UpdatedDate, Is.EqualTo(DateTimeOffset.UtcNow).Within(500).Milliseconds); // this could fail in debug
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task CanDelete_LeaveOrphanHistory()
|
||||
{
|
||||
// arrange
|
||||
var extractId = _dbContext.Extracts.Last().ExtractId;
|
||||
_dbContext.ChangeTracker.Clear();
|
||||
|
||||
// act
|
||||
await _extractService.DeleteAsync(extractId, CancellationToken.None);
|
||||
|
||||
var result = await _extractService.GetByIdAsync(extractId, CancellationToken.None);
|
||||
var histories = new ExtractExportHistoryService(_dbContext, _claimsPrincipalAccessor, _mapper).GetByExtractIdAsync(extractId, CancellationToken.None);
|
||||
|
||||
|
||||
// assert
|
||||
Assert.IsNull(result);
|
||||
Assert.IsNotNull(histories);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,336 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Moq;
|
||||
using NUnit.Framework;
|
||||
using Snowflake.Data.Client;
|
||||
using Strata.Analytics.Biz.DataQuery;
|
||||
using Strata.Analytics.Biz.Extracts;
|
||||
using Strata.Analytics.Biz.Extracts.Export;
|
||||
using Strata.Analytics.Biz.Extracts.Jobs;
|
||||
using Strata.Analytics.Biz.Extracts.Models;
|
||||
using Strata.Analytics.Biz.Extracts.Services;
|
||||
using Strata.Analytics.Biz.Hangfire;
|
||||
using Strata.Analytics.Test.Unit.Biz.TestData;
|
||||
using Strata.CoreLib.Claims;
|
||||
using Strata.DataSchema.Models.Query;
|
||||
|
||||
namespace Strata.Analytics.Test.Unit.Biz.Extracts
|
||||
{
|
||||
[TestFixture]
|
||||
internal class ExtractExportJobTest
|
||||
{
|
||||
private Mock<IS3BucketService> _s3BucketService;
|
||||
private Mock<IExtractService> _extractService;
|
||||
private Mock<IExtractExportHistoryService> _historyService;
|
||||
private Mock<IRecurringJobService> _recurringJobService;
|
||||
private Mock<IExtractScheduleService> _scheduleService;
|
||||
private Mock<ISftpService> _sftpService;
|
||||
private Mock<IDataQueryService> _dataQueryService;
|
||||
private Mock<ILogger<ExtractExportJob>> _logger;
|
||||
private IClaimsPrincipalAccessor _claimsPrincipalAccessor;
|
||||
private Mock<IOptions<ExtractConfiguration>> _config;
|
||||
|
||||
|
||||
[Test]
|
||||
public void ExecuteExport_3Files_NoOutbound()
|
||||
{
|
||||
const string SFTP_PATH = "SFTP://directory/folder/subpath";
|
||||
var exportJob = CreateExtractExportJob();
|
||||
var extract = GetRawSqlExtract();
|
||||
var exportRequest = GetExtractExportRequest(extract);
|
||||
var externalStageTokens = new ExternalStageUrlTokens
|
||||
{ BucketName = "Big Bucket", FolderPath = "s3://somewhere/out/there/" };
|
||||
var files = GetFiles(exportRequest.OutputFileName, externalStageTokens, 3);
|
||||
var outboundFiles = files.Select(x => new OutboundFileInfo() { Destination = SFTP_PATH, FileName = x.FilePath, FileSize = x.FileSize, RowCount = x.RowCount }).ToList();
|
||||
|
||||
_extractService.Setup(x => x.GetByIdForExecutionAsync(It.IsAny<int>(), It.IsAny<CancellationToken>())).Returns(Task.FromResult(extract));
|
||||
_dataQueryService.Setup(x => x.BuildSqlQuery(It.IsAny<QueryConfig>(), It.IsAny<CancellationToken>())).Returns(Task.FromResult(new SqlResponse { SqlQuery = extract.RawSql }));
|
||||
_dataQueryService.Setup(x => x.GetExternalStageUrlTokensAsync(It.IsAny<CancellationToken>())).Returns(Task.FromResult(externalStageTokens));
|
||||
_dataQueryService.Setup(x => x.ExecuteSqlQueryAsync(It.IsAny<string>(), It.IsAny<IEnumerable<KeyValuePair<string, object>>>())).Returns(Task.FromResult(new List<dynamic>().AsEnumerable()));
|
||||
_s3BucketService
|
||||
.Setup(x => x.GetFiles(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(),
|
||||
It.IsAny<string>(), It.IsAny<int>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.FromResult(files));
|
||||
_sftpService.Setup(x => x.UploadFilesFromS3(It.IsAny<string>(), files, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>())).Returns(Task.FromResult(outboundFiles));
|
||||
_historyService
|
||||
.Setup(x => x.CreateAsync(It.IsAny<ExtractExportHistorySaveInfo>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.FromResult(1));
|
||||
_historyService
|
||||
.Setup(x => x.UpdateAsync(It.IsAny<int>(), It.IsAny<ExtractExportHistorySaveInfo>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.FromResult(new ExtractExportHistory
|
||||
{ ExtractId = extract.ExtractId, ExtractHistoryId = 1, ExportStartDate = DateTimeOffset.UtcNow, ExportEndDate = DateTimeOffset.UtcNow }));
|
||||
_historyService
|
||||
.Setup(x => x.HasRunningExtractByExtractId(It.IsAny<int>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.FromResult(false));
|
||||
|
||||
//assert
|
||||
Assert.DoesNotThrowAsync(() => exportJob.ExecuteExport(It.IsAny<string>(),It.IsAny<Guid>(),It.IsAny<int>(), It.IsAny<Guid>(), exportRequest, new PerformContextMock().Object, It.IsAny<CancellationToken>()));
|
||||
_extractService.Verify(s => s.GetByIdForExecutionAsync(extract.ExtractId, It.IsAny<CancellationToken>()), Times.Once);
|
||||
_dataQueryService.Verify(s => s.GetExternalStageUrlTokensAsync(It.IsAny<CancellationToken>()), Times.Once);
|
||||
_dataQueryService.Verify(s => s.ExecuteSqlQueryAsync(It.Is<string>(query => query.Contains(extract.RawSql)), It.IsAny<IEnumerable<KeyValuePair<string, object>>>()), Times.Once);
|
||||
_s3BucketService.Verify(s => s.GetFiles(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(),
|
||||
It.IsAny<string>(), It.IsAny<int>(), It.IsAny<CancellationToken>()), Times.Once);
|
||||
_sftpService.Verify(
|
||||
s => s.UploadFilesFromS3(externalStageTokens.BucketName,
|
||||
files, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Once);
|
||||
_historyService.Verify(
|
||||
s => s.HasRunningExtractByExtractId(It.IsAny<int>(), It.IsAny<CancellationToken>()),
|
||||
Times.Once);
|
||||
_historyService.Verify(
|
||||
s => s.CreateAsync(
|
||||
It.Is<ExtractExportHistorySaveInfo>(h => HistorySaveInfosAreEquivalent(h,
|
||||
new ExtractExportHistorySaveInfo
|
||||
{
|
||||
ExtractId = extract.ExtractId,
|
||||
OutputFilename = exportRequest.OutputFileName,
|
||||
SqlExecuted = extract.RawSql,
|
||||
TotalFileSize = files.Sum(f => f.FileSize),
|
||||
TotalRowCount = files.Sum(f => f.RowCount),
|
||||
Destination = SFTP_PATH
|
||||
})),
|
||||
It.IsAny<CancellationToken>()),
|
||||
Times.Once);
|
||||
}
|
||||
|
||||
private static bool HistorySaveInfosAreEquivalent(ExtractExportHistorySaveInfo actual, ExtractExportHistorySaveInfo expected)
|
||||
{
|
||||
Assert.That(actual.ExtractId, Is.EqualTo(expected.ExtractId));
|
||||
Assert.That(actual.OutputFilename, Is.EqualTo(expected.OutputFilename));
|
||||
Assert.That(actual.SqlExecuted.Contains(expected.SqlExecuted));
|
||||
Assert.That(actual.TotalFileSize, Is.EqualTo(expected.TotalFileSize));
|
||||
Assert.That(actual.TotalRowCount, Is.EqualTo(expected.TotalRowCount));
|
||||
Assert.That(actual.Destination, Is.EqualTo(expected.Destination));
|
||||
Assert.That(actual.ExportStartDate, Is.LessThan(actual.ExportEndDate));
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ExecuteExport_FileSizeException()
|
||||
{
|
||||
var exportJob = CreateExtractExportJob();
|
||||
var extract = GetRawSqlExtract();
|
||||
var exportRequest = GetExtractExportRequest(extract);
|
||||
var externalStageTokens = new ExternalStageUrlTokens
|
||||
{ BucketName = "Big Bucket", FolderPath = "s3://somewhere/out/there/" };
|
||||
var files = GetFiles(exportRequest.OutputFileName, externalStageTokens, 3);
|
||||
|
||||
_extractService.Setup(x => x.GetByIdForExecutionAsync(It.IsAny<int>(), It.IsAny<CancellationToken>())).Returns(Task.FromResult(extract));
|
||||
_dataQueryService.Setup(x => x.BuildSqlQuery(It.IsAny<QueryConfig>(), It.IsAny<CancellationToken>())).Returns(Task.FromResult(new SqlResponse { SqlQuery = extract.RawSql }));
|
||||
_dataQueryService.Setup(x => x.GetExternalStageUrlTokensAsync(It.IsAny<CancellationToken>())).Returns(Task.FromResult(externalStageTokens));
|
||||
_dataQueryService.Setup(x => x.ExecuteSqlQueryAsync(It.IsAny<string>(), It.IsAny<IEnumerable<KeyValuePair<string, object>>>())).Throws(new SnowflakeDbException(string.Empty, 0, "Max file size exceeded for unload"));
|
||||
_s3BucketService
|
||||
.Setup(x => x.GetFiles(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(),
|
||||
It.IsAny<string>(), It.IsAny<int>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.FromResult(files));
|
||||
_sftpService.Setup(x => x.UploadFilesFromS3(It.IsAny<string>(), files, "test", It.IsAny<string>(), It.IsAny<CancellationToken>()));
|
||||
_historyService
|
||||
.Setup(x => x.CreateAsync(It.IsAny<ExtractExportHistorySaveInfo>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.FromResult(1));
|
||||
_historyService
|
||||
.Setup(x => x.HasRunningExtractByExtractId(It.IsAny<int>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.FromResult(false));
|
||||
|
||||
Assert.ThrowsAsync<SnowflakeDbException>(() => exportJob.ExecuteExport(It.IsAny<string>(),It.IsAny<Guid>(), It.IsAny<int>(), It.IsAny<Guid>(), exportRequest, new PerformContextMock().Object, It.IsAny<CancellationToken>()));
|
||||
|
||||
_extractService.Verify(s => s.GetByIdForExecutionAsync(extract.ExtractId, It.IsAny<CancellationToken>()), Times.Once);
|
||||
_dataQueryService.Verify(s => s.GetExternalStageUrlTokensAsync(It.IsAny<CancellationToken>()), Times.Once);
|
||||
_dataQueryService.Verify(s => s.ExecuteSqlQueryAsync(It.Is<string>(query => query.Contains(extract.RawSql)), It.IsAny<IEnumerable<KeyValuePair<string, object>>>()), Times.Once);
|
||||
_s3BucketService.Verify(s => s.GetFiles(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(),
|
||||
It.IsAny<string>(), It.IsAny<int>(), It.IsAny<CancellationToken>()), Times.Never);
|
||||
_sftpService.Verify(
|
||||
s => s.UploadFilesFromS3(externalStageTokens.BucketName,
|
||||
files, "test", It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Never);
|
||||
|
||||
_historyService.Verify(
|
||||
s => s.HasRunningExtractByExtractId(It.IsAny<int>(), It.IsAny<CancellationToken>()),
|
||||
Times.Once);
|
||||
// should be created
|
||||
_historyService.Verify(
|
||||
s => s.CreateAsync(It.IsAny<ExtractExportHistorySaveInfo>(), It.IsAny<CancellationToken>()),
|
||||
Times.Once);
|
||||
// should be updated with an error status
|
||||
_historyService.Verify(
|
||||
s => s.UpdateAsync(1, It.Is<ExtractExportHistorySaveInfo>(h => h.Status == ExtractExportStatus.Error), It.IsAny<CancellationToken>()),
|
||||
Times.Once);
|
||||
}
|
||||
[Test]
|
||||
public void ExecuteExport_3Files_WithOutbound()
|
||||
{
|
||||
const string SFTP_PATH = "SFTP://directory/folder/subpath";
|
||||
|
||||
var exportJob = CreateExtractExportJob();
|
||||
var extract = GetRawSqlExtract("SELECT * FROM dbo.ALLDATAEVER");
|
||||
var exportRequest = GetExtractExportRequest(extract);
|
||||
var externalStageTokens = new ExternalStageUrlTokens
|
||||
{ BucketName = "Big Bucket", FolderPath = "s3://somewhere/out/there/" };
|
||||
var files = GetFiles(exportRequest.OutputFileName, externalStageTokens, 3);
|
||||
var outboundFiles = files.Select(x => new OutboundFileInfo() { Destination = SFTP_PATH, FileName = x.FilePath, FileSize = x.FileSize, RowCount = x.RowCount }).ToList();
|
||||
|
||||
_extractService.Setup(x => x.GetByIdForExecutionAsync(It.IsAny<int>(), It.IsAny<CancellationToken>())).Returns(Task.FromResult(extract));
|
||||
_dataQueryService.Setup(x => x.GetExternalStageUrlTokensAsync(It.IsAny<CancellationToken>())).Returns(Task.FromResult(externalStageTokens));
|
||||
_dataQueryService.Setup(x => x.ExecuteSqlQueryAsync(It.IsAny<string>(), It.IsAny<IEnumerable<KeyValuePair<string, object>>>())).Returns(Task.FromResult(new List<dynamic>().AsEnumerable()));
|
||||
_s3BucketService
|
||||
.Setup(x => x.GetFiles(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(),
|
||||
It.IsAny<string>(), It.IsAny<int>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.FromResult(files));
|
||||
_sftpService.Setup(x => x.UploadFilesFromS3(It.IsAny<string>(), files, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>())).Returns(Task.FromResult(outboundFiles));
|
||||
_historyService
|
||||
.Setup(x => x.CreateAsync(It.IsAny<ExtractExportHistorySaveInfo>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.FromResult(1));
|
||||
_historyService
|
||||
.Setup(x => x.UpdateAsync(It.IsAny<int>(), It.IsAny<ExtractExportHistorySaveInfo>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.FromResult(new ExtractExportHistory
|
||||
{ ExtractId = extract.ExtractId, ExtractHistoryId = 1, ExportStartDate = DateTimeOffset.UtcNow, ExportEndDate = DateTimeOffset.UtcNow }));
|
||||
|
||||
//assert
|
||||
Assert.DoesNotThrowAsync(() => exportJob.ExecuteExport(It.IsAny<string>(),It.IsAny<Guid>(), It.IsAny<int>(), It.IsAny<Guid>(), exportRequest, new PerformContextMock().Object, It.IsAny<CancellationToken>()));
|
||||
_extractService.Verify(s => s.GetByIdForExecutionAsync(extract.ExtractId, It.IsAny<CancellationToken>()), Times.Once);
|
||||
_dataQueryService.Verify(s => s.GetExternalStageUrlTokensAsync(It.IsAny<CancellationToken>()), Times.Once);
|
||||
_dataQueryService.Verify(s => s.ExecuteSqlQueryAsync(It.Is<string>(query => query.Contains(extract.RawSql)), It.IsAny<IEnumerable<KeyValuePair<string, object>>>()), Times.Exactly(2));
|
||||
_s3BucketService.Verify(s => s.GetFiles(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(),
|
||||
It.IsAny<string>(), It.IsAny<int>(), It.IsAny<CancellationToken>()), Times.Once);
|
||||
_sftpService.Verify(
|
||||
s => s.UploadFilesFromS3(externalStageTokens.BucketName,
|
||||
files, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Once);
|
||||
_sftpService.Verify(s => s.UploadFilesFromMemory(It.IsAny<MemoryStream>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Once);
|
||||
_historyService.Verify(
|
||||
s => s.CreateAsync(
|
||||
It.Is<ExtractExportHistorySaveInfo>(h => HistorySaveInfosAreEquivalent(h,
|
||||
new ExtractExportHistorySaveInfo
|
||||
{
|
||||
ExtractId = extract.ExtractId,
|
||||
OutputFilename = exportRequest.OutputFileName,
|
||||
SqlExecuted = extract.RawSql,
|
||||
TotalFileSize = files.Sum(f => f.FileSize),
|
||||
TotalRowCount = files.Sum(f => f.RowCount),
|
||||
Destination = SFTP_PATH
|
||||
})),
|
||||
It.IsAny<CancellationToken>()),
|
||||
Times.Once);
|
||||
}
|
||||
[Test]
|
||||
public async Task TestDataToString()
|
||||
{
|
||||
const string SFTP_PATH = "SFTP://directory/folder/subpath";
|
||||
|
||||
var exportJob = CreateExtractExportJob();
|
||||
var files = new List<OutboundFileInfo>
|
||||
{
|
||||
new()
|
||||
{
|
||||
FileName = "/123123131231/121212412/query_2023-04-07-18-56-36.csv_0_0_0.csv.gz",
|
||||
Destination = SFTP_PATH,
|
||||
FileSize = 100,
|
||||
RowCount = 200
|
||||
},
|
||||
new()
|
||||
{
|
||||
FileName = "/123123131231/121212412/query_2024-04-07-18-56-36.csv_0_0_0.csv.gz",
|
||||
Destination = SFTP_PATH,
|
||||
FileSize = 100,
|
||||
RowCount = 200
|
||||
}
|
||||
};
|
||||
var data = new Dictionary<string, List<string>>()
|
||||
{
|
||||
{ "Column 1", new List<string> {"value1","value2","value3"} },
|
||||
{ "Column 2", new List<string> {"value1","value3"} },
|
||||
{ "Column 3", new List<string> {"value1"} },
|
||||
};
|
||||
|
||||
var stringWriter = await exportJob.CollapseDataAndSendFileAsync(files, data, It.IsAny<DateTime>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>());
|
||||
var stringToTest = stringWriter.ToString();
|
||||
var numberOfColumns = stringToTest.Count(x => x == '|');
|
||||
var numberOfTimesDataSeparated = stringToTest.Count(x => x == ',');
|
||||
var numberOfNewLines = Regex.Matches(stringToTest, "\r\n").Count;
|
||||
Assert.That(numberOfColumns, Is.EqualTo(24));
|
||||
Assert.That(numberOfTimesDataSeparated, Is.EqualTo(6));
|
||||
Assert.IsFalse(stringToTest.Contains("/123123131231/121212412/"));
|
||||
Assert.That(numberOfNewLines, Is.EqualTo(3));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ExecuteExport_ConcurrentExtractExportException()
|
||||
{
|
||||
var exportJob = CreateExtractExportJob();
|
||||
var extract = GetRawSqlExtract();
|
||||
var exportRequest = GetExtractExportRequest(extract);
|
||||
_historyService
|
||||
.Setup(x => x.HasRunningExtractByExtractId(It.IsAny<int>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.FromResult(true));
|
||||
|
||||
Assert.ThrowsAsync<ConcurrentExtractExportException>(() => exportJob.ExecuteExport(It.IsAny<string>(),It.IsAny<Guid>(), It.IsAny<int>(), It.IsAny<Guid>(), exportRequest, new PerformContextMock().Object, It.IsAny<CancellationToken>()));
|
||||
|
||||
_historyService.Verify(
|
||||
s => s.HasRunningExtractByExtractId(It.IsAny<int>(), It.IsAny<CancellationToken>()),
|
||||
Times.Once);
|
||||
|
||||
|
||||
}
|
||||
|
||||
private static Extract GetRawSqlExtract(string outboundFileSql = null)
|
||||
{
|
||||
var extract = new Extract()
|
||||
{
|
||||
ExtractId = 1001,
|
||||
Name = "Extraction Point",
|
||||
Description = "insert something here...",
|
||||
ExtractType = ExtractType.CustomSql,
|
||||
StrataId = 1,
|
||||
RawSql = "SELECT * FROM dbo.ALLDATAEVER",
|
||||
OutboundFileSql = outboundFileSql
|
||||
|
||||
};
|
||||
return extract;
|
||||
}
|
||||
|
||||
private static List<S3FileInfo> GetFiles(string fileName, ExternalStageUrlTokens externalStage, int fileNumber)
|
||||
{
|
||||
var files = new List<S3FileInfo>();
|
||||
for (var i = 0; i < fileNumber; i++)
|
||||
{
|
||||
files.Add(new S3FileInfo { FilePath = $"{externalStage.BucketName}/{externalStage.FolderPath}/{fileName}{i}.csv.gz", FileSize = 100, RowCount = 1000 });
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
private ExtractExportJob CreateExtractExportJob()
|
||||
{
|
||||
_s3BucketService = new Mock<IS3BucketService>();
|
||||
_extractService = new Mock<IExtractService>();
|
||||
_historyService = new Mock<IExtractExportHistoryService>();
|
||||
_recurringJobService = new Mock<IRecurringJobService>();
|
||||
_sftpService = new Mock<ISftpService>();
|
||||
_dataQueryService = new Mock<IDataQueryService>();
|
||||
_claimsPrincipalAccessor = IClaimsPrincipalAccessorMock.GetClaimsPrincipalAccessor();
|
||||
_logger = new Mock<ILogger<ExtractExportJob>>();
|
||||
_scheduleService = new Mock<IExtractScheduleService>();
|
||||
_config = new Mock<IOptions<ExtractConfiguration>>();
|
||||
|
||||
|
||||
var config = new ExtractConfiguration{FileExtension = "txt", MaxFileSize = 1000, SnowflakeMaxFileSize = 1000, OutboundFileExtension = "csv", MaxRowsPerFile = 10};
|
||||
_config.Setup(c => c.Value).Returns(config);
|
||||
|
||||
return new ExtractExportJob(_s3BucketService.Object, _extractService.Object, _historyService.Object, _recurringJobService.Object, _sftpService.Object,
|
||||
_dataQueryService.Object, _claimsPrincipalAccessor, _logger.Object, _scheduleService.Object, _config.Object);
|
||||
}
|
||||
|
||||
private static ExtractExportRequest GetExtractExportRequest(Extract extract)
|
||||
{
|
||||
return new ExtractExportRequest
|
||||
{
|
||||
ExtractId = extract.ExtractId,
|
||||
IncludeOutboundFile = false,
|
||||
OutputFileName = $"{extract.Name}_extended"
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FluentAssertions;
|
||||
using NUnit.Framework;
|
||||
using Strata.Analytics.Biz.Extracts.Models;
|
||||
using Strata.Analytics.Biz.Extracts.Services;
|
||||
using Strata.Analytics.Models.Security;
|
||||
using Strata.Analytics.Test.Unit.Biz.TestData;
|
||||
using Strata.CoreLib.Claims.Extensions;
|
||||
using Strata.Testing.Categories;
|
||||
|
||||
namespace Strata.Analytics.Test.Unit.Biz.Extracts
|
||||
{
|
||||
[NUnit.Framework.Category(TestCategory.Unit)]
|
||||
[Parallelizable(ParallelScope.Self)]
|
||||
public class ExtractGetCurrentUserRoleTest
|
||||
{
|
||||
[Test]
|
||||
public async Task ExtractCurrentUserRole_IsNull_GivenNoRoleAssignments()
|
||||
{
|
||||
// arrange
|
||||
var claimsPrincipal = IClaimsPrincipalAccessorMock.GetClaimsPrincipalAccessor();
|
||||
var dbContext = AnalyticsDbContextMock.GetAnalyticsDbContext(claimsPrincipal);
|
||||
var extractService = new ExtractService(dbContext, claimsPrincipal, AutoMapperMock.GetAutoMapper());
|
||||
|
||||
// act
|
||||
var role = await extractService.GetCurrentUserRoleAsync(5, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
role.HasValue.Should().BeFalse();
|
||||
}
|
||||
|
||||
|
||||
[Test]
|
||||
public async Task ExtractCurrentUserRole_IsEditor_GivenUserIsStrata()
|
||||
{
|
||||
// arrange
|
||||
var claimsPrincipal = IClaimsPrincipalAccessorMock.GetClaimsPrincipalAccessor(5, true);
|
||||
var dbContext = AnalyticsDbContextMock.GetAnalyticsDbContext(claimsPrincipal);
|
||||
var extractService = new ExtractService(dbContext, claimsPrincipal, AutoMapperMock.GetAutoMapper());
|
||||
|
||||
// act
|
||||
var role = await extractService.GetCurrentUserRoleAsync(5, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
role.Should().Be(ExtractRole.Editor);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ExtractCurrentUserRole_UsesCurrentUser_GivenMultipleUsersAssignedToMultipleRoles()
|
||||
{
|
||||
// arrange
|
||||
var claimsPrincipal = IClaimsPrincipalAccessorMock.GetClaimsPrincipalAccessor();
|
||||
var dbContext = AnalyticsDbContextMock.GetAnalyticsDbContext(claimsPrincipal);
|
||||
var extractService = new ExtractService(dbContext, claimsPrincipal, AutoMapperMock.GetAutoMapper());
|
||||
|
||||
var userRoleAssignemnt = new ExtractRoleAssignment() { StrataId = 5, ExtractId = 5, IdentityGuid = claimsPrincipal.GetUserGuid(), IdentityName = claimsPrincipal.GetUsername(), IdentityType = IdentityType.User, Role = ExtractRole.Viewer };
|
||||
var differentUserRoleAssignemnt = new ExtractRoleAssignment() { StrataId = 5, ExtractId = 5, IdentityGuid = Guid.NewGuid(), IdentityName = "Random User", IdentityType = IdentityType.User, Role = ExtractRole.Editor };
|
||||
|
||||
dbContext.ExtractRoleAssignments.Add(userRoleAssignemnt);
|
||||
dbContext.ExtractRoleAssignments.Add(differentUserRoleAssignemnt);
|
||||
|
||||
await dbContext.SaveChangesAsync();
|
||||
|
||||
// act
|
||||
var role = await extractService.GetCurrentUserRoleAsync(5, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
role.Should().Be(userRoleAssignemnt.Role);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ExtractCurrentUserRole_ReturnsMostPrivilege_GivenUserAssignedToMultipleRoles()
|
||||
{
|
||||
// arrange
|
||||
var userGroup = "DSS - Admin";
|
||||
var userGroupGuid = Guid.NewGuid();
|
||||
var claimsPrincipal = IClaimsPrincipalAccessorMock.GetClaimsPrincipalAccessor(5, false, false, new string[] { userGroup });
|
||||
var dbContext = AnalyticsDbContextMock.GetAnalyticsDbContext(claimsPrincipal);
|
||||
var extractService = new ExtractService(dbContext, claimsPrincipal, AutoMapperMock.GetAutoMapper());
|
||||
|
||||
var viewerRoleAssignment = new ExtractRoleAssignment() { StrataId = 5, ExtractId = 5, IdentityGuid = claimsPrincipal.GetUserGuid(), IdentityName = claimsPrincipal.GetUsername(), IdentityType = IdentityType.User, Role = ExtractRole.Viewer };
|
||||
var editorRoleAssignment = new ExtractRoleAssignment() { StrataId = 5, ExtractId = 5, IdentityGuid = userGroupGuid, IdentityName = userGroup, IdentityType = IdentityType.UserGroup, Role = ExtractRole.Editor };
|
||||
dbContext.ExtractRoleAssignments.Add(viewerRoleAssignment);
|
||||
dbContext.ExtractRoleAssignments.Add(editorRoleAssignment);
|
||||
|
||||
await dbContext.SaveChangesAsync();
|
||||
|
||||
// act
|
||||
var role = await extractService.GetCurrentUserRoleAsync(5, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
role.Should().Be(ExtractRole.Editor);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutoMapper;
|
||||
using NUnit.Framework;
|
||||
using Strata.Analytics.Biz.DbContexts;
|
||||
using Strata.Analytics.Biz.Extracts.Models;
|
||||
using Strata.Analytics.Biz.Extracts.Services;
|
||||
using Strata.Analytics.Test.Unit.Biz.TestData;
|
||||
using Strata.CoreLib.Claims;
|
||||
using Strata.CoreLib.Claims.Extensions;
|
||||
using Strata.Testing.Categories;
|
||||
|
||||
namespace Strata.Analytics.Test.Unit.Biz.Extracts
|
||||
{
|
||||
[Category(TestCategory.Unit)]
|
||||
[Parallelizable(ParallelScope.Self)]
|
||||
public class ExtractHistoryCrudTests
|
||||
{
|
||||
private IClaimsPrincipalAccessor _claimsPrincipalAccessor;
|
||||
private AnalyticsDbContext _dbContext;
|
||||
private IMapper _mapper;
|
||||
private ExtractExportHistoryService _historyService;
|
||||
|
||||
[OneTimeSetUp]
|
||||
public void OneTimeSetup()
|
||||
{
|
||||
_claimsPrincipalAccessor = IClaimsPrincipalAccessorMock.GetClaimsPrincipalAccessor();
|
||||
_dbContext = AnalyticsDbContextMock.GetAnalyticsDbContext(_claimsPrincipalAccessor);
|
||||
_mapper = AutoMapperMock.GetAutoMapper();
|
||||
_historyService = new ExtractExportHistoryService(_dbContext, _claimsPrincipalAccessor, _mapper);
|
||||
_dbContext.Extracts.AddRange(ExtractTestData.Extracts());
|
||||
_dbContext.ExtractExportHistory.AddRange(ExtractTestData.ExtractExportHistories());
|
||||
_dbContext.SaveChanges();
|
||||
}
|
||||
|
||||
[OneTimeTearDown]
|
||||
public async Task OneTimeTearDown()
|
||||
{
|
||||
await _dbContext.Database.EnsureDeletedAsync();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task CanCreate()
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
// arrange
|
||||
var saveInfo = new ExtractExportHistorySaveInfo()
|
||||
{
|
||||
ExtractId = 2,
|
||||
OutputFilename = "Newest Export File",
|
||||
ExportStartDate = now.AddMinutes(-10),
|
||||
ExportEndDate = now.AddMinutes(-1),
|
||||
SqlExecuted = "This is the Sql Executed",
|
||||
TotalFileSize = 1000,
|
||||
TotalRowCount = 50000
|
||||
};
|
||||
|
||||
// act
|
||||
var historyId = await _historyService.CreateAsync(saveInfo, CancellationToken.None);
|
||||
var result = await _historyService.GetByIdAsync(historyId, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
Assert.IsNotNull(result);
|
||||
Assert.That(result.ExtractHistoryId, Is.GreaterThan(ExtractTestData.ExtractExportHistories().Max(h => h.ExtractHistoryId)));
|
||||
Assert.That(result.ExtractId, Is.EqualTo(saveInfo.ExtractId));
|
||||
Assert.That(result.OutputFilename, Is.EqualTo(saveInfo.OutputFilename));
|
||||
Assert.That(result.ExportStartDate, Is.EqualTo(saveInfo.ExportStartDate));
|
||||
Assert.That(result.ExportEndDate, Is.EqualTo(saveInfo.ExportEndDate));
|
||||
Assert.That(result.SqlExecuted, Is.EqualTo(saveInfo.SqlExecuted));
|
||||
Assert.That(result.TotalFileSize, Is.EqualTo(saveInfo.TotalFileSize));
|
||||
Assert.That(result.TotalRowCount, Is.EqualTo(saveInfo.TotalRowCount));
|
||||
Assert.That(result.CreatedByUserGuid, Is.EqualTo(_claimsPrincipalAccessor.GetCurrentClaimsPrincipal().GetUserGuid()));
|
||||
Assert.IsNotNull(result.CreatedDate);
|
||||
}
|
||||
|
||||
|
||||
[Test]
|
||||
public async Task CanGetByExtractId()
|
||||
{
|
||||
// act
|
||||
var result = await _historyService.GetByExtractIdAsync(1, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
Assert.IsNotNull(result);
|
||||
Assert.That(result.Count, Is.EqualTo(ExtractTestData.ExtractExportHistories().Count(h => h.ExtractId == 1)));
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FluentAssertions;
|
||||
using NUnit.Framework;
|
||||
using Strata.Analytics.Biz.Extracts.Models;
|
||||
using Strata.Analytics.Biz.Extracts.Services;
|
||||
using Strata.Analytics.Models.Security;
|
||||
using Strata.Analytics.Models.Utils;
|
||||
using Strata.Analytics.Test.Unit.Biz.TestData;
|
||||
using Strata.Testing.Categories;
|
||||
|
||||
namespace Strata.Analytics.Test.Unit.Biz.Extracts
|
||||
{
|
||||
[NUnit.Framework.Category(TestCategory.Unit)]
|
||||
[Parallelizable(ParallelScope.Self)]
|
||||
public class ExtractRoleAssignmentTest
|
||||
{
|
||||
|
||||
[Test]
|
||||
public async Task ExtractRoleAssignment_Created_GivenRoleAssignmentWithIdentityIsNotAlreadyInDatabase()
|
||||
{
|
||||
// arrange
|
||||
var claimsPrincipal = IClaimsPrincipalAccessorMock.GetClaimsPrincipalAccessor();
|
||||
var dbContext = AnalyticsDbContextMock.GetAnalyticsDbContext(claimsPrincipal);
|
||||
var extractService = new ExtractService(dbContext, claimsPrincipal, AutoMapperMock.GetAutoMapper());
|
||||
|
||||
var batch = new Batch<ExtractRoleAssignmentSaveInfo, Guid>()
|
||||
{
|
||||
Updates = new List<ExtractRoleAssignmentSaveInfo>()
|
||||
{
|
||||
new ExtractRoleAssignmentSaveInfo() { IdentityGuid = Guid.NewGuid(), IdentityName="Admin", IdentityType=IdentityType.User, Role=ExtractRole.Editor}
|
||||
}
|
||||
};
|
||||
|
||||
// act
|
||||
await extractService.SaveRoleAssignmentsAsync(5, batch, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
dbContext.ExtractRoleAssignments.Should().HaveCount(1);
|
||||
dbContext.ExtractRoleAssignments.FirstOrDefault().ExtractId.Should().Be(5);
|
||||
dbContext.ExtractRoleAssignments.FirstOrDefault().IdentityGuid.Should().Be(batch.Updates.First().IdentityGuid);
|
||||
dbContext.ExtractRoleAssignments.FirstOrDefault().IdentityName.Should().Be(batch.Updates.First().IdentityName);
|
||||
dbContext.ExtractRoleAssignments.FirstOrDefault().Role.Should().Be(batch.Updates.First().Role);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ExtractRoleAssignment_Updated_GivenRoleAssignmentWithIdentityIsAlreadyInDatabase()
|
||||
{
|
||||
// arrange
|
||||
var strataId = 5;
|
||||
var claimsPrincipal = IClaimsPrincipalAccessorMock.GetClaimsPrincipalAccessor(strataId);
|
||||
var dbContext = AnalyticsDbContextMock.GetAnalyticsDbContext(claimsPrincipal);
|
||||
var extractService = new ExtractService(dbContext, claimsPrincipal, AutoMapperMock.GetAutoMapper());
|
||||
|
||||
var existingRoleAssigment = new ExtractRoleAssignment() { StrataId = strataId, ExtractId = 5, IdentityGuid = Guid.NewGuid(), IdentityName = "Admin", IdentityType = IdentityType.User, Role = ExtractRole.Viewer };
|
||||
dbContext.ExtractRoleAssignments.Add(existingRoleAssigment);
|
||||
await dbContext.SaveChangesAsync();
|
||||
|
||||
// change role to Editor
|
||||
var batch = new Batch<ExtractRoleAssignmentSaveInfo, Guid>()
|
||||
{
|
||||
Updates = new List<ExtractRoleAssignmentSaveInfo>()
|
||||
{
|
||||
new ExtractRoleAssignmentSaveInfo() { IdentityGuid = existingRoleAssigment.IdentityGuid, IdentityName="Admin", IdentityType=IdentityType.User, Role=ExtractRole.Editor}
|
||||
}
|
||||
};
|
||||
|
||||
// act
|
||||
await extractService.SaveRoleAssignmentsAsync(5, batch, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
dbContext.ExtractRoleAssignments.Should().HaveCount(1);
|
||||
dbContext.ExtractRoleAssignments.FirstOrDefault().ExtractId.Should().Be(5);
|
||||
dbContext.ExtractRoleAssignments.FirstOrDefault().IdentityGuid.Should().Be(batch.Updates.First().IdentityGuid);
|
||||
dbContext.ExtractRoleAssignments.FirstOrDefault().IdentityName.Should().Be(batch.Updates.First().IdentityName);
|
||||
dbContext.ExtractRoleAssignments.FirstOrDefault().Role.Should().Be(batch.Updates.First().Role);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ExtractRoleAssignment_Deleted_GivenIdentityGuidIsPartOfBatchDeletes()
|
||||
{
|
||||
// arrange
|
||||
var strataId = 5;
|
||||
var claimsPrincipal = IClaimsPrincipalAccessorMock.GetClaimsPrincipalAccessor(strataId);
|
||||
var dbContext = AnalyticsDbContextMock.GetAnalyticsDbContext(claimsPrincipal);
|
||||
var extractService = new ExtractService(dbContext, claimsPrincipal, AutoMapperMock.GetAutoMapper());
|
||||
|
||||
var existingRoleAssigment = new ExtractRoleAssignment() { StrataId = strataId, ExtractId = 5, IdentityGuid = Guid.NewGuid(), IdentityName = "Admin", IdentityType = IdentityType.User, Role = ExtractRole.Editor };
|
||||
dbContext.ExtractRoleAssignments.Add(existingRoleAssigment);
|
||||
await dbContext.SaveChangesAsync();
|
||||
|
||||
var batch = new Batch<ExtractRoleAssignmentSaveInfo, Guid>()
|
||||
{
|
||||
Deletes = new List<Guid>() { existingRoleAssigment.IdentityGuid }
|
||||
};
|
||||
|
||||
// act
|
||||
await extractService.SaveRoleAssignmentsAsync(5, batch, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
dbContext.ExtractRoleAssignments.Should().HaveCount(0);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,362 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutoMapper;
|
||||
using Moq;
|
||||
using NUnit.Framework;
|
||||
using Strata.Analytics.Biz.DbContexts;
|
||||
using Strata.Analytics.Biz.Extracts.Models;
|
||||
using Strata.Analytics.Biz.Extracts.Services;
|
||||
using Strata.Analytics.Biz.Hangfire;
|
||||
using Strata.Analytics.Biz.Utils;
|
||||
using Strata.Analytics.Test.Unit.Biz.TestData;
|
||||
using Strata.CoreLib.Claims;
|
||||
using Strata.CoreLib.Claims.Extensions;
|
||||
using Strata.Testing.Categories;
|
||||
|
||||
namespace Strata.Analytics.Test.Unit.Biz.Extracts
|
||||
{
|
||||
[Category(TestCategory.Unit)]
|
||||
[Parallelizable(ParallelScope.None)]
|
||||
public class ExtractScheduleTests
|
||||
{
|
||||
private IClaimsPrincipalAccessor _claimsPrincipalAccessor;
|
||||
private AnalyticsDbContext _dbContext;
|
||||
private IMapper _mapper;
|
||||
private IExtractScheduleService _scheduleService;
|
||||
private Mock<IRecurringJobService> _recurringJobService;
|
||||
private string _userTimeZone;
|
||||
private static readonly DateTime Now = DateTime.Now;
|
||||
private static readonly DateOnly NowDate = new(Now.Year, Now.Month, Now.Day);
|
||||
private static readonly TimeOnly NowTime = new(Now.Hour, Now.Minute);
|
||||
|
||||
[OneTimeSetUp]
|
||||
public void OneTimeSetup()
|
||||
{
|
||||
_claimsPrincipalAccessor = IClaimsPrincipalAccessorMock.GetClaimsPrincipalAccessor(5, true);
|
||||
_dbContext = AnalyticsDbContextMock.GetAnalyticsDbContext(_claimsPrincipalAccessor);
|
||||
_mapper = AutoMapperMock.GetAutoMapper();
|
||||
_recurringJobService = new Mock<IRecurringJobService>();
|
||||
_scheduleService = new ExtractScheduleService(_dbContext, _claimsPrincipalAccessor, _mapper, _recurringJobService.Object);
|
||||
_dbContext.Extracts.AddRange(ExtractTestData.Extracts());
|
||||
_dbContext.ExtractSchedule.AddRange(ExtractTestData.ExtractSchedules());
|
||||
_dbContext.SaveChanges();
|
||||
_userTimeZone = _claimsPrincipalAccessor.GetTimezone();
|
||||
}
|
||||
|
||||
[OneTimeTearDown]
|
||||
public async Task OneTimeTearDown()
|
||||
{
|
||||
await _dbContext.Database.EnsureDeletedAsync();
|
||||
}
|
||||
|
||||
|
||||
[Test]
|
||||
public async Task CanCreate_And_Add_RecurringJob()
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
// arrange
|
||||
var saveInfo = new ExtractScheduleSaveInfo
|
||||
{
|
||||
ExtractId = 1,
|
||||
Enabled = true,
|
||||
Repeater = ScheduleRepeaterType.Daily,
|
||||
OnDays = "",
|
||||
StartDate = NowDate,
|
||||
StartTime = NowTime,
|
||||
MaxOccurrences = 12
|
||||
};
|
||||
|
||||
// act
|
||||
var result = await _scheduleService.CreateAsync(saveInfo, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
Assert.IsNotNull(result);
|
||||
Assert.That(result.ExtractScheduleId, Is.GreaterThan(ExtractTestData.ExtractSchedules().Max(h => h.ExtractScheduleId)));
|
||||
Assert.That(result.ExtractId, Is.EqualTo(saveInfo.ExtractId));
|
||||
Assert.That(result.Enabled, Is.EqualTo(saveInfo.Enabled));
|
||||
Assert.That(result.ToCronString(), Is.EqualTo($"{NowTime.Minute} {NowTime.Hour} * * *")); // utc to CST conversion
|
||||
Assert.That(result.StartDate, Is.EqualTo(saveInfo.StartDate));
|
||||
Assert.IsNull(result.EndDate);
|
||||
Assert.That(result.CreatedByUserGuid, Is.EqualTo(_claimsPrincipalAccessor.GetCurrentClaimsPrincipal().GetUserGuid()));
|
||||
Assert.That(result.UpdatedByUserGuid, Is.EqualTo(_claimsPrincipalAccessor.GetCurrentClaimsPrincipal().GetUserGuid()));
|
||||
Assert.IsNotNull(result.CreatedDate);
|
||||
Assert.IsNotNull(result.UpdatedDate);
|
||||
|
||||
_recurringJobService.Verify(
|
||||
s => s.AddOrUpdateExtractExportJobAsync(It.Is<ExtractExportRequest>(r =>
|
||||
r.ExtractScheduleId == result.ExtractScheduleId && r.ExtractId == result.ExtractId &&
|
||||
r.Cron == result.ToCronString()), It.IsAny<CancellationToken>()), Times.Once);
|
||||
|
||||
}
|
||||
|
||||
|
||||
[Test]
|
||||
public async Task CanGetById()
|
||||
{
|
||||
// act
|
||||
var result = await _scheduleService.GetByIdAsync(1, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
Assert.IsNotNull(result);
|
||||
}
|
||||
|
||||
|
||||
[Test]
|
||||
public async Task CanUpdate()
|
||||
{
|
||||
// arrange
|
||||
var scheduleId = _dbContext.ExtractSchedule.First(s => s.MaxOccurrences.HasValue).ExtractId;
|
||||
|
||||
var schedule = await _scheduleService.GetByIdAsync(scheduleId, CancellationToken.None);
|
||||
|
||||
var saveInfo = new ExtractScheduleSaveInfo();
|
||||
_mapper.Map(schedule, saveInfo);
|
||||
|
||||
saveInfo.EndDate = NowDate.AddYears(1);
|
||||
saveInfo.MaxOccurrences = null;
|
||||
saveInfo.Repeater = ScheduleRepeaterType.Monthly;
|
||||
saveInfo.OnDays = "";
|
||||
_dbContext.ChangeTracker.Clear();
|
||||
|
||||
// act
|
||||
var result = await _scheduleService.UpdateAsync(schedule.ExtractId, saveInfo, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
Assert.IsNotNull(result);
|
||||
Assert.That(result.ExtractId, Is.EqualTo(schedule.ExtractId));
|
||||
Assert.That(result.EndDate, Is.EqualTo(saveInfo.EndDate));
|
||||
Assert.IsNull(result.MaxOccurrences);
|
||||
Assert.That(result.ToCronString(), Is.EqualTo($"{schedule.StartTime.Minute} {schedule.StartTime.Hour} {schedule.StartDate.Day} * *"));
|
||||
Assert.That(result.CreatedByUserGuid, Is.EqualTo(schedule.CreatedByUserGuid));
|
||||
Assert.That(result.UpdatedByUserGuid, Is.EqualTo(_claimsPrincipalAccessor.GetCurrentClaimsPrincipal().GetUserGuid()));
|
||||
Assert.That(result.CreatedDate, Is.EqualTo(schedule.CreatedDate));
|
||||
Assert.That(result.UpdatedDate, Is.EqualTo(DateTimeOffset.UtcNow).Within(500).Milliseconds); // this could fail in debug
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task CanGetAll_FilterNotEnabled()
|
||||
{
|
||||
// arrange
|
||||
var now = DateTime.UtcNow;
|
||||
// arrange
|
||||
var saveInfo = new ExtractScheduleSaveInfo
|
||||
{
|
||||
ExtractId = 1,
|
||||
Enabled = false,
|
||||
Repeater = ScheduleRepeaterType.Daily,
|
||||
OnDays = "",
|
||||
StartDate = NowDate,
|
||||
StartTime = NowTime,
|
||||
MaxOccurrences = 12
|
||||
};
|
||||
|
||||
var disabledSchedule = await _scheduleService.CreateAsync(saveInfo, CancellationToken.None);
|
||||
|
||||
// act
|
||||
var result = (await _scheduleService.GetAllAsync(CancellationToken.None)).ToList();
|
||||
|
||||
// assert
|
||||
Assert.IsNotNull(result);
|
||||
Assert.That(result.Count, Is.GreaterThan(0));
|
||||
Assert.IsFalse(result.Select(s => s.ExtractScheduleId).Contains(disabledSchedule.ExtractScheduleId));
|
||||
Assert.IsTrue(result.TrueForAll(s => s.Enabled));
|
||||
}
|
||||
|
||||
|
||||
[Test]
|
||||
public async Task Disabling_Removes_RecurringJob()
|
||||
{
|
||||
// arrange
|
||||
var scheduleId = _dbContext.ExtractSchedule.Last(s => s.Enabled).ExtractId;
|
||||
|
||||
var schedule = await _scheduleService.GetByIdAsync(scheduleId, CancellationToken.None);
|
||||
|
||||
var saveInfo = new ExtractScheduleSaveInfo();
|
||||
_mapper.Map(schedule, saveInfo);
|
||||
|
||||
saveInfo.Enabled = false;
|
||||
_dbContext.ChangeTracker.Clear();
|
||||
|
||||
// act
|
||||
await _scheduleService.UpdateAsync(schedule.ExtractId, saveInfo, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
|
||||
_recurringJobService.Verify(
|
||||
s => s.RemoveExtractExportJobAsync(It.IsAny<int>(), It.Is<int>(r => r == schedule.ExtractScheduleId),
|
||||
It.IsAny<CancellationToken>()), Times.Once);
|
||||
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task HittingMaxOccurrenceLimit_Removes_RecurringJob()
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
// arrange
|
||||
var saveInfo = new ExtractScheduleSaveInfo
|
||||
{
|
||||
ExtractId = 1,
|
||||
Enabled = true,
|
||||
Repeater = ScheduleRepeaterType.Daily,
|
||||
OnDays = "",
|
||||
StartDate = NowDate.AddDays(-7), // starting 7 days ago
|
||||
StartTime = NowTime,
|
||||
MaxOccurrences = 6 // run a total of 6 times
|
||||
};
|
||||
|
||||
var schedule = await _scheduleService.CreateAsync(saveInfo, CancellationToken.None);
|
||||
_dbContext.ChangeTracker.Clear();
|
||||
|
||||
// act
|
||||
var result = await _scheduleService.UpdateLastOccurrence(schedule.ExtractScheduleId, DateTimeOffset.UtcNow,
|
||||
_userTimeZone, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
Assert.That(result.Enabled, Is.EqualTo(false));
|
||||
|
||||
_recurringJobService.Verify(
|
||||
s => s.RemoveExtractExportJobAsync(It.IsAny<int>(), It.Is<int>(r => r == schedule.ExtractScheduleId),
|
||||
It.IsAny<CancellationToken>()), Times.Once);
|
||||
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task HittingEndDateLimit_Removes_RecurringJob()
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
// arrange
|
||||
var saveInfo = new ExtractScheduleSaveInfo
|
||||
{
|
||||
ExtractId = 1,
|
||||
Enabled = true,
|
||||
Repeater = ScheduleRepeaterType.Daily,
|
||||
OnDays = "",
|
||||
StartDate = NowDate.AddDays(-7), // starting 7 days ago
|
||||
StartTime = NowTime,
|
||||
EndDate = NowDate // run until today
|
||||
};
|
||||
|
||||
var schedule = await _scheduleService.CreateAsync(saveInfo, CancellationToken.None);
|
||||
_dbContext.ChangeTracker.Clear();
|
||||
|
||||
// act
|
||||
var result = await _scheduleService.UpdateLastOccurrence(schedule.ExtractScheduleId, DateTimeOffset.UtcNow,
|
||||
_userTimeZone, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
Assert.That(result.Enabled, Is.EqualTo(false));
|
||||
|
||||
_recurringJobService.Verify(
|
||||
s => s.RemoveExtractExportJobAsync(It.IsAny<int>(), It.Is<int>(r => r == schedule.ExtractScheduleId),
|
||||
It.IsAny<CancellationToken>()), Times.Once);
|
||||
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task CanDelete_And_Remove_RecurringJob()
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
// arrange
|
||||
var saveInfo = new ExtractScheduleSaveInfo
|
||||
{
|
||||
ExtractId = 1,
|
||||
Enabled = true,
|
||||
Repeater = ScheduleRepeaterType.Daily,
|
||||
OnDays = "",
|
||||
StartDate = NowDate.AddDays(-7), // starting 7 days ago
|
||||
StartTime = NowTime,
|
||||
};
|
||||
|
||||
var schedule = await _scheduleService.CreateAsync(saveInfo, CancellationToken.None);
|
||||
_dbContext.ChangeTracker.Clear();
|
||||
|
||||
// act
|
||||
await _scheduleService.DeleteAsync(schedule.ExtractScheduleId, CancellationToken.None);
|
||||
|
||||
// assert
|
||||
var result = await _scheduleService.ContainsAsync(schedule.ExtractScheduleId, CancellationToken.None);
|
||||
|
||||
Assert.That(result, Is.EqualTo(false));
|
||||
|
||||
_recurringJobService.Verify(
|
||||
s => s.RemoveExtractExportJobAsync(It.IsAny<int>(), It.Is<int>(r => r == schedule.ExtractScheduleId),
|
||||
It.IsAny<CancellationToken>()), Times.Once);
|
||||
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void CanGetDailyCronFromSchedule()
|
||||
{
|
||||
// arrange
|
||||
var schedule = new ExtractSchedule
|
||||
{
|
||||
Repeater = ScheduleRepeaterType.Daily,
|
||||
OnDays = "",
|
||||
StartDate = new DateOnly(2023, 01, 01),
|
||||
StartTime = new TimeOnly(15, 30, 00)
|
||||
};
|
||||
|
||||
// assert
|
||||
|
||||
Assert.That(schedule.ToCronString(), Is.EqualTo("30 15 * * *"));
|
||||
}
|
||||
|
||||
|
||||
[Test]
|
||||
public void CanGetWeekdayCronFromSchedule()
|
||||
{
|
||||
// arrange
|
||||
var schedule = new ExtractSchedule
|
||||
{
|
||||
Repeater = ScheduleRepeaterType.Weekdays,
|
||||
StartDate = new DateOnly(2023, 01, 01),
|
||||
StartTime = new TimeOnly(15, 30, 00)
|
||||
};
|
||||
|
||||
// assert
|
||||
|
||||
Assert.That(schedule.ToCronString(), Is.EqualTo("30 15 * * MON-FRI"));
|
||||
}
|
||||
|
||||
[TestCase("MON,WED,FRI")]
|
||||
[TestCase("SUN,SAT")]
|
||||
[TestCase("SUN,THU")]
|
||||
public void CanGetWeeklyCronFromSchedule(string onDays)
|
||||
{
|
||||
// arrange
|
||||
var schedule = new ExtractSchedule
|
||||
{
|
||||
Repeater = ScheduleRepeaterType.Weekly,
|
||||
OnDays = onDays,
|
||||
StartDate = new DateOnly(2023, 01, 01),
|
||||
StartTime = new TimeOnly(15, 30, 00)
|
||||
};
|
||||
|
||||
// assert
|
||||
|
||||
Assert.That(schedule.ToCronString(),
|
||||
onDays == string.Empty
|
||||
? Is.EqualTo("30 15 * * *")
|
||||
: Is.EqualTo($"30 15 * * {onDays}"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void CanGetMonthlyCronFromSchedule()
|
||||
{
|
||||
// arrange
|
||||
var schedule = new ExtractSchedule
|
||||
{
|
||||
Repeater = ScheduleRepeaterType.Monthly,
|
||||
OnDays = "",
|
||||
StartDate = new DateOnly(2023, 01, 01),
|
||||
StartTime = new TimeOnly(15, 30, 00)
|
||||
};
|
||||
|
||||
// assert
|
||||
|
||||
Assert.That(schedule.ToCronString(), Is.EqualTo("30 15 1 * *"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json;
|
||||
using NUnit.Framework;
|
||||
using Strata.Analytics.Biz.Extracts.Export;
|
||||
|
||||
namespace Strata.Analytics.Test.Unit.Biz.Extracts
|
||||
{
|
||||
[TestFixture]
|
||||
public class ExtractSqlHelperTest
|
||||
{
|
||||
[Test]
|
||||
public void GetFolderPathFromExternalStage_UrlStringOneLevel_GetFilePath()
|
||||
{
|
||||
//Arrange
|
||||
const string urlString = "s3://sdt-dev-tabularreporter/b8018250af34403ebdcc1770714042b2/";
|
||||
const string expectedString = "b8018250af34403ebdcc1770714042b2";
|
||||
|
||||
//Act
|
||||
var externalStageUrlTokens = ExtractSqlHelper.StripS3BucketFromUrl(urlString);
|
||||
|
||||
//Assert
|
||||
Assert.AreEqual(externalStageUrlTokens.FolderPath, expectedString);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ExtractUrlValue_HasUrlProperty_GetUrlList()
|
||||
{
|
||||
|
||||
//Arrange
|
||||
var urlList = new List<string> { "someUrl" };
|
||||
var urlListString = JsonSerializer.Serialize(urlList);
|
||||
List<dynamic> dynamicList = new() { new Dictionary<string, object>() { { "property", "URL" }, { "property_value", urlListString } } };
|
||||
|
||||
//Act
|
||||
var urlValue = ExtractSqlHelper.ExtractUrlValue(dynamicList);
|
||||
|
||||
//Assert
|
||||
Assert.AreEqual(urlValue, urlList);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GetFolderPathFromExternalStage_UrlStringTwoLevels_GetFilePath()
|
||||
{
|
||||
//Arrange
|
||||
const string urlString = "s3://sdt-dev-tabularreporter/b8018250af34403ebdcc1770714042b2/b8018250af34403ebdcc1770714042b2/";
|
||||
const string expectedString = "b8018250af34403ebdcc1770714042b2/b8018250af34403ebdcc1770714042b2";
|
||||
|
||||
//Act
|
||||
var externalStageUrlTokens = ExtractSqlHelper.StripS3BucketFromUrl(urlString);
|
||||
|
||||
//Assert
|
||||
Assert.AreEqual(externalStageUrlTokens.FolderPath, expectedString);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GetFolderPathFromExternalStage_UrlStringNoLevels_GetFilePath()
|
||||
{
|
||||
//Arrange
|
||||
const string urlString = "s3://sdt-dev-tabularreporter/";
|
||||
const string expectedString = "";
|
||||
|
||||
//Act
|
||||
var externalStageUrlTokens = ExtractSqlHelper.StripS3BucketFromUrl(urlString);
|
||||
|
||||
//Assert
|
||||
Assert.AreEqual(externalStageUrlTokens.FolderPath, expectedString);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GetBucketNameFromExternalStage_UrlStringNoLevels_GetBucketName()
|
||||
{
|
||||
//Arrange
|
||||
const string urlString = "s3://sdt-dev-tabularreporter/";
|
||||
const string expectedString = "sdt-dev-tabularreporter";
|
||||
|
||||
//Act
|
||||
var externalStageUrlTokens = ExtractSqlHelper.StripS3BucketFromUrl(urlString);
|
||||
|
||||
//Assert
|
||||
Assert.AreEqual(externalStageUrlTokens.BucketName, expectedString);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GetBucketNameFromExternalStage_UrlStringOneLevel_GetBucketName()
|
||||
{
|
||||
//Arrange
|
||||
const string urlString = "s3://sdt-dev-tabularreporter/b8018250af34403ebdcc1770714042b2/";
|
||||
const string expectedString = "sdt-dev-tabularreporter";
|
||||
|
||||
//Act
|
||||
var externalStageUrlTokens = ExtractSqlHelper.StripS3BucketFromUrl(urlString);
|
||||
|
||||
//Assert
|
||||
Assert.AreEqual(externalStageUrlTokens.BucketName, expectedString);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user