🔧 updated terraform output handler to trigger user run request when v…

…ersion greater than 2.13
ssc-sp · Sep 28, 2023 · 6503f53 · 6503f53
1 parent 2c7053b
commit 6503f53
Show file tree

Hide file tree

Showing 8 changed files with 172 additions and 80 deletions.
diff --git a/Portal/src/Datahub.Application/Services/IResourceMessagingService.cs b/Portal/src/Datahub.Application/Services/IResourceMessagingService.cs
@@ -7,5 +7,7 @@ public interface IResourceMessagingService
 {
     public Task SendToTerraformQueue(CreateResourceData project);
 
-    public Task SendToUserQueue(WorkspaceDefinition workspaceDefinition);
+    public Task SendToUserQueue(WorkspaceDefinition workspaceDefinition, string? connectionString = null, string? queueName = null);
+
+    public Task<WorkspaceDefinition> GetWorkspaceDefinition(string projectAcronym, string? requestingUserEmail = null);
 }
diff --git a/Portal/src/Datahub.Core/Services/Projects/IRequestManagementService.cs b/Portal/src/Datahub.Core/Services/Projects/IRequestManagementService.cs
@@ -40,5 +40,4 @@ public interface IRequestManagementService
     Task RequestServiceWithDefaults(Datahub_ProjectServiceRequests request);
     Task SaveResourceInputDefinitionJson(string resourceType, string jsonContent);
     Task<bool> UpdateResourceInputParameters(Guid resourceId, Dictionary<string, string> inputParams);
-    Task<WorkspaceDefinition> GetWorkspaceDefinition(string projectAcronym);
 }
diff --git a/Portal/src/Datahub.Functions/AzureConfig.cs b/Portal/src/Datahub.Functions/AzureConfig.cs
@@ -33,6 +33,7 @@ public AzureConfig(IConfiguration config)
 
     public string StorageQueueConnection => _config["DatahubStorageConnectionString"] ?? "";
     public string MaxStorageCapacity => _config["MAX_STORAGE_CAPACITY"] ?? "180000000000"; //"2000000000000";
+    public string UserRunRequestQueueName => _config["UserRunRequestQueueName"] ?? "user-run-request";
 
     #endregion
 

diff --git a/Portal/src/Datahub.Functions/Program.cs b/Portal/src/Datahub.Functions/Program.cs
@@ -1,17 +1,17 @@
 using Datahub.Core.Model.Datahub;
 using Datahub.Functions;
-using Datahub.Infrastructure.Queues.MessageHandlers;
 using Datahub.Infrastructure.Services;
 using Datahub.Infrastructure.Services.Azure;
 using Datahub.Infrastructure.Services.Projects;
-using MediatR;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Polly;
 using Polly.Contrib.WaitAndRetry;
 using System.Net;
+using Datahub.Application.Configuration;
+using Datahub.Application.Services;
 
 var host = new HostBuilder()
     .ConfigureFunctionsWorkerDefaults()
@@ -46,6 +46,9 @@
         services.AddSingleton<AzureManagementService>();
         services.AddScoped<ProjectUsageService>();
         services.AddScoped<QueuePongService>();
+        services.AddScoped<IResourceMessagingService, ResourceMessagingService>();
+        services.AddSingleton<DatahubPortalConfiguration>();
+
     })
     .Build();
 

diff --git a/Portal/src/Datahub.Functions/TerraformOutputHandler.cs b/Portal/src/Datahub.Functions/TerraformOutputHandler.cs
@@ -1,8 +1,10 @@
 using System.Text.Json;
 using System.Text.Json.Nodes;
 using System.Transactions;
+using Datahub.Application.Services;
 using Datahub.Core.Enums;
 using Datahub.Core.Model.Datahub;
+using Datahub.Core.Services.Projects;
 using Datahub.Infrastructure.Services;
 using Datahub.ProjectTools.Services;
 using Datahub.Shared;
@@ -18,12 +20,18 @@ public class TerraformOutputHandler
     private readonly DatahubProjectDBContext _projectDbContext;
     private readonly ILogger _logger;
     private readonly QueuePongService _pongService;
+    private readonly IResourceMessagingService _resourceMessagingService;
+    private readonly AzureConfig _config;
+    private const string TerraformOutputHandlerName = "terraform-output-handler";
 
-    public TerraformOutputHandler(ILoggerFactory loggerFactory, DatahubProjectDBContext projectDbContext, QueuePongService pongService)
+    public TerraformOutputHandler(ILoggerFactory loggerFactory, DatahubProjectDBContext projectDbContext,
+        QueuePongService pongService, IResourceMessagingService resourceMessagingService, AzureConfig config)
     {
         _projectDbContext = projectDbContext;
         _logger = loggerFactory.CreateLogger("TerraformOutputHandler");
         _pongService = pongService;
+        _resourceMessagingService = resourceMessagingService;
+        _config = config;
     }
 
     [Function("TerraformOutputHandler")]
@@ -64,15 +72,65 @@ public async Task RunAsync(
             throw;
         }
 
+        await ProcessPostTerraformTriggers(output);
+
+
         _logger.LogInformation("C# Queue trigger function finished");
     }
 
+    private async Task ProcessPostTerraformTriggers(IReadOnlyDictionary<string, TerraformOutputVariable> output)
+    {
+        _logger.LogInformation("Terraform processing complete, triggering post terraform triggers");
+
+        // check if there's a workspace version variable
+        if (!output.ContainsKey(TerraformVariables.OutputWorkspaceVersion) ||
+            string.IsNullOrWhiteSpace(output[TerraformVariables.OutputWorkspaceVersion].Value))
+        {
+            _logger.LogInformation("Project version is null or empty, skipping post terraform triggers");
+            return;
+        }
+        var projectVersionString = output[TerraformVariables.OutputWorkspaceVersion].Value;
+
+        // exclude the first character, which is a v
+        var projectVersion = new Version(projectVersionString[1..]);
+
+        // double check it's above version 2.13.0
+        if (projectVersion < new Version(2, 13, 0))
+        {
+            _logger.LogInformation("Project version is below 2.13.0, skipping post terraform triggers");
+            return;
+        }
+
+        // handle external user permissions
+        var projectAcronym = output[TerraformVariables.OutputProjectAcronym];
+        var project = await _projectDbContext.Projects
+            .FirstOrDefaultAsync(p => p.Project_Acronym_CD == projectAcronym.Value);
+
+        if (project is null)
+        {
+            _logger.LogError("Project not found for acronym {ProjectId}", projectAcronym.Value);
+            throw new Exception($"Project not found for acronym {projectAcronym.Value}");
+        }
+
+        _logger.LogInformation("Processing user updates to external permissions for project {ProjectAcronym}",
+            projectAcronym.Value);
+        var workspaceDefinition =
+            await _resourceMessagingService.GetWorkspaceDefinition(project.Project_Acronym_CD,
+                TerraformOutputHandlerName);
+        await _resourceMessagingService.SendToUserQueue(workspaceDefinition,
+            _config.StorageQueueConnection, _config.UserRunRequestQueueName);
+        _logger.LogInformation(
+            "Processing complete for user updates to external permissions for project {ProjectAcronym}",
+            projectAcronym.Value);
+    }
+
     private async Task ProcessTerraformOutputVariables(
         IReadOnlyDictionary<string, TerraformOutputVariable> outputVariables)
     {
         try
         {
-            using var transactionScope = new TransactionScope(TransactionScopeOption.Required, TransactionScopeAsyncFlowOption.Enabled);
+            using var transactionScope =
+                new TransactionScope(TransactionScopeOption.Required, TransactionScopeAsyncFlowOption.Enabled);
             await ProcessProjectStatus(outputVariables);
             await ProcessAzureStorageBlob(outputVariables);
             await ProcessAzureDatabricks(outputVariables);
@@ -85,7 +143,7 @@ private async Task ProcessTerraformOutputVariables(
         }
     }
 
-    private async Task ProcessAzureDatabricks(IReadOnlyDictionary<string,TerraformOutputVariable> outputVariables)
+    private async Task ProcessAzureDatabricks(IReadOnlyDictionary<string, TerraformOutputVariable> outputVariables)
     {
         var projectAcronym = outputVariables[TerraformVariables.OutputProjectAcronym];
         var terraformServiceType = RequestManagementService.GetTerraformServiceType(TerraformTemplate.AzureDatabricks);
@@ -98,10 +156,12 @@ private async Task ProcessAzureDatabricks(IReadOnlyDictionary<string,TerraformOu
 
         if (projectRequest is null)
         {
-            _logger.LogInformation("Project request not found for project acronym {ProjectAcronymValue} and service type {TerraformServiceType}", projectAcronym.Value, terraformServiceType);
+            _logger.LogInformation(
+                "Project request not found for project acronym {ProjectAcronymValue} and service type {TerraformServiceType}",
+                projectAcronym.Value, terraformServiceType);
             return;
         }
-        
+
         var databricksStatus = GetStatusMapping(outputVariables[TerraformVariables.OutputAzureDatabricksStatus].Value);
         if (databricksStatus == TerraformOutputStatus.Completed)
         {
@@ -111,24 +171,24 @@ private async Task ProcessAzureDatabricks(IReadOnlyDictionary<string,TerraformOu
         {
             _logger.LogInformation("Azure Databricks status is not completed. Status: {Status}", databricksStatus);
         }
-        
+
         var projectResource = _projectDbContext.Project_Resources2
             .Where(x => x.ProjectId == projectRequest.Project.Project_ID)
             .FirstOrDefault(x => x.ResourceType == terraformServiceType);
-        
+
         if (projectResource is null)
         {
             var inputParameters = new Dictionary<string, string>();
             projectResource = RequestManagementService.CreateEmptyProjectResource(projectRequest, inputParameters);
             _projectDbContext.Project_Resources2.Add(projectResource);
         }
-        
+
         if (!projectResource.TimeCreated.HasValue)
         {
             var workspaceId = outputVariables[TerraformVariables.OutputAzureDatabricksWorkspaceId];
             var workspaceUrl = outputVariables[TerraformVariables.OutputAzureDatabricksWorkspaceUrl];
             var workspaceName = outputVariables[TerraformVariables.OutputAzureDatabricksWorkspaceName];
-            
+
             var jsonContent = new JsonObject
             {
                 ["workspace_id"] = workspaceId.Value,
@@ -137,14 +197,16 @@ private async Task ProcessAzureDatabricks(IReadOnlyDictionary<string,TerraformOu
             };
 
             var inputJsonContent = new JsonObject();
-            
+
             projectResource.TimeCreated = DateTime.Now;
             projectResource.JsonContent = jsonContent.ToString();
             projectResource.InputJsonContent = inputJsonContent.ToString();
         }
         else
         {
-            _logger.LogInformation("Project resource already exists for project {ProjectAcronym} and service type {ServiceType}", projectAcronym.Value, terraformServiceType);
+            _logger.LogInformation(
+                "Project resource already exists for project {ProjectAcronym} and service type {ServiceType}",
+                projectAcronym.Value, terraformServiceType);
         }
 
         await _projectDbContext.SaveChangesAsync();
@@ -163,11 +225,14 @@ private async Task ProcessAzureStorageBlob(IReadOnlyDictionary<string, Terraform
 
         if (projectRequest is null)
         {
-            _logger.LogInformation("Project request not found for project acronym {ProjectAcronymValue} and service type {TerraformServiceType}", projectAcronym.Value, terraformServiceType);
+            _logger.LogInformation(
+                "Project request not found for project acronym {ProjectAcronymValue} and service type {TerraformServiceType}",
+                projectAcronym.Value, terraformServiceType);
             return;
         }
 
-        var storageBlobStatus = GetStatusMapping(outputVariables[TerraformVariables.OutputAzureStorageBlobStatus].Value);
+        var storageBlobStatus =
+            GetStatusMapping(outputVariables[TerraformVariables.OutputAzureStorageBlobStatus].Value);
         if (storageBlobStatus == TerraformOutputStatus.Completed)
         {
             projectRequest.Is_Completed = DateTime.Now;
@@ -176,19 +241,19 @@ private async Task ProcessAzureStorageBlob(IReadOnlyDictionary<string, Terraform
         {
             _logger.LogInformation("Azure storage blob status is not completed. Status: {Status}", storageBlobStatus);
         }
-        
-        
+
+
         var projectResource = _projectDbContext.Project_Resources2
             .Where(x => x.ProjectId == projectRequest.Project.Project_ID)
             .FirstOrDefault(x => x.ResourceType == terraformServiceType);
-        
+
         if (projectResource is null)
         {
             var inputParameters = new Dictionary<string, string>();
             projectResource = RequestManagementService.CreateEmptyProjectResource(projectRequest, inputParameters);
             _projectDbContext.Project_Resources2.Add(projectResource);
         }
-        
+
         if (!projectResource.TimeCreated.HasValue)
         {
             var accountName = outputVariables[TerraformVariables.OutputAzureStorageAccountName];
@@ -206,14 +271,16 @@ private async Task ProcessAzureStorageBlob(IReadOnlyDictionary<string, Terraform
             {
                 ["storage_type"] = TerraformVariables.AzureStorageType
             };
-            
+
             projectResource.TimeCreated = DateTime.Now;
             projectResource.JsonContent = jsonContent.ToString();
             projectResource.InputJsonContent = inputJsonContent.ToString();
         }
         else
         {
-            _logger.LogInformation("Project resource already exists for project {ProjectAcronym} and service type {ServiceType}", projectAcronym.Value, terraformServiceType);
+            _logger.LogInformation(
+                "Project resource already exists for project {ProjectAcronym} and service type {ServiceType}",
+                projectAcronym.Value, terraformServiceType);
         }
 
         await _projectDbContext.SaveChangesAsync();
@@ -236,12 +303,21 @@ private async Task ProcessProjectStatus(IReadOnlyDictionary<string, TerraformOut
         {
             project.Project_Phase = outputPhase;
         }
-
-        var workspaceVersion = outputVariables[TerraformVariables.OutputWorkspaceVersion].Value;
-        if (project.Version != workspaceVersion)
+
+        // check if there's a workspace version variable
+        if (outputVariables.ContainsKey(TerraformVariables.OutputWorkspaceVersion))
+        {
+            var workspaceVersion = outputVariables[TerraformVariables.OutputWorkspaceVersion].Value;
+            if (project.Version != workspaceVersion)
+            {
+                project.Version = workspaceVersion;
+            }
+        }
+        else
         {
-            project.Version = workspaceVersion;
+            _logger.LogInformation("Workspace version not found in output variables");
         }
+
         await _projectDbContext.SaveChangesAsync();
     }