Merge pull request #596 from Laixer/develop

v3.3.9
Laixer · Sep 24, 2021 · 77b6b18 · 77b6b18
2 parents ce0f58a + 3606004
commit 77b6b18
Show file tree

Hide file tree

Showing 9 changed files with 117 additions and 59 deletions.
diff --git a/README.md b/README.md
@@ -2,32 +2,45 @@
 
 ![FunderMaps Ecosystem](https://github.com/Laixer/FunderMaps/workflows/FunderMaps%20Ecosystem/badge.svg)
 
-More and more foundation issues occur and are actively investigated by local government. Gaining insight and acquiring more detailed data, **FunderMaps** enables municipalities and large property owners to register this data at the property level. Enrichment with data from Remote Monitoring or national data layers, various analyzes are performed that provide insight into the risk of foundation damage.
+Foundation issues occur and are actively investigated by local government. Gaining insight and acquiring more detailed data, **FunderMaps** enables municipalities and large property owners to register this data at the property level. Enrichment with data from Remote Monitoring or national data layers, various analyzes are performed that provide insight into the risk of foundation damage.
 
-## Running the application
+## Requirements
 
-After cloning or downloading the application you should be able to run it with a persistent database. You will need to run its Entity Framework Core migrations before you will be able to run the app, and update the `ConfigureServices` method in `Startup.cs` (see below).
+* .NET 5 Runtime and SDK
+* Docker
 
-### Configuring the sample to use PostgreSQL
+## Running the application on localhost
 
-1. Ensure your connection strings in `appsettings.json` point to a PostgreSQL instance.
+After cloning or downloading the application you will be able to run the application on localhost. You will need to run the setup and load scripts to configure the local database.
 
-1. Open a command prompt in the Web folder and execute the following commands:
+1. Run from solution directory. This will setup the PostgreSQL instance and seed the database.
 
 ```
-dotnet restore
-dotnet ef database update
+./scripts/setupdb.sh
+./scripts/loaddb.sh
 ```
 
-These commands will create for the database for the app's user credentials and identity data.
+3. Ensure your connection strings in `appsettings.{ENV}.json` point to the local PostgreSQL instance. For example the *FunderMaps.WebApi* would have `Server=localhost;Database=fundermaps;User Id=fundermaps_webapp` where the user id corresponds to the application.
 
-1. Run the application.
+4. Enter the project directory in `src/{project}`.
 
-The first time you run the application, it will seed both databases with data such that you should see products in the store, and you should be able to log in using the admin@contoso.com account.
+5. Run the application with `ASPNETCORE_ENVIRONMENT={ENV} dotnet run`. The application should keep running in the foreground. The foreground logging shows the connection details.
 
-Note: If you need to create migrations, you can use this commands:
+When using VS Code the 'debug' section should list all the applications. Just run the application (or hit F5).
 
-```
--- create migration (from Web folder CLI)
-dotnet ef migrations add InitialModel -o Data/Migrations
-```
+## Configuration
+
+See the `contrib/etc/_appsettings.{ENV}.json` directory for configuration files for each environment. You can copy these configuration files to the project source directory.
+
+## Using the application
+
+See the user table below. All users use the same password: `fundermaps`.
+
+| Name           | Email                 | Function      |
+|----------------|-----------------------|---------------|
+| Administrator  | admin@fundermaps.com  | Administrator |
+|                | Javier40@yahoo.com    | Superuser     |
+| kihn           | Freda@contoso.com     | Reviewer      |
+| Patsy Brekke   | patsy@contoso.com     | Writer        |
+| Lester Bednar  | lester@contoso.com    | Reader        |
+|                | corene@contoso.com    | Reader        |
diff --git a/src/FunderMaps.AspNetCore/FunderMaps.AspNetCore.csproj b/src/FunderMaps.AspNetCore/FunderMaps.AspNetCore.csproj
@@ -12,7 +12,7 @@
 
   <ItemGroup>
     <PackageReference Include="AutoMapper.Extensions.Microsoft.DependencyInjection" Version="8.1.1" />
-    <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="5.0.8" />
+    <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="5.0.10" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/FunderMaps.Core/FunderMaps.Core.csproj b/src/FunderMaps.Core/FunderMaps.Core.csproj
@@ -25,7 +25,7 @@
     <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="5.0.0" />
-    <PackageReference Include="Scriban" Version="4.0.1" />
+    <PackageReference Include="Scriban" Version="4.0.2" />
   </ItemGroup>
 
 </Project>
diff --git a/src/FunderMaps.Core/MapBundle/Jobs/ExportJob.cs b/src/FunderMaps.Core/MapBundle/Jobs/ExportJob.cs
@@ -5,6 +5,8 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
 using System.IO;
+using System.Linq;
+using System;
 
 namespace FunderMaps.Core.MapBundle.Jobs
 {
@@ -37,53 +39,92 @@ public ExportJob(IConfiguration configuration, IMapService mapService)
         }
 
         /// <summary>
-        ///     Run the background command.
+        ///     Export dataset from database to mapservice.
         /// </summary>
-        /// <param name="context">Command task execution context.</param>
-        public override async Task ExecuteCommandAsync(CommandTaskContext context)
+        /// <remarks>
+        ///     The export is performed in 10 parts to that each individual part is
+        ///     less likely to fail the upload, and does not claim to much disk space.
+        ///     A tileset source can be composed of up to 10 source files so this number
+        ///     should not be increased.
+        /// </remarks>
+        /// <param name="layer">Layer name.</param>
+        /// <returns><c>True</c> if the export and upload succeeded, false otherwise.</returns>
+        private async Task<bool> ExportDatasetAsync(string layer)
         {
-            foreach (var layer in exportLayers)
+            const int features_per_part = 1000000;
+
+            for (int i = 0; i < 10; i++)
             {
-                await _mapService.DeleteDatasetAsync(layer);
+                FileDataSource fileDump = new()
+                {
+                    Format = GeometryFormat.GeoJSONSeq,
+                    PathPrefix = CreateDirectory("out"),
+                    Extension = ".json",
+                    Name = $"{layer}_part{i}",
+                };
+
+                CommandInfo command = new VectorDatasetBuilder()
+                    .InputDataset(new PostreSQLDataSource(connectionString))
+                    .InputLayers(new BundleLayerSource(layer, Context.Workspace, i * features_per_part, features_per_part))
+                    .OutputDataset(fileDump)
+                    .Build(formatName: "GeoJSONSeq");
 
-                for (int i = 0; i < 10; i++)
+                Logger.LogDebug($"Export layer: {layer}, part: {i}");
+
+                if (await RunCommandAsync(command) == 0)
                 {
-                    FileDataSource fileDump = new()
-                    {
-                        Format = GeometryFormat.GeoJSONSeq,
-                        PathPrefix = CreateDirectory("out"),
-                        Extension = ".json",
-                        Name = $"{layer}_part{i}",
-                    };
-
-                    CommandInfo command = new VectorDatasetBuilder()
-                        .InputDataset(new PostreSQLDataSource(connectionString))
-                        .InputLayers(new BundleLayerSource(layer, Context.Workspace, i * 1000000, 1000000))
-                        .OutputDataset(fileDump)
-                        .Build(formatName: "GeoJSONSeq");
-
-                    if (await RunCommandAsync(command) == 0)
+                    long length = new FileInfo(fileDump.ToString()).Length;
+
+                    if (length > 0)
                     {
-                        long length = new FileInfo(fileDump.ToString()).Length;
+                        bool success = await _mapService.UploadDatasetAsync(layer, fileDump.ToString());
 
-                        if (length > 0)
-                        {
-                            bool success = await _mapService.UploadDatasetAsync(layer, fileDump.ToString());
+                        Logger.LogDebug($"Upload layer: {layer}, part: {i}, success: {success}");
+                    }
 
-                            Logger.LogDebug($"Upload layer: {layer}, success: {success}");
-                        }
+                    File.Delete(fileDump.ToString());
 
-                        File.Delete(fileDump.ToString());
+                    if (length == 0)
+                    {
+                        return true;
                     }
                 }
-
-                if (await _mapService.PublishAsync(layer))
+                else
                 {
-                    Logger.LogInformation($"Layer {layer} published with success");
+                    Logger.LogError($"Export layer: {layer}, part: {i} failed");
+                    return false;
                 }
-                else
+            }
+
+            return true;
+        }
+
+        /// <summary>
+        ///     Run the background command.
+        /// </summary>
+        /// <remarks>
+        ///     Select the export layers in random order.
+        /// </remarks>
+        /// <param name="context">Command task execution context.</param>
+        public override async Task ExecuteCommandAsync(CommandTaskContext context)
+        {
+            var rng = new Random();
+
+            foreach (var layer in exportLayers.OrderBy(i => rng.Next()))
+            {
+                // NOTE: We *must* delete the old dataset first.
+                await _mapService.DeleteDatasetAsync(layer);
+
+                if (await ExportDatasetAsync(layer))
                 {
-                    Logger.LogError($"Layer {layer} was not published");
+                    if (await _mapService.PublishAsync(layer))
+                    {
+                        Logger.LogInformation($"Layer {layer} published with success");
+                    }
+                    else
+                    {
+                        Logger.LogError($"Layer {layer} was not published");
+                    }
                 }
             }
         }

diff --git a/src/FunderMaps.Core/MapBundle/PostreSQLDataSource.cs b/src/FunderMaps.Core/MapBundle/PostreSQLDataSource.cs
@@ -23,6 +23,10 @@ private string ParseCommandString(CommandInfo commandInfo)
                         commandInfo.Environment.Add("PGHOST", keyValue[1]);
                         break;
 
+                    case "port":
+                        commandInfo.Environment.Add("PGPORT", keyValue[1]);
+                        break;
+
                     case "user id":
                         commandInfo.Environment.Add("PGUSER", keyValue[1]);
                         break;

diff --git a/src/FunderMaps.Data/FunderMaps.Data.csproj b/src/FunderMaps.Data/FunderMaps.Data.csproj
@@ -11,7 +11,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Npgsql" Version="5.0.7" />
+    <PackageReference Include="Npgsql" Version="5.0.10" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/FunderMaps.Infrastructure/FunderMaps.Infrastructure.csproj b/src/FunderMaps.Infrastructure/FunderMaps.Infrastructure.csproj
@@ -12,8 +12,8 @@
 
   <ItemGroup>
     <PackageReference Include="System.Linq.Async" Version="5.0.0" />
-    <PackageReference Include="AWSSDK.S3" Version="3.7.1.14" />
-    <PackageReference Include="MailKit" Version="2.13.0" />
+    <PackageReference Include="AWSSDK.S3" Version="3.7.3.3" />
+    <PackageReference Include="MailKit" Version="2.15.0" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/FunderMaps.Webservice/FunderMaps.Webservice.csproj b/src/FunderMaps.Webservice/FunderMaps.Webservice.csproj
@@ -12,8 +12,8 @@
 
   <ItemGroup>
     <PackageReference Include="AutoMapper.Extensions.Microsoft.DependencyInjection" Version="8.1.1" />
-    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.1.4" />
-    <PackageReference Include="Swashbuckle.AspNetCore.Annotations" Version="6.1.4" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.2.2" />
+    <PackageReference Include="Swashbuckle.AspNetCore.Annotations" Version="6.2.2" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/tests/FunderMaps.IntegrationTests/FunderMaps.IntegrationTests.csproj b/tests/FunderMaps.IntegrationTests/FunderMaps.IntegrationTests.csproj
@@ -7,9 +7,9 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Bogus" Version="33.0.2" />
-    <PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="5.0.8" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.10.0" />
+    <PackageReference Include="Bogus" Version="33.1.1" />
+    <PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="5.0.10" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
     <PackageReference Include="System.Collections" Version="4.3.0" />
     <PackageReference Include="System.Net.Http.Json" Version="5.0.0" />
     <PackageReference Include="xunit" Version="2.4.1" />