Skip to content

Commit

Permalink
[DotLiquid] Add TraceInfo for HL7 v2 conversion (#123)
Browse files Browse the repository at this point in the history
* Add unused segments as the one in Handlebars

* Merge and solve conflict

* Add index in output and refine code

* Simplify output trace info

* Add end index and make them readonly

Co-authored-by: Qiwei Jin <qiwjin@microsoft.com>
  • Loading branch information
BoyaWu10 and qiwjin authored Dec 14, 2020
1 parent 483d847 commit b7d3d84
Show file tree
Hide file tree
Showing 27 changed files with 409 additions and 83 deletions.
3 changes: 3 additions & 0 deletions Fhir.Liquid.Converter.sln
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.TemplateManagement.FunctionalTests", "src\Microsoft.Health.Fhir.TemplateManagement.FunctionalTests\Microsoft.Health.Fhir.TemplateManagement.FunctionalTests.csproj", "{530D0D28-B6AC-4B2F-8B6A-DD77CAB537E6}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{FB441B10-F38D-40E7-B2B4-D8424DA1D595}"
ProjectSection(SolutionItems) = preProject
.editorconfig = .editorconfig
EndProjectSection
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.TemplateManagement.UnitTests", "src\Microsoft.Health.Fhir.TemplateManagement.UnitTests\Microsoft.Health.Fhir.TemplateManagement.UnitTests.csproj", "{C4049E7B-7977-4691-BE86-4BD7E3E8C401}"
EndProject
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
using DotLiquid;
using Microsoft.Health.Fhir.Liquid.Converter.Exceptions;
using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2;
using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.Models;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Xunit;
Expand Down Expand Up @@ -54,7 +55,8 @@ public void GivenHl7v2Message_WhenConverting_ExpectedFhirResourceShouldBeReturne

var inputContent = File.ReadAllText(inputFile);
var expectedContent = File.ReadAllText(expectedFile);
var actualContent = hl7v2Processor.Convert(inputContent, rootTemplate, new Hl7v2TemplateProvider(templateDirectory));
var traceInfo = new Hl7v2TraceInfo();
var actualContent = hl7v2Processor.Convert(inputContent, rootTemplate, new Hl7v2TemplateProvider(templateDirectory), traceInfo);

// Remove ID
var regex = new Regex(@"(?<=(""urn:uuid:|""|/))([A-Za-z0-9\-]{36})(?="")");
Expand All @@ -66,6 +68,7 @@ public void GivenHl7v2Message_WhenConverting_ExpectedFhirResourceShouldBeReturne
var expectedObject = serializer.Deserialize<JObject>(new JsonTextReader(new StringReader(expectedContent)));
var actualObject = serializer.Deserialize<JObject>(new JsonTextReader(new StringReader(actualContent)));
Assert.True(JToken.DeepEquals(expectedObject, actualObject));
Assert.True(traceInfo.UnusedSegments.Count > 0);
}

[Fact]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2;
using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.Models;
using Microsoft.Health.Fhir.Liquid.Converter.Models;
using Microsoft.Health.Fhir.Liquid.Converter.Tool.Models;
using Newtonsoft.Json;
Expand All @@ -18,89 +18,48 @@ namespace Microsoft.Health.Fhir.Liquid.Converter.Tool
internal static class ConverterLogicHandler
{
private const string MetadataFileName = "metadata.json";
private static readonly ILogger Logger;

static ConverterLogicHandler()
{
using var loggerFactory = LoggerFactory.Create(builder =>
{
builder.AddFilter("Microsoft.Health.Fhir.Liquid.Converter", LogLevel.Trace)
.AddConsole();
});
FhirConverterLogging.LoggerFactory = loggerFactory;
Logger = FhirConverterLogging.CreateLogger(typeof(ConverterLogicHandler));
}

internal static void Convert(ConverterOptions options)
{
if (!IsValidOptions(options))
{
Logger.LogError("Invalid command-line options.");
return;
throw new InputParameterException("Invalid command-line options.");
}

var dataType = GetDataTypes(options.TemplateDirectory);
var dataProcessor = CreateDataProcessor(dataType);
var templateProvider = CreateTemplateProvider(dataType, options.TemplateDirectory);

if (!string.IsNullOrEmpty(options.InputDataContent))
{
ConvertSingleFile(options);
ConvertSingleFile(dataProcessor, templateProvider, dataType, options.RootTemplate, options.InputDataContent, options.OutputDataFile, options.IsTraceInfo);
}
else
{
ConvertBatchFiles(options);
ConvertBatchFiles(dataProcessor, templateProvider, dataType, options.RootTemplate, options.InputDataFolder, options.OutputDataFolder, options.IsTraceInfo);
}

Console.WriteLine($"Conversion completed!");
}

private static void ConvertSingleFile(ConverterOptions options)
private static void ConvertSingleFile(IFhirConverter dataProcessor, ITemplateProvider templateProvider, DataType dataType, string rootTemplate, string inputContent, string outputFile, bool isTraceInfo)
{
try
{
var dataType = GetDataTypes(options.TemplateDirectory);
var dataProcessor = CreateDataProcessor(dataType);
var templateProvider = CreateTemplateProvider(dataType, options.TemplateDirectory);
var resultString = dataProcessor.Convert(options.InputDataContent, options.RootTemplate, templateProvider);
var result = new ConverterResult(ProcessStatus.OK, resultString);
WriteOutputFile(options.OutputDataFile, JsonConvert.SerializeObject(result, Formatting.Indented));
Logger.LogInformation("Process completed");
}
catch (Exception ex)
{
var error = new ConverterError(ex, options.TemplateDirectory);
WriteOutputFile(options.OutputDataFile, JsonConvert.SerializeObject(error, Formatting.Indented));
Logger.LogError($"Error occurred when converting input data: {error.ErrorMessage}");
}
var traceInfo = CreateTraceInfo(dataType, isTraceInfo);
var resultString = dataProcessor.Convert(inputContent, rootTemplate, templateProvider, traceInfo);
var result = new ConverterResult(ProcessStatus.OK, resultString, traceInfo);
SaveConverterResult(outputFile, result);
}

private static void ConvertBatchFiles(ConverterOptions options)
private static void ConvertBatchFiles(IFhirConverter dataProcessor, ITemplateProvider templateProvider, DataType dataType, string rootTemplate, string inputFolder, string outputFolder, bool isTraceInfo)
{
try
var files = GetInputFiles(dataType, inputFolder);
foreach (var file in files)
{
int succeededCount = 0;
int failedCount = 0;
var dataType = GetDataTypes(options.TemplateDirectory);
var dataProcessor = CreateDataProcessor(dataType);
var templateProvider = CreateTemplateProvider(dataType, options.TemplateDirectory);
var files = GetInputFiles(dataType, options.InputDataFolder);
foreach (var file in files)
{
try
{
var result = dataProcessor.Convert(File.ReadAllText(file), options.RootTemplate, templateProvider);
var outputFileDirectory = Path.Join(options.OutputDataFolder, Path.GetRelativePath(options.InputDataFolder, Path.GetDirectoryName(file)));
var outputFilePath = Path.Join(outputFileDirectory, Path.GetFileNameWithoutExtension(file) + ".json");
WriteOutputFile(outputFilePath, result);
succeededCount++;
}
catch (Exception ex)
{
Logger.LogError($"Error occurred when converting file {file}: {ex.Message}");
failedCount++;
}
}

Logger.LogInformation($"Process completed with {succeededCount} files succeeded and {failedCount} files failed");
}
catch (Exception ex)
{
Logger.LogError(ex.Message);
Console.WriteLine($"Processing {Path.GetFullPath(file)}");
var fileContent = File.ReadAllText(file);
var outputFileDirectory = Path.Join(outputFolder, Path.GetRelativePath(inputFolder, Path.GetDirectoryName(file)));
var outputFilePath = Path.Join(outputFileDirectory, Path.GetFileNameWithoutExtension(file) + ".json");
ConvertSingleFile(dataProcessor, templateProvider, dataType, rootTemplate, fileContent, outputFilePath, isTraceInfo);
}
}

Expand Down Expand Up @@ -151,6 +110,11 @@ private static ITemplateProvider CreateTemplateProvider(DataType dataType, strin
throw new NotImplementedException($"The conversion from data type {dataType} to FHIR is not supported");
}

private static TraceInfo CreateTraceInfo(DataType dataType, bool isTraceInfo)
{
return isTraceInfo ? (dataType == DataType.Hl7v2 ? new Hl7v2TraceInfo() : new TraceInfo()) : null;
}

private static List<string> GetInputFiles(DataType dataType, string inputDataFolder)
{
if (dataType == DataType.Hl7v2)
Expand All @@ -161,10 +125,12 @@ private static List<string> GetInputFiles(DataType dataType, string inputDataFol
return new List<string>();
}

private static void WriteOutputFile(string outputFilePath, string content)
private static void SaveConverterResult(string outputFilePath, ConverterResult result)
{
var outputFileDirectory = Path.GetDirectoryName(outputFilePath);
Directory.CreateDirectory(outputFileDirectory);

var content = JsonConvert.SerializeObject(result, Formatting.Indented, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore });
File.WriteAllText(outputFilePath, content);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,5 +27,8 @@ public class ConverterOptions

[Option('o', "OutputDataFolder", Required = false, HelpText = "Output data folder")]
public string OutputDataFolder { get; set; }

[Option('t', "IsTraceInfo", Required = false, HelpText = "Provide trace information in the output")]
public bool IsTraceInfo { get; set; }
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,24 @@
// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
// -------------------------------------------------------------------------------------------------

using Microsoft.Health.Fhir.Liquid.Converter.Models;
using Newtonsoft.Json.Linq;

namespace Microsoft.Health.Fhir.Liquid.Converter.Tool.Models
{
public class ConverterResult
{
public ConverterResult(ProcessStatus status, string fhirResource)
public ConverterResult(ProcessStatus status, string fhirResource, TraceInfo traceInfo)
{
Status = status;
FhirResource = new JRaw(fhirResource);
TraceInfo = traceInfo;
}

public ProcessStatus Status { get; set; }

public JRaw FhirResource { get; set; }

public TraceInfo TraceInfo { get; set; }
}
}
2 changes: 1 addition & 1 deletion src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -37,4 +37,4 @@ private static void HandleOptionsParseError(ParserResult<object> parseResult)
throw new InputParameterException(usageText);
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public void GivenValidEvaluateTemplateContent_WhenParseAndRender_CorrectResultSh
timeout: 0,
formatProvider: CultureInfo.InvariantCulture);

Assert.Empty(template.Render(new RenderParameters(CultureInfo.InvariantCulture) { Context = context }));
Assert.Empty(template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture)));
}

[Theory]
Expand All @@ -81,7 +81,7 @@ public void GivenInvalidSnippet_WhenRender_ExceptionsShouldBeThrown()
maxIterations: 0,
timeout: 0,
formatProvider: CultureInfo.InvariantCulture);
Assert.Throws<FileSystemException>(() => template.Render(new RenderParameters(CultureInfo.InvariantCulture) { Context = context }));
Assert.Throws<FileSystemException>(() => template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture)));

// Valid template file system but no such template
template = Template.Parse(@"{% evaluate bundleId using 'ID/Foo' Data: hl7v2Data -%}");
Expand All @@ -93,7 +93,7 @@ public void GivenInvalidSnippet_WhenRender_ExceptionsShouldBeThrown()
maxIterations: 0,
timeout: 0,
formatProvider: CultureInfo.InvariantCulture);
Assert.Throws<Exceptions.RenderException>(() => template.Render(new RenderParameters(CultureInfo.InvariantCulture) { Context = context }));
Assert.Throws<Exceptions.RenderException>(() => template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture)));
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
// -------------------------------------------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
// -------------------------------------------------------------------------------------------------

using System.Collections.Generic;
using Microsoft.Health.Fhir.Liquid.Converter.Extensions;
using Xunit;

namespace Microsoft.Health.Fhir.Liquid.Converter.UnitTests.Extensions
{
public class StringExtensionsTests
{
public static IEnumerable<object[]> GetIndexOfNthOccurrenceData()
{
// Null or empty string or char
yield return new object[] { null, null, 1, -1 };
yield return new object[] { string.Empty, null, 1, -1 };
yield return new object[] { "abc|abc", null, 1, -1 };

// Nth occurrence smaller than one
yield return new object[] { "abc|abc", '|', 0, -1 };
yield return new object[] { "abc|abc", '|', -1, -1 };

// Nth occurrence hit
yield return new object[] { "abc|abc", '|', 1, 3 };
yield return new object[] { "abc|abc|abc", '|', 2, 7 };

// Nth occurrence not hit
yield return new object[] { "abc|abc", '|', 3, -1 };
yield return new object[] { "abc|abc", '^', 1, -1 };
}

[Theory]
[MemberData(nameof(GetIndexOfNthOccurrenceData))]
public void IndexOfNthOccurrenceTests(string s, char c, int n, int expected)
{
Assert.Equal(expected, s.IndexOfNthOccurrence(c, n));
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public void FiltersRenderingTest()
var context = new Context(CultureInfo.InvariantCulture);
context.AddFilters(typeof(Filters));

var actual = template.Render(new RenderParameters(CultureInfo.InvariantCulture) { Context = context });
var actual = template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture));
Assert.Equal(Expected, actual);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public void GivenValidHl7v2Message_WhenParse_CorrectHl7v2DataShouldBeReturned()
// Hl7v2Field tests
var field = (Hl7v2Field)segment[5];
Assert.Equal("(130) 724-0433^PRN^PH^^^431^2780404~(330) 274-8214^ORN^PH^^^330^2748214", field["Value"]);
Assert.Equal(2, ((List<Hl7v2Field>)field["Repeats"]).Count);
Assert.Equal(2, ((SafeList<Hl7v2Field>)field["Repeats"]).Count);
Assert.Equal("(130) 724-0433", ((Hl7v2Component)field[1]).Value);
Assert.Throws<RenderException>(() => (Hl7v2Component)field[null]);
Assert.Throws<RenderException>(() => (Hl7v2Component)field[1.2]);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
// -------------------------------------------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
// -------------------------------------------------------------------------------------------------

using System.Collections.Generic;
using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2;
using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.Models;
using Xunit;

namespace Microsoft.Health.Fhir.Liquid.Converter.UnitTests.Hl7v2.Models
{
public class Hl7v2TraceInfoTests
{
[Fact]
public void GivenHl7v2Data_WhenCreate_CorrectHl7v2TraceInfoShouldBeReturned()
{
// Null Hl7v2Data
var traceInfo = Hl7v2TraceInfo.CreateTraceInfo(null);
Assert.Empty(traceInfo.UnusedSegments);

// Empty Hl7v2Data
var data = new Hl7v2Data();
traceInfo = Hl7v2TraceInfo.CreateTraceInfo(data);
Assert.Empty(traceInfo.UnusedSegments);

// Null data
data = new Hl7v2Data()
{
Meta = null,
Data = null,
};
traceInfo = Hl7v2TraceInfo.CreateTraceInfo(data);
Assert.Empty(traceInfo.UnusedSegments);

// Null segment
data = new Hl7v2Data()
{
Meta = new List<string>() { null },
Data = new List<Hl7v2Segment>() { null },
};
traceInfo = Hl7v2TraceInfo.CreateTraceInfo(data);
Assert.Empty(traceInfo.UnusedSegments);

// Valid Hl7v2Data before render
var content = @"MSH|^~\&|AccMgr|1|||20050110045504||ADT^A01|599102|P|2.3|||
PID|1||10006579^^^1^MR^1||DUCK^DONALD^D||19241010|M||1|111 DUCK ST^^FOWL^CA^999990000^^M|1|8885551212|8885551212|1|2||40007716^^^AccMgr^VN^1|123121234|||||||||||NO ";
var parser = new Hl7v2DataParser();
data = parser.Parse(content);
traceInfo = Hl7v2TraceInfo.CreateTraceInfo(data);
Assert.Equal(2, traceInfo.UnusedSegments.Count);
Assert.Equal(27, traceInfo.UnusedSegments[1].Components.Count);

// Valid Hl7v2Data after render
var processor = new Hl7v2Processor();
var templateProvider = new Hl7v2TemplateProvider(Constants.Hl7v2TemplateDirectory);
_ = processor.Convert(content, "ADT_A01", templateProvider, traceInfo);
Assert.Equal(2, traceInfo.UnusedSegments.Count);

var unusedPid = traceInfo.UnusedSegments[1];
Assert.Equal("PID", unusedPid.Type);
Assert.Equal(1, unusedPid.Line);
Assert.Equal(6, unusedPid.Components.Count);
Assert.Equal(118, unusedPid.Components[2].Start);
Assert.Equal(126, unusedPid.Components[2].End);
Assert.Equal("40007716", unusedPid.Components[2].Value);
}
}
}
Loading

0 comments on commit b7d3d84

Please sign in to comment.