From b7d3d84fd3730383963cd2a0597d4d85edf2b1fa Mon Sep 17 00:00:00 2001 From: Boya Wu <38548227+BoyaWu10@users.noreply.github.com> Date: Mon, 14 Dec 2020 14:13:15 +0800 Subject: [PATCH] [DotLiquid] Add TraceInfo for HL7 v2 conversion (#123) * Add unused segments as the one in Handlebars * Merge and solve conflict * Add index in output and refine code * Simplify output trace info * Add end index and make them readonly Co-authored-by: Qiwei Jin --- Fhir.Liquid.Converter.sln | 3 + .../FunctionalTests.cs | 5 +- .../ConverterLogicHandler.cs | 96 ++++++------------ .../Models/ConverterOptions.cs | 3 + .../Models/ConverterResult.cs | 6 +- .../Program.cs | 2 +- .../DotLiquids/EvaluateTests.cs | 6 +- .../Extensions/StringExtensionsTests.cs | 41 ++++++++ .../Filters/FiltersRenderingTests.cs | 2 +- .../Hl7v2/Hl7v2DataParserTests.cs | 2 +- .../Hl7v2/Models/Hl7v2TraceInfoTests.cs | 69 +++++++++++++ .../DotLiquids/SafeList.cs | 10 +- .../Extensions/StringExtensions.cs | 28 ++++++ .../Filters/GeneralFilters.cs | 2 +- .../Hl7v2/Hl7v2DataParser.cs | 1 + .../Hl7v2/Hl7v2Processor.cs | 14 ++- .../Hl7v2/Models/Hl7v2Component.cs | 4 + .../Hl7v2/Models/Hl7v2Data.cs | 2 + .../Hl7v2/Models/Hl7v2Field.cs | 17 +++- .../Hl7v2/Models/Hl7v2Segment.cs | 18 ++++ .../Hl7v2/Models/Hl7v2TraceInfo.cs | 98 +++++++++++++++++++ .../Hl7v2/Models/UnusedHl7v2Component.cs | 23 +++++ .../Hl7v2/Models/UnusedHl7v2Segment.cs | 25 +++++ .../Models/FhirConverterErrorCode.cs | 1 + .../Models/TraceInfo.cs | 2 +- .../Resources.Designer.cs | 9 ++ .../Resources.resx | 3 + 27 files changed, 409 insertions(+), 83 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Extensions/StringExtensionsTests.cs create mode 100644 src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Models/Hl7v2TraceInfoTests.cs create mode 100644 src/Microsoft.Health.Fhir.Liquid.Converter/Extensions/StringExtensions.cs create mode 100644 src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2TraceInfo.cs create mode 100644 src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/UnusedHl7v2Component.cs create mode 100644 src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/UnusedHl7v2Segment.cs diff --git a/Fhir.Liquid.Converter.sln b/Fhir.Liquid.Converter.sln index bd3b14191..6d373eb76 100644 --- a/Fhir.Liquid.Converter.sln +++ b/Fhir.Liquid.Converter.sln @@ -16,6 +16,9 @@ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.TemplateManagement.FunctionalTests", "src\Microsoft.Health.Fhir.TemplateManagement.FunctionalTests\Microsoft.Health.Fhir.TemplateManagement.FunctionalTests.csproj", "{530D0D28-B6AC-4B2F-8B6A-DD77CAB537E6}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{FB441B10-F38D-40E7-B2B4-D8424DA1D595}" + ProjectSection(SolutionItems) = preProject + .editorconfig = .editorconfig + EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.TemplateManagement.UnitTests", "src\Microsoft.Health.Fhir.TemplateManagement.UnitTests\Microsoft.Health.Fhir.TemplateManagement.UnitTests.csproj", "{C4049E7B-7977-4691-BE86-4BD7E3E8C401}" EndProject diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/FunctionalTests.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/FunctionalTests.cs index 49b817924..1e21aa6ea 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/FunctionalTests.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/FunctionalTests.cs @@ -11,6 +11,7 @@ using DotLiquid; using Microsoft.Health.Fhir.Liquid.Converter.Exceptions; using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2; +using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.Models; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Xunit; @@ -54,7 +55,8 @@ public void GivenHl7v2Message_WhenConverting_ExpectedFhirResourceShouldBeReturne var inputContent = File.ReadAllText(inputFile); var expectedContent = File.ReadAllText(expectedFile); - var actualContent = hl7v2Processor.Convert(inputContent, rootTemplate, new Hl7v2TemplateProvider(templateDirectory)); + var traceInfo = new Hl7v2TraceInfo(); + var actualContent = hl7v2Processor.Convert(inputContent, rootTemplate, new Hl7v2TemplateProvider(templateDirectory), traceInfo); // Remove ID var regex = new Regex(@"(?<=(""urn:uuid:|""|/))([A-Za-z0-9\-]{36})(?="")"); @@ -66,6 +68,7 @@ public void GivenHl7v2Message_WhenConverting_ExpectedFhirResourceShouldBeReturne var expectedObject = serializer.Deserialize(new JsonTextReader(new StringReader(expectedContent))); var actualObject = serializer.Deserialize(new JsonTextReader(new StringReader(actualContent))); Assert.True(JToken.DeepEquals(expectedObject, actualObject)); + Assert.True(traceInfo.UnusedSegments.Count > 0); } [Fact] diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/ConverterLogicHandler.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/ConverterLogicHandler.cs index 5473560d9..b400d33e0 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/ConverterLogicHandler.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/ConverterLogicHandler.cs @@ -7,8 +7,8 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using Microsoft.Extensions.Logging; using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2; +using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.Models; using Microsoft.Health.Fhir.Liquid.Converter.Models; using Microsoft.Health.Fhir.Liquid.Converter.Tool.Models; using Newtonsoft.Json; @@ -18,89 +18,48 @@ namespace Microsoft.Health.Fhir.Liquid.Converter.Tool internal static class ConverterLogicHandler { private const string MetadataFileName = "metadata.json"; - private static readonly ILogger Logger; - - static ConverterLogicHandler() - { - using var loggerFactory = LoggerFactory.Create(builder => - { - builder.AddFilter("Microsoft.Health.Fhir.Liquid.Converter", LogLevel.Trace) - .AddConsole(); - }); - FhirConverterLogging.LoggerFactory = loggerFactory; - Logger = FhirConverterLogging.CreateLogger(typeof(ConverterLogicHandler)); - } internal static void Convert(ConverterOptions options) { if (!IsValidOptions(options)) { - Logger.LogError("Invalid command-line options."); - return; + throw new InputParameterException("Invalid command-line options."); } + var dataType = GetDataTypes(options.TemplateDirectory); + var dataProcessor = CreateDataProcessor(dataType); + var templateProvider = CreateTemplateProvider(dataType, options.TemplateDirectory); + if (!string.IsNullOrEmpty(options.InputDataContent)) { - ConvertSingleFile(options); + ConvertSingleFile(dataProcessor, templateProvider, dataType, options.RootTemplate, options.InputDataContent, options.OutputDataFile, options.IsTraceInfo); } else { - ConvertBatchFiles(options); + ConvertBatchFiles(dataProcessor, templateProvider, dataType, options.RootTemplate, options.InputDataFolder, options.OutputDataFolder, options.IsTraceInfo); } + + Console.WriteLine($"Conversion completed!"); } - private static void ConvertSingleFile(ConverterOptions options) + private static void ConvertSingleFile(IFhirConverter dataProcessor, ITemplateProvider templateProvider, DataType dataType, string rootTemplate, string inputContent, string outputFile, bool isTraceInfo) { - try - { - var dataType = GetDataTypes(options.TemplateDirectory); - var dataProcessor = CreateDataProcessor(dataType); - var templateProvider = CreateTemplateProvider(dataType, options.TemplateDirectory); - var resultString = dataProcessor.Convert(options.InputDataContent, options.RootTemplate, templateProvider); - var result = new ConverterResult(ProcessStatus.OK, resultString); - WriteOutputFile(options.OutputDataFile, JsonConvert.SerializeObject(result, Formatting.Indented)); - Logger.LogInformation("Process completed"); - } - catch (Exception ex) - { - var error = new ConverterError(ex, options.TemplateDirectory); - WriteOutputFile(options.OutputDataFile, JsonConvert.SerializeObject(error, Formatting.Indented)); - Logger.LogError($"Error occurred when converting input data: {error.ErrorMessage}"); - } + var traceInfo = CreateTraceInfo(dataType, isTraceInfo); + var resultString = dataProcessor.Convert(inputContent, rootTemplate, templateProvider, traceInfo); + var result = new ConverterResult(ProcessStatus.OK, resultString, traceInfo); + SaveConverterResult(outputFile, result); } - private static void ConvertBatchFiles(ConverterOptions options) + private static void ConvertBatchFiles(IFhirConverter dataProcessor, ITemplateProvider templateProvider, DataType dataType, string rootTemplate, string inputFolder, string outputFolder, bool isTraceInfo) { - try + var files = GetInputFiles(dataType, inputFolder); + foreach (var file in files) { - int succeededCount = 0; - int failedCount = 0; - var dataType = GetDataTypes(options.TemplateDirectory); - var dataProcessor = CreateDataProcessor(dataType); - var templateProvider = CreateTemplateProvider(dataType, options.TemplateDirectory); - var files = GetInputFiles(dataType, options.InputDataFolder); - foreach (var file in files) - { - try - { - var result = dataProcessor.Convert(File.ReadAllText(file), options.RootTemplate, templateProvider); - var outputFileDirectory = Path.Join(options.OutputDataFolder, Path.GetRelativePath(options.InputDataFolder, Path.GetDirectoryName(file))); - var outputFilePath = Path.Join(outputFileDirectory, Path.GetFileNameWithoutExtension(file) + ".json"); - WriteOutputFile(outputFilePath, result); - succeededCount++; - } - catch (Exception ex) - { - Logger.LogError($"Error occurred when converting file {file}: {ex.Message}"); - failedCount++; - } - } - - Logger.LogInformation($"Process completed with {succeededCount} files succeeded and {failedCount} files failed"); - } - catch (Exception ex) - { - Logger.LogError(ex.Message); + Console.WriteLine($"Processing {Path.GetFullPath(file)}"); + var fileContent = File.ReadAllText(file); + var outputFileDirectory = Path.Join(outputFolder, Path.GetRelativePath(inputFolder, Path.GetDirectoryName(file))); + var outputFilePath = Path.Join(outputFileDirectory, Path.GetFileNameWithoutExtension(file) + ".json"); + ConvertSingleFile(dataProcessor, templateProvider, dataType, rootTemplate, fileContent, outputFilePath, isTraceInfo); } } @@ -151,6 +110,11 @@ private static ITemplateProvider CreateTemplateProvider(DataType dataType, strin throw new NotImplementedException($"The conversion from data type {dataType} to FHIR is not supported"); } + private static TraceInfo CreateTraceInfo(DataType dataType, bool isTraceInfo) + { + return isTraceInfo ? (dataType == DataType.Hl7v2 ? new Hl7v2TraceInfo() : new TraceInfo()) : null; + } + private static List GetInputFiles(DataType dataType, string inputDataFolder) { if (dataType == DataType.Hl7v2) @@ -161,10 +125,12 @@ private static List GetInputFiles(DataType dataType, string inputDataFol return new List(); } - private static void WriteOutputFile(string outputFilePath, string content) + private static void SaveConverterResult(string outputFilePath, ConverterResult result) { var outputFileDirectory = Path.GetDirectoryName(outputFilePath); Directory.CreateDirectory(outputFileDirectory); + + var content = JsonConvert.SerializeObject(result, Formatting.Indented, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); File.WriteAllText(outputFilePath, content); } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/ConverterOptions.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/ConverterOptions.cs index 5186d73ea..1f2f7c549 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/ConverterOptions.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/ConverterOptions.cs @@ -27,5 +27,8 @@ public class ConverterOptions [Option('o', "OutputDataFolder", Required = false, HelpText = "Output data folder")] public string OutputDataFolder { get; set; } + + [Option('t', "IsTraceInfo", Required = false, HelpText = "Provide trace information in the output")] + public bool IsTraceInfo { get; set; } } } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/ConverterResult.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/ConverterResult.cs index d1ae8f8dc..2ea653153 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/ConverterResult.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/ConverterResult.cs @@ -3,20 +3,24 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using Microsoft.Health.Fhir.Liquid.Converter.Models; using Newtonsoft.Json.Linq; namespace Microsoft.Health.Fhir.Liquid.Converter.Tool.Models { public class ConverterResult { - public ConverterResult(ProcessStatus status, string fhirResource) + public ConverterResult(ProcessStatus status, string fhirResource, TraceInfo traceInfo) { Status = status; FhirResource = new JRaw(fhirResource); + TraceInfo = traceInfo; } public ProcessStatus Status { get; set; } public JRaw FhirResource { get; set; } + + public TraceInfo TraceInfo { get; set; } } } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Program.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Program.cs index db39df56e..c318f82c1 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Program.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Program.cs @@ -37,4 +37,4 @@ private static void HandleOptionsParseError(ParserResult parseResult) throw new InputParameterException(usageText); } } -} +} \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/EvaluateTests.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/EvaluateTests.cs index f78422e0f..cafb33701 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/EvaluateTests.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/EvaluateTests.cs @@ -55,7 +55,7 @@ public void GivenValidEvaluateTemplateContent_WhenParseAndRender_CorrectResultSh timeout: 0, formatProvider: CultureInfo.InvariantCulture); - Assert.Empty(template.Render(new RenderParameters(CultureInfo.InvariantCulture) { Context = context })); + Assert.Empty(template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture))); } [Theory] @@ -81,7 +81,7 @@ public void GivenInvalidSnippet_WhenRender_ExceptionsShouldBeThrown() maxIterations: 0, timeout: 0, formatProvider: CultureInfo.InvariantCulture); - Assert.Throws(() => template.Render(new RenderParameters(CultureInfo.InvariantCulture) { Context = context })); + Assert.Throws(() => template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture))); // Valid template file system but no such template template = Template.Parse(@"{% evaluate bundleId using 'ID/Foo' Data: hl7v2Data -%}"); @@ -93,7 +93,7 @@ public void GivenInvalidSnippet_WhenRender_ExceptionsShouldBeThrown() maxIterations: 0, timeout: 0, formatProvider: CultureInfo.InvariantCulture); - Assert.Throws(() => template.Render(new RenderParameters(CultureInfo.InvariantCulture) { Context = context })); + Assert.Throws(() => template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture))); } } } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Extensions/StringExtensionsTests.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Extensions/StringExtensionsTests.cs new file mode 100644 index 000000000..5a2e22eb0 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Extensions/StringExtensionsTests.cs @@ -0,0 +1,41 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Microsoft.Health.Fhir.Liquid.Converter.Extensions; +using Xunit; + +namespace Microsoft.Health.Fhir.Liquid.Converter.UnitTests.Extensions +{ + public class StringExtensionsTests + { + public static IEnumerable GetIndexOfNthOccurrenceData() + { + // Null or empty string or char + yield return new object[] { null, null, 1, -1 }; + yield return new object[] { string.Empty, null, 1, -1 }; + yield return new object[] { "abc|abc", null, 1, -1 }; + + // Nth occurrence smaller than one + yield return new object[] { "abc|abc", '|', 0, -1 }; + yield return new object[] { "abc|abc", '|', -1, -1 }; + + // Nth occurrence hit + yield return new object[] { "abc|abc", '|', 1, 3 }; + yield return new object[] { "abc|abc|abc", '|', 2, 7 }; + + // Nth occurrence not hit + yield return new object[] { "abc|abc", '|', 3, -1 }; + yield return new object[] { "abc|abc", '^', 1, -1 }; + } + + [Theory] + [MemberData(nameof(GetIndexOfNthOccurrenceData))] + public void IndexOfNthOccurrenceTests(string s, char c, int n, int expected) + { + Assert.Equal(expected, s.IndexOfNthOccurrence(c, n)); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Filters/FiltersRenderingTests.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Filters/FiltersRenderingTests.cs index 0e07836a7..6e6045570 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Filters/FiltersRenderingTests.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Filters/FiltersRenderingTests.cs @@ -42,7 +42,7 @@ public void FiltersRenderingTest() var context = new Context(CultureInfo.InvariantCulture); context.AddFilters(typeof(Filters)); - var actual = template.Render(new RenderParameters(CultureInfo.InvariantCulture) { Context = context }); + var actual = template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture)); Assert.Equal(Expected, actual); } } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Hl7v2DataParserTests.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Hl7v2DataParserTests.cs index a1f12dbfb..69f0bc1eb 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Hl7v2DataParserTests.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Hl7v2DataParserTests.cs @@ -66,7 +66,7 @@ public void GivenValidHl7v2Message_WhenParse_CorrectHl7v2DataShouldBeReturned() // Hl7v2Field tests var field = (Hl7v2Field)segment[5]; Assert.Equal("(130) 724-0433^PRN^PH^^^431^2780404~(330) 274-8214^ORN^PH^^^330^2748214", field["Value"]); - Assert.Equal(2, ((List)field["Repeats"]).Count); + Assert.Equal(2, ((SafeList)field["Repeats"]).Count); Assert.Equal("(130) 724-0433", ((Hl7v2Component)field[1]).Value); Assert.Throws(() => (Hl7v2Component)field[null]); Assert.Throws(() => (Hl7v2Component)field[1.2]); diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Models/Hl7v2TraceInfoTests.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Models/Hl7v2TraceInfoTests.cs new file mode 100644 index 000000000..5875765bc --- /dev/null +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Models/Hl7v2TraceInfoTests.cs @@ -0,0 +1,69 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2; +using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.Models; +using Xunit; + +namespace Microsoft.Health.Fhir.Liquid.Converter.UnitTests.Hl7v2.Models +{ + public class Hl7v2TraceInfoTests + { + [Fact] + public void GivenHl7v2Data_WhenCreate_CorrectHl7v2TraceInfoShouldBeReturned() + { + // Null Hl7v2Data + var traceInfo = Hl7v2TraceInfo.CreateTraceInfo(null); + Assert.Empty(traceInfo.UnusedSegments); + + // Empty Hl7v2Data + var data = new Hl7v2Data(); + traceInfo = Hl7v2TraceInfo.CreateTraceInfo(data); + Assert.Empty(traceInfo.UnusedSegments); + + // Null data + data = new Hl7v2Data() + { + Meta = null, + Data = null, + }; + traceInfo = Hl7v2TraceInfo.CreateTraceInfo(data); + Assert.Empty(traceInfo.UnusedSegments); + + // Null segment + data = new Hl7v2Data() + { + Meta = new List() { null }, + Data = new List() { null }, + }; + traceInfo = Hl7v2TraceInfo.CreateTraceInfo(data); + Assert.Empty(traceInfo.UnusedSegments); + + // Valid Hl7v2Data before render + var content = @"MSH|^~\&|AccMgr|1|||20050110045504||ADT^A01|599102|P|2.3||| +PID|1||10006579^^^1^MR^1||DUCK^DONALD^D||19241010|M||1|111 DUCK ST^^FOWL^CA^999990000^^M|1|8885551212|8885551212|1|2||40007716^^^AccMgr^VN^1|123121234|||||||||||NO "; + var parser = new Hl7v2DataParser(); + data = parser.Parse(content); + traceInfo = Hl7v2TraceInfo.CreateTraceInfo(data); + Assert.Equal(2, traceInfo.UnusedSegments.Count); + Assert.Equal(27, traceInfo.UnusedSegments[1].Components.Count); + + // Valid Hl7v2Data after render + var processor = new Hl7v2Processor(); + var templateProvider = new Hl7v2TemplateProvider(Constants.Hl7v2TemplateDirectory); + _ = processor.Convert(content, "ADT_A01", templateProvider, traceInfo); + Assert.Equal(2, traceInfo.UnusedSegments.Count); + + var unusedPid = traceInfo.UnusedSegments[1]; + Assert.Equal("PID", unusedPid.Type); + Assert.Equal(1, unusedPid.Line); + Assert.Equal(6, unusedPid.Components.Count); + Assert.Equal(118, unusedPid.Components[2].Start); + Assert.Equal(126, unusedPid.Components[2].End); + Assert.Equal("40007716", unusedPid.Components[2].Value); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/DotLiquids/SafeList.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/DotLiquids/SafeList.cs index f3b795a24..e9127a9c5 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/DotLiquids/SafeList.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/DotLiquids/SafeList.cs @@ -3,13 +3,14 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System.Collections; using System.Collections.Generic; using System.Linq; using DotLiquid; namespace Microsoft.Health.Fhir.Liquid.Converter.DotLiquids { - public class SafeList : IIndexable, ILiquidizable + public class SafeList : IEnumerable, IIndexable, ILiquidizable where T : class { private IList _internalList; @@ -54,11 +55,16 @@ public virtual bool ContainsKey(object key) return key is int index && index >= 0 && index < _internalList.Count; } - public void Add(T item) + public virtual void Add(T item) { _internalList.Add(item); } + public virtual IEnumerator GetEnumerator() + { + return _internalList.GetEnumerator(); + } + public virtual object ToLiquid() { return this; diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Extensions/StringExtensions.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Extensions/StringExtensions.cs new file mode 100644 index 000000000..b218ea9a4 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Extensions/StringExtensions.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Linq; + +namespace Microsoft.Health.Fhir.Liquid.Converter.Extensions +{ + public static class StringExtensions + { + public static int IndexOfNthOccurrence(this string s, char c, int n) + { + if (n <= 0) + { + return -1; + } + + var result = s? + .Select((c, i) => new { Char = c, Index = i }) + .Where(item => item.Char == c) + .Skip(n - 1) + .FirstOrDefault(); + + return result != null ? result.Index : -1; + } + } +} diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Filters/GeneralFilters.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Filters/GeneralFilters.cs index 0baefffaa..8db519a0d 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Filters/GeneralFilters.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Filters/GeneralFilters.cs @@ -26,7 +26,7 @@ public static string GetProperty(Context context, string originalCode, string ma return null; } - var map = ((CodeSystemMapping)context["CodeSystemMapping"])?.Mapping; + var map = (context["CodeSystemMapping"] as CodeSystemMapping)?.Mapping; return map != null && map.ContainsKey(mapping) && map[mapping].ContainsKey(originalCode) && diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2DataParser.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2DataParser.cs index 417b1a356..c36e195ec 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2DataParser.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2DataParser.cs @@ -31,6 +31,7 @@ public Hl7v2Data Parse(string message) var segments = message.Split(SegmentSeparators, StringSplitOptions.RemoveEmptyEntries); _validator.ValidateMessageHeader(segments[0]); var encodingCharacters = ParseHl7v2EncodingCharacters(segments[0]); + result.EncodingCharacters = encodingCharacters; for (var i = 0; i < segments.Length; ++i) { diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2Processor.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2Processor.cs index 6ecc7d1c6..2f154a0dd 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2Processor.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2Processor.cs @@ -10,6 +10,7 @@ using System.Threading; using DotLiquid; using Microsoft.Health.Fhir.Liquid.Converter.Exceptions; +using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.Models; using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.OutputProcessor; using Microsoft.Health.Fhir.Liquid.Converter.Models; using Newtonsoft.Json; @@ -44,9 +45,15 @@ public string Convert(string data, string rootTemplate, ITemplateProvider templa throw new RenderException(FhirConverterErrorCode.TemplateNotFound, string.Format(Resources.TemplateNotFound, rootTemplate)); } - var context = CreateContext(templateProvider, data); + var hl7v2Data = _dataParser.Parse(data); + var context = CreateContext(templateProvider, hl7v2Data); var rawResult = RenderTemplates(template, context); var result = PostProcessor.Process(rawResult); + if (traceInfo is Hl7v2TraceInfo hl7V2TraceInfo) + { + hl7V2TraceInfo.UnusedSegments = Hl7v2TraceInfo.CreateTraceInfo(hl7v2Data).UnusedSegments; + } + return result.ToString(Formatting.Indented); } @@ -56,10 +63,9 @@ public string Convert(string data, string rootTemplate, ITemplateProvider templa return Convert(data, rootTemplate, templateProvider, traceInfo); } - private Context CreateContext(ITemplateProvider templateProvider, string data) + private Context CreateContext(ITemplateProvider templateProvider, Hl7v2Data hl7v2Data) { // Load data and templates - var hl7v2Data = _dataParser.Parse(data); var timeout = _settings != null ? _settings.TimeOut : 0; var context = new Context( environments: new List() { Hash.FromAnonymousObject(new { hl7v2Data }) }, @@ -88,7 +94,7 @@ private string RenderTemplates(Template template, Context context) try { template.MakeThreadSafe(); - return template.Render(new RenderParameters(CultureInfo.InvariantCulture) { Context = context }); + return template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture)); } catch (TimeoutException ex) { diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2Component.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2Component.cs index 903e088cf..181ccbbd5 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2Component.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2Component.cs @@ -16,10 +16,13 @@ public class Hl7v2Component : Drop { public Hl7v2Component(string value, IEnumerable subcomponents) { + IsAccessed = false; Value = value; Subcomponents = new SafeList(subcomponents); } + public bool IsAccessed { get; set; } + public string Value { get; set; } public SafeList Subcomponents { get; set; } @@ -33,6 +36,7 @@ public override object this[object index] throw new RenderException(FhirConverterErrorCode.PropertyNotFound, string.Format(Resources.PropertyNotFound, index, this.GetType().Name)); } + IsAccessed = true; var indexString = index.ToString(); if (string.Equals(indexString, "Value", StringComparison.InvariantCultureIgnoreCase)) { diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2Data.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2Data.cs index efdbe5430..f83e6a227 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2Data.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2Data.cs @@ -19,6 +19,8 @@ public Hl7v2Data(string value = null) public string Value { get; set; } + public Hl7v2EncodingCharacters EncodingCharacters { get; set; } + public List Meta { get; set; } public List Data { get; set; } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2Field.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2Field.cs index ad0fc9740..397cf69d2 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2Field.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2Field.cs @@ -18,14 +18,14 @@ public Hl7v2Field(string value, IEnumerable components) { Value = value; Components = new SafeList(components); - Repeats = new List(); + Repeats = new SafeList(); } public string Value { get; set; } public SafeList Components { get; set; } - public List Repeats { get; set; } + public SafeList Repeats { get; set; } public override object this[object index] { @@ -39,6 +39,7 @@ public override object this[object index] var indexString = index.ToString(); if (string.Equals(indexString, "Value", StringComparison.InvariantCultureIgnoreCase)) { + SetAccessForAllComponents(); return Value; } else if (string.Equals(indexString, "Components", StringComparison.InvariantCultureIgnoreCase)) @@ -47,6 +48,7 @@ public override object this[object index] } else if (string.Equals(indexString, "Repeats", StringComparison.InvariantCultureIgnoreCase)) { + SetAccessForAllComponents(); return Repeats; } else if (int.TryParse(indexString, out int result)) @@ -59,5 +61,16 @@ public override object this[object index] } } } + + private void SetAccessForAllComponents() + { + foreach (var component in Components) + { + if (component is Hl7v2Component hl7V2Component) + { + hl7V2Component.IsAccessed = true; + } + } + } } } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2Segment.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2Segment.cs index 5c10da4e1..d0e330065 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2Segment.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2Segment.cs @@ -36,6 +36,7 @@ public override object this[object index] var indexString = index.ToString(); if (string.Equals(indexString, "Value", StringComparison.InvariantCultureIgnoreCase)) { + SetAccessForAllComponents(); return Value; } else if (string.Equals(indexString, "Fields", StringComparison.InvariantCultureIgnoreCase)) @@ -52,5 +53,22 @@ public override object this[object index] } } } + + private void SetAccessForAllComponents() + { + foreach (var field in Fields) + { + if (field is Hl7v2Field hl7v2Field) + { + foreach (var component in hl7v2Field.Components) + { + if (component is Hl7v2Component hl7V2Component) + { + hl7V2Component.IsAccessed = true; + } + } + } + } + } } } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2TraceInfo.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2TraceInfo.cs new file mode 100644 index 000000000..0f22f3b81 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2TraceInfo.cs @@ -0,0 +1,98 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Microsoft.Health.Fhir.Liquid.Converter.Exceptions; +using Microsoft.Health.Fhir.Liquid.Converter.Extensions; +using Microsoft.Health.Fhir.Liquid.Converter.Models; + +namespace Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.Models +{ + public class Hl7v2TraceInfo : TraceInfo + { + public Hl7v2TraceInfo() + { + } + + public Hl7v2TraceInfo(List unusedHl7V2Segments) + { + UnusedSegments = unusedHl7V2Segments; + } + + public List UnusedSegments { get; set; } + + public static Hl7v2TraceInfo CreateTraceInfo(Hl7v2Data hl7v2Data) + { + var unusedSegments = new List(); + try + { + for (var i = 0; i < hl7v2Data?.Data?.Count; ++i) + { + var segment = hl7v2Data.Data[i]; + var unusedSegment = new UnusedHl7v2Segment(i); + for (var j = 0; j < segment?.Fields?.Count; ++j) + { + // Encoding characters field is treated as accessed + if (i == 0 && j == 1) + { + continue; + } + + // Segment id field is treated as accessed + if (j == 0 && segment.Fields[j] is Hl7v2Field segmentIdField) + { + unusedSegment.Type = segmentIdField.Value; + continue; + } + + if (j > 0 && segment.Fields[j] is Hl7v2Field field) + { + var unusedComponents = new List(); + for (var k = 0; k < field.Components.Count; ++k) + { + if (field.Components[k] is Hl7v2Component component && component.IsAccessed == false) + { + var indexInSegment = FindOffsetInSegment(segment.Value, hl7v2Data.EncodingCharacters, j, k - 1); + var unusedComponent = new UnusedHl7v2Component(indexInSegment, indexInSegment + component.Value.Length, component.Value); + unusedComponents.Add(unusedComponent); + } + } + + if (unusedComponents.Count > 0) + { + unusedSegment.Components.AddRange(unusedComponents); + } + } + } + + if (unusedSegment.Components.Count > 0) + { + unusedSegments.Add(unusedSegment); + } + } + } + catch (Exception ex) + { + throw new PostprocessException(FhirConverterErrorCode.TraceInfoError, string.Format(Resources.TraceInfoError, ex.Message)); + } + + return new Hl7v2TraceInfo(unusedSegments); + } + + private static int FindOffsetInSegment(string segmentValue, Hl7v2EncodingCharacters encodingCharacters, int fieldIndex, int componentIndex) + { + var startFieldIndex = segmentValue.IndexOfNthOccurrence(encodingCharacters.FieldSeparator, fieldIndex) + 1; + var endFieldIndex = segmentValue.IndexOfNthOccurrence(encodingCharacters.FieldSeparator, fieldIndex + 1); + if (endFieldIndex == -1) + { + endFieldIndex = segmentValue.Length; + } + + var fieldValue = segmentValue.Substring(startFieldIndex, endFieldIndex - startFieldIndex); + return startFieldIndex + fieldValue.IndexOfNthOccurrence(encodingCharacters.ComponentSeparator, componentIndex) + 1; + } + } +} diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/UnusedHl7v2Component.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/UnusedHl7v2Component.cs new file mode 100644 index 000000000..8784a8ddb --- /dev/null +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/UnusedHl7v2Component.cs @@ -0,0 +1,23 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.Models +{ + public class UnusedHl7v2Component + { + public UnusedHl7v2Component(int start, int end, string value) + { + Start = start; + End = end; + Value = value; + } + + public int Start { get; } + + public int End { get; } + + public string Value { get; } + } +} diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/UnusedHl7v2Segment.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/UnusedHl7v2Segment.cs new file mode 100644 index 000000000..d4c56d8ed --- /dev/null +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/UnusedHl7v2Segment.cs @@ -0,0 +1,25 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; + +namespace Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.Models +{ + public class UnusedHl7v2Segment + { + public UnusedHl7v2Segment(int line) + { + Type = string.Empty; + Line = line; + Components = new List(); + } + + public string Type { get; set; } + + public int Line { get; set; } + + public List Components { get; set; } + } +} diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Models/FhirConverterErrorCode.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Models/FhirConverterErrorCode.cs index 31387d52c..6fede11bd 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Models/FhirConverterErrorCode.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Models/FhirConverterErrorCode.cs @@ -42,5 +42,6 @@ public enum FhirConverterErrorCode // PostprocessException JsonParsingError = 1401, JsonMergingError = 1402, + TraceInfoError = 1403, } } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Models/TraceInfo.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Models/TraceInfo.cs index 7b0f646a2..efb7ec083 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Models/TraceInfo.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Models/TraceInfo.cs @@ -7,7 +7,7 @@ namespace Microsoft.Health.Fhir.Liquid.Converter.Models { /// /// TraceInfo records processing details during conversion. - /// It is going to be implemented in coming releases. + /// For HL7 v2 conversion, please use its inherited class Hl7v2TraceInfo. /// public class TraceInfo { diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Resources.Designer.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Resources.Designer.cs index 0076ee178..662968d9e 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Resources.Designer.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Resources.Designer.cs @@ -266,5 +266,14 @@ internal static string TimeoutError { return ResourceManager.GetString("TimeoutError", resourceCulture); } } + + /// + /// Looks up a localized string similar to Error happened when processing TraceInfo: {0}. + /// + internal static string TraceInfoError { + get { + return ResourceManager.GetString("TraceInfoError", resourceCulture); + } + } } } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Resources.resx b/src/Microsoft.Health.Fhir.Liquid.Converter/Resources.resx index 8258098ec..6b5e8e3b4 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Resources.resx +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Resources.resx @@ -186,4 +186,7 @@ Time out when rendering templates. + + Error happened when processing TraceInfo: {0} + \ No newline at end of file