diff --git a/README.md b/README.md index b0c651d5..4f9ef05f 100644 --- a/README.md +++ b/README.md @@ -450,6 +450,8 @@ The options that we support for calculation are as follows: - `[buildStatementLevelHTML]`<[boolean](#calculation-options)>: Builds and returns HTML at the statement level (default: `false`) - `[calculateClauseCoverage]`<[boolean](#calculation-options)>: Include HTML structure with clause coverage highlighting (default: `true`) +- `[calculateClauseUncoverage]`<[boolean](#calculation-options)>: Include HTML structure with clause uncoverage highlighting (default: `false`) +- `[calculateCoverageDetails]`<[boolean](#calculation-options)>: Include details on logic clause coverage. (default: `false`) - `[calculateHTML]`<[boolean](#calculation-options)>: Include HTML structure for highlighting (default: `true`) - `[calculateSDEs]`<[boolean](#calculation-options)>: Include Supplemental Data Elements (SDEs) in calculation (default: `true`) - `[clearElmJsonsCache]`<[boolean](#calculation-options)>: If `true`, clears ELM JSON cache before running calculation (default: `false`) @@ -931,9 +933,11 @@ have a blue background and dashed underline in the highlighted HTML, indicating The highlighted HTML also provides an approximate percentage for what percentage of the measure logic is covered by the patients that were passed in to execution. +This option, `calculateClauseCoverage`, defaults to enabled. When running the CLI in `--debug` mode this option will be enabled and output to the debug directory. + ![Screenshot of Highlighted Clause Coverage HTML](./static/coverage-highlighting-example.png) -HTML strings are returned for each group defined in the `Measure` resource as a lookup object `groupClauseCoverageHTML` which maps `group ID -> HTML string` +HTML strings are returned for each group defined in the `Measure` resource in the lookup object, `groupClauseCoverageHTML`, which maps `group ID -> HTML string` ```typescript import { Calculator } from 'fqm-execution'; @@ -953,6 +957,68 @@ const { results, groupClauseCoverageHTML } = await Calculator.calculate(measureB */ ``` +### Uncoverage Highlighting + +`fqm-execution` can also generate highlighted HTML that indicate which pieces of the measure logic did NOT have a "truthy" value during calculation. This is the complement to coverage. Clauses that are not covered will be highlighted red. + +This option, `calculateClauseUncoverage`, defaults to disabled. When running the CLI in `--debug` mode this option will be enabled and output to the debug directory. + +HTML strings are returned for each group defined in the `Measure` resource in the lookup object,`groupClauseUncoverageHTML`, which is structured similarly to Group Clause Coverage. + +```typescript +import { Calculator } from 'fqm-execution'; + +const { results, groupClauseUncoverageHTML } = await Calculator.calculate(measureBundle, patientBundles, { + calculateClauseUncoverage: true /* false by default */ +}); + +// groupClauseUncoverageHTML +/* ^? + { + 'population-group-1': '

population-group-1 Clause Uncoverage: X of Y clauses

...' + ... + } + + +*/ +``` + +### Coverage Details + +Details on clause coverage can also be returned. This includes a count of how many clauses there are, how many are covered and uncovered, and information about which clauses are uncovered. + +This option, `calculateCoverageDetails`, defaults to disabled. When running the CLI in `--debug` mode this option will be enabled and output to the debug directory. + +This information is returned for each group defined in the `Measure` resource in the lookup object,`groupClauseCoverageDetails`, which maps `group ID -> coverageDetails`. See example below for structure of this object. + +```typescript +import { Calculator } from 'fqm-execution'; + +const { results, groupClauseCoverageDetails } = await Calculator.calculate(measureBundle, patientBundles, { + calculateCoverageDetails: true /* false by default */ +}); + +// groupClauseCoverageDetails +/* ^? + { + "population-group-1": { + "totalClauseCount": 333, + "coveredClauseCount": 330, + "uncoveredClauseCount": 3, + "uncoveredClauses": [ + { + "localId": "97", + "libraryName": "MAT6725TestingMeasureCoverage", + "statementName": "Has Medical or Patient Reason for Not Ordering ACEI or ARB or ARNI" + }, + ... + ] + } + ... + } +*/ +``` + ### Clause Coverage of Null and False Literal Values Since clause coverage calculation and highlighting are based on whether individual pieces of the measure logic CQL were processed at all during calculation and held "truthy" values, `Null` and false `Literal`s that are processed during calculation would prevent 100% clause coverage and highlighting. In order to handle this special case, these clauses that will never hold "truthy" values will be highlighted and counted as covered if they were simply processed during calculation. diff --git a/src/calculation/Calculator.ts b/src/calculation/Calculator.ts index 6c2ccc3e..0849921e 100644 --- a/src/calculation/Calculator.ts +++ b/src/calculation/Calculator.ts @@ -14,7 +14,8 @@ import { OneOrManyBundles, OneOrMultiPatient, PopulationGroupResult, - DetailedPopulationGroupResult + DetailedPopulationGroupResult, + ClauseCoverageDetails } from '../types/Calculator'; import { PopulationType, MeasureScoreType, ImprovementNotation } from '../types/Enums'; import * as Execution from '../execution/Execution'; @@ -70,6 +71,8 @@ export async function calculate( options.calculateHTML = options.calculateHTML ?? true; options.calculateSDEs = options.calculateSDEs ?? true; options.calculateClauseCoverage = options.calculateClauseCoverage ?? true; + options.calculateClauseUncoverage = options.calculateClauseUncoverage ?? false; + options.calculateCoverageDetails = options.calculateCoverageDetails ?? false; options.disableHTMLOrdering = options.disableHTMLOrdering ?? false; options.buildStatementLevelHTML = options.buildStatementLevelHTML ?? false; @@ -87,6 +90,8 @@ export async function calculate( const executionResults: ExecutionResult[] = []; let overallClauseCoverageHTML: string | undefined; let groupClauseCoverageHTML: Record | undefined; + let groupClauseUncoverageHTML: Record | undefined; + let groupClauseCoverageDetails: Record | undefined; let newValueSetCache: fhir4.ValueSet[] | undefined = [...valueSetCache]; const allELM: ELM[] = []; @@ -251,12 +256,8 @@ export async function calculate( patientSource = resolvePatientSource(patientBundles, options); if (!isCompositeExecution && options.calculateClauseCoverage) { - groupClauseCoverageHTML = generateClauseCoverageHTML( - measure, - executedELM, - executionResults, - options.disableHTMLOrdering - ); + const coverage = generateClauseCoverageHTML(measure, executedELM, executionResults, options); + groupClauseCoverageHTML = coverage.coverage; overallClauseCoverageHTML = ''; Object.entries(groupClauseCoverageHTML).forEach(([groupId, result]) => { overallClauseCoverageHTML += result; @@ -272,6 +273,25 @@ export async function calculate( } } }); + + // pull out uncoverage html + if (options.calculateClauseUncoverage && coverage.uncoverage) { + groupClauseUncoverageHTML = coverage.uncoverage; + if (debugObject && options.enableDebugOutput) { + Object.entries(groupClauseUncoverageHTML).forEach(([groupId, result]) => { + const debugUncoverageHTML = { + name: `clause-uncoverage-${groupId}.html`, + html: result + }; + if (Array.isArray(debugObject.html)) { + debugObject.html.push(debugUncoverageHTML); + } else { + debugObject.html = [debugUncoverageHTML]; + } + }); + } + } + // don't necessarily need this file, but adding it for backwards compatibility if (debugObject && options.enableDebugOutput) { debugObject.html?.push({ @@ -279,6 +299,14 @@ export async function calculate( html: overallClauseCoverageHTML }); } + + // grab coverage details + if (options.calculateCoverageDetails && coverage.details) { + groupClauseCoverageDetails = coverage.details; + if (debugObject && options.enableDebugOutput) { + debugObject.coverageDetails = groupClauseCoverageDetails; + } + } } } @@ -314,7 +342,9 @@ export async function calculate( ) }), ...(overallClauseCoverageHTML && { coverageHTML: overallClauseCoverageHTML }), - ...(groupClauseCoverageHTML && { groupClauseCoverageHTML: groupClauseCoverageHTML }) + ...(groupClauseCoverageHTML && { groupClauseCoverageHTML: groupClauseCoverageHTML }), + ...(groupClauseUncoverageHTML && { groupClauseUncoverageHTML: groupClauseUncoverageHTML }), + ...(groupClauseCoverageDetails && { groupClauseCoverageDetails: groupClauseCoverageDetails }) }; } diff --git a/src/calculation/HTMLBuilder.ts b/src/calculation/HTMLBuilder.ts index 71c5dc00..1fde3110 100644 --- a/src/calculation/HTMLBuilder.ts +++ b/src/calculation/HTMLBuilder.ts @@ -2,6 +2,7 @@ import { Annotation, ELM } from '../types/ELMTypes'; import Handlebars from 'handlebars'; import { CalculationOptions, + ClauseCoverageDetails, ClauseResult, DetailedPopulationGroupResult, ExecutionResult, @@ -91,6 +92,29 @@ Handlebars.registerHelper('highlightCoverage', (localId, context) => { return ''; }); +// apply highlighting style to uncovered clauses +Handlebars.registerHelper('highlightUncoverage', (localId, context) => { + const libraryName: string = context.data.root.libraryName; + + if ( + (context.data.root.uncoveredClauses as ClauseResult[]).some( + result => result.libraryName === libraryName && result.localId === localId + ) + ) { + // Mark with red styling if clause is found in uncoverage list + return objToCSS(cqlLogicClauseFalseStyle); + } else if ( + (context.data.root.coveredClauses as ClauseResult[]).some( + result => result.libraryName === libraryName && result.localId === localId + ) + ) { + // Mark with white (clear out styling) if the clause is in coverage list + return objToCSS(cqlLogicUncoveredClauseStyle); + } + // If this clause has no results then it should not be styled + return ''; +}); + /** * Sort statements into population, then non-functions, then functions */ @@ -212,14 +236,20 @@ export function generateHTML( * @returns a lookup object where the key is the groupId and the value is the * clause coverage HTML */ -export function generateClauseCoverageHTML( +export function generateClauseCoverageHTML( measure: fhir4.Measure, elmLibraries: ELM[], executionResults: ExecutionResult[], - disableHTMLOrdering?: boolean -): Record { + options: T +): { + coverage: Record; + uncoverage?: Record; + details?: Record; +} { const groupResultLookup: Record = {}; - const htmlGroupLookup: Record = {}; + const coverageHtmlGroupLookup: Record = {}; + const uncoverageHtmlGroupLookup: Record = {}; + const coverageDetailsGroupLookup: Record = {}; // get the detailed result for each group within each patient and add it // to the key in groupResults that matches the groupId @@ -244,7 +274,7 @@ export function generateClauseCoverageHTML( // Filter out any statement results where the statement relevance is NA const uniqueRelevantStatements = flattenedStatementResults.filter(s => s.relevance !== Relevance.NA); - if (!disableHTMLOrdering) { + if (!options.disableHTMLOrdering) { sortStatements(measure, groupId, uniqueRelevantStatements); } @@ -270,28 +300,72 @@ export function generateClauseCoverageHTML( } }); - let htmlString = `

${groupId} Clause Coverage: ${calculateClauseCoverage( - uniqueRelevantStatements, - flattenedClauseResults - )}%

`; + const clauseCoverage = calculateClauseCoverage(uniqueRelevantStatements, flattenedClauseResults); + const uniqueCoverageClauses = clauseCoverage.coveredClauses.concat(clauseCoverage.uncoveredClauses); + + // setup initial html for coverage + let coverageHtmlString = `

${groupId} Clause Coverage: ${clauseCoverage.percentage}%

`; + + // setup initial html for uncoverage + let uncoverageHtmlString = ''; + if (options.calculateClauseUncoverage) { + uncoverageHtmlString = `

${groupId} Clause Uncoverage: ${clauseCoverage.uncoveredClauses.length} of ${ + clauseCoverage.coveredClauses.length + clauseCoverage.uncoveredClauses.length + } clauses

`; + } // generate HTML clauses using hbs template for each annotation statementAnnotations.forEach(a => { - const res = main({ + coverageHtmlString += main({ libraryName: a.libraryName, statementName: a.statementName, - clauseResults: flattenedClauseResults, + clauseResults: uniqueCoverageClauses, ...a.annotation[0].s, highlightCoverage: true }); - htmlString += res; + + // calculate for uncoverage + if (options.calculateClauseUncoverage) { + uncoverageHtmlString += main({ + libraryName: a.libraryName, + statementName: a.statementName, + uncoveredClauses: clauseCoverage.uncoveredClauses, + coveredClauses: clauseCoverage.coveredClauses, + ...a.annotation[0].s, + highlightUncoverage: true + }); + } }); - htmlString += '
'; + coverageHtmlString += '
'; + uncoverageHtmlString += '
'; + + coverageHtmlGroupLookup[groupId] = coverageHtmlString; + if (options.calculateClauseUncoverage) { + uncoverageHtmlGroupLookup[groupId] = uncoverageHtmlString; + } - htmlGroupLookup[groupId] = htmlString; + // If details on coverage are requested, tally them up and add them to the map. + if (options.calculateCoverageDetails) { + coverageDetailsGroupLookup[groupId] = { + totalClauseCount: clauseCoverage.coveredClauses.length + clauseCoverage.uncoveredClauses.length, + coveredClauseCount: clauseCoverage.coveredClauses.length, + uncoveredClauseCount: clauseCoverage.uncoveredClauses.length, + uncoveredClauses: clauseCoverage.uncoveredClauses.map(uncoveredClause => { + return { + localId: uncoveredClause.localId, + libraryName: uncoveredClause.libraryName, + statementName: uncoveredClause.statementName + }; + }) + }; + } }); - return htmlGroupLookup; + return { + coverage: coverageHtmlGroupLookup, + ...(options.calculateClauseUncoverage && { uncoverage: uncoverageHtmlGroupLookup }), + ...(options.calculateCoverageDetails && { details: coverageDetailsGroupLookup }) + }; } /** @@ -301,7 +375,10 @@ export function generateClauseCoverageHTML( * @param clauseResults ClauseResult array from calculation * @returns percentage out of 100, represented as a string */ -export function calculateClauseCoverage(relevantStatements: StatementResult[], clauseResults: ClauseResult[]): string { +export function calculateClauseCoverage( + relevantStatements: StatementResult[], + clauseResults: ClauseResult[] +): { percentage: string; coveredClauses: ClauseResult[]; uncoveredClauses: ClauseResult[] } { // find all relevant clauses using statementName and libraryName from relevant statements const allRelevantClauses = clauseResults.filter(c => relevantStatements.some( @@ -317,5 +394,14 @@ export function calculateClauseCoverage(relevantStatements: StatementResult[], c allRelevantClauses.filter(clause => clause.final === FinalResult.TRUE), (c1, c2) => c1.libraryName === c2.libraryName && c1.localId === c2.localId ); - return ((coveredClauses.length / allUniqueClauses.length) * 100).toPrecision(3); + + const uncoveredClauses = allUniqueClauses.filter(c => { + return !coveredClauses.find(coveredC => c.libraryName === coveredC.libraryName && c.localId === coveredC.localId); + }); + + return { + percentage: ((coveredClauses.length / allUniqueClauses.length) * 100).toPrecision(3), + coveredClauses, + uncoveredClauses + }; } diff --git a/src/cli.ts b/src/cli.ts index a04b9b48..caf9880a 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -114,6 +114,8 @@ async function calc( if (program.outputType === 'raw') { result = await calculateRaw(measureBundle, patientBundles, calcOptions, valueSetCache); } else if (program.outputType === 'detailed') { + calcOptions.calculateClauseUncoverage = true; + calcOptions.calculateCoverageDetails = true; result = await calculate(measureBundle, patientBundles, calcOptions, valueSetCache); } else if (program.outputType === 'reports') { calcOptions.reportType = program.reportType || 'individual'; @@ -242,6 +244,10 @@ populatePatientBundles().then(async patientBundles => { dumpObject(debugOutput.gaps, 'gaps.json'); } + if (debugOutput?.coverageDetails) { + dumpObject(debugOutput.coverageDetails, 'coverageDetails.json'); + } + // Dump ELM if (debugOutput?.elm) { dumpELMJSONs(debugOutput.elm); diff --git a/src/templates/clause.ts b/src/templates/clause.ts index 00b735b3..9d855ee5 100644 --- a/src/templates/clause.ts +++ b/src/templates/clause.ts @@ -10,6 +10,18 @@ export default `{{~#if @root.highlightCoverage~}} {{~/if~}} {{~else~}} +{{~#if @root.highlightUncoverage~}} + +{{~#if value ~}} +{{ concat value }} +{{~/if ~}} +{{~#if s~}} +{{~#each s~}} +{{> clause ~}} +{{~/each ~}} +{{~/if~}} + +{{~else~}} {{~#if value ~}} {{ concat value }} @@ -20,4 +32,5 @@ export default `{{~#if @root.highlightCoverage~}} {{~/each ~}} {{~/if~}} +{{~/if~}} {{~/if~}}`; diff --git a/src/types/Calculator.ts b/src/types/Calculator.ts index f2456e83..8b007950 100644 --- a/src/types/Calculator.ts +++ b/src/types/Calculator.ts @@ -24,6 +24,10 @@ export interface CalculationOptions { calculateHTML?: boolean; /** Include HTML structure with clause coverage highlighting */ calculateClauseCoverage?: boolean; + /** Include HTML structure with clause uncoverage highlighting. Highlights what is not covered */ + calculateClauseUncoverage?: boolean; + /** Include details about the clause coverage. Total clause count, total covered count, info on uncovered clauses. */ + calculateCoverageDetails?: boolean; /** Enable debug output including CQL, ELM, results */ enableDebugOutput?: boolean; /** Enables the return of ELM Libraries and name of main library to be used for further processing. ex. gaps in care */ @@ -354,6 +358,7 @@ export interface DebugOutput { retrieves?: DataTypeQuery[]; bundle?: fhir4.Bundle | fhir4.Bundle[]; }; + coverageDetails?: Record; } /* @@ -395,6 +400,22 @@ export interface CalculationOutput extends Calcula parameters?: { [key: string]: any }; coverageHTML?: string; groupClauseCoverageHTML?: Record; + groupClauseUncoverageHTML?: Record; + groupClauseCoverageDetails?: Record; +} + +/** + * Details on clause coverage for group calculation. + */ +export interface ClauseCoverageDetails { + totalClauseCount: number; + coveredClauseCount: number; + uncoveredClauseCount: number; + uncoveredClauses: { + libraryName: string; + statementName: string; + localId: string; + }[]; } /** diff --git a/test/unit/HTMLBuilder.test.ts b/test/unit/HTMLBuilder.test.ts index 924eb0c0..5d933a45 100644 --- a/test/unit/HTMLBuilder.test.ts +++ b/test/unit/HTMLBuilder.test.ts @@ -219,10 +219,10 @@ describe('HTMLBuilder', () => { ] } ]; - const res = generateClauseCoverageHTML(simpleMeasure, [elm], executionResults); + const res = generateClauseCoverageHTML(simpleMeasure, [elm], executionResults, {}); - expect(res.test.replace(/\s/g, '')).toEqual(expectedHTML); - expect(res.test.includes(coverageStyleString)).toBeTruthy(); + expect(res.coverage.test.replace(/\s/g, '')).toEqual(expectedHTML); + expect(res.coverage.test.includes(coverageStyleString)).toBeTruthy(); }); test('simple HTML for two groups with generation with clause coverage styling', () => { @@ -246,12 +246,12 @@ describe('HTMLBuilder', () => { ] } ]; - const res = generateClauseCoverageHTML(simpleMeasure, [elm], executionResults); + const res = generateClauseCoverageHTML(simpleMeasure, [elm], executionResults, {}); - expect(res.test.replace(/\s/g, '')).toEqual(expectedHTML); - expect(res.test2.replace(/\s/g, '')).toEqual(expectedHTML2); - expect(res.test.includes(coverageStyleString)).toBeTruthy(); - expect(res.test2.includes(coverageStyleString)).toBeTruthy(); + expect(res.coverage.test.replace(/\s/g, '')).toEqual(expectedHTML); + expect(res.coverage.test2.replace(/\s/g, '')).toEqual(expectedHTML2); + expect(res.coverage.test.includes(coverageStyleString)).toBeTruthy(); + expect(res.coverage.test2.includes(coverageStyleString)).toBeTruthy(); }); test('ordered HTML with generation with clause coverage styling', () => { @@ -267,10 +267,10 @@ describe('HTMLBuilder', () => { ] } ]; - const res = generateClauseCoverageHTML(singlePopMeasure, [popRetrieveFuncElm], executionResults); + const res = generateClauseCoverageHTML(singlePopMeasure, [popRetrieveFuncElm], executionResults, {}); - expect(res.test.indexOf('ipp')).toBeLessThan(res.test.indexOf('SimpleVSRetrieve')); - expect(res.test.indexOf('SimpleVSRetrieve')).toBeLessThan(res.test.indexOf('A Function')); + expect(res.coverage.test.indexOf('ipp')).toBeLessThan(res.coverage.test.indexOf('SimpleVSRetrieve')); + expect(res.coverage.test.indexOf('SimpleVSRetrieve')).toBeLessThan(res.coverage.test.indexOf('A Function')); }); test('no library found should error', () => { @@ -336,7 +336,8 @@ describe('HTMLBuilder', () => { } ]; const results = calculateClauseCoverage(statementResults, clauseResults); - expect(results).toEqual('100'); + expect(results.percentage).toEqual('100'); + expect(results.uncoveredClauses).toEqual([]); }); test('html generation orders population first, then other, then function', () => {