feat: command line option for exit code 2 in case of non-errored / su…

…ccessful evaluation "fail" (Fixes #409 )

packages/ace-cli-shared/src/index.js

@@ -35,6 +35,8 @@ const meowHelpMessage = `
     -l, --lang  <language> language code for localized messages (e.g. "fr"), default is "en"
 
     -T, --timeout <milliseconds> (default is 240000 per document)
+
+    -E, --exiterror2 exit process with code 2 when fail (return-2-on-validation-error)
   Examples
     $ ace -o out ~/Documents/book.epub
 
@@ -82,6 +84,10 @@ const meowOptions = {
     timeout: {
       alias: 'T',
       type: 'string'
+    },
+    exiterror2: { // process exit code
+      alias: 'E',
+      type: 'boolean'
     }
   }
 };
@@ -156,11 +162,12 @@ ${overrides.map(file => `  - ${file}`).join('\n')}
     jobId: '',
     lang: cli.flags.lang,
     timeout: cli.flags.timeout || undefined,
+    exiterror2: cli.flags.exiterror2 || undefined,
   }, axeRunner)
   .then(async (jobData) => {
     var reportJson = jobData[1];
     // if there were violations from the validation process, return 2
-    const fail = cliConfig['return-2-on-validation-error'] && reportJson['earl:result']['earl:outcome'] === 'fail';
+    const fail = (!!cli.flags.exiterror2 || cliConfig['return-2-on-validation-error']) && reportJson['earl:result']['earl:outcome'] === 'fail';
     const res = await winston.logAndWaitFinish('info', 'Closing logs.');
     quit(fail ? 2 : 0);
   })

packages/ace-http/src/index.js

@@ -44,6 +44,8 @@ const meowHelpMessage = `
     -l, --lang  <language> language code for localized messages (e.g. "fr"), default is "en"
 
     -T, --timeout <milliseconds> (default is 240000 per document)
+
+    -E, --exiterror2 exit process with code 2 when fail (return-2-on-validation-error)
   Examples
     $ ace-http -p 3000`;
 const meowOptions = {
@@ -76,6 +78,14 @@ const meowOptions = {
     lang: {
       alias: 'l',
       type: 'string'
+    },
+    timeout: {
+      alias: 'T',
+      type: 'string'
+    },
+    exiterror2: { // process exit code
+      alias: 'E',
+      type: 'boolean'
     }
   }
 };

tests/__tests__/__snapshots__/cli.test.js.snap

@@ -48,6 +48,8 @@ exports[`Running the CLI with no input should fail 2`] = `
     -l, --lang  <language> language code for localized messages (e.g. \"fr\"), default is \"en\"
 
     -T, --timeout <milliseconds> (default is 240000 per document)
+
+    -E, --exiterror2 exit process with code 2 when fail (return-2-on-validation-error)
   Examples
     $ ace -o out ~/Documents/book.epub
 
@@ -79,6 +81,8 @@ exports[`Running the CLI with the -h option should print help 1`] = `
     -l, --lang  <language> language code for localized messages (e.g. \"fr\"), default is \"en\"
 
     -T, --timeout <milliseconds> (default is 240000 per document)
+
+    -E, --exiterror2 exit process with code 2 when fail (return-2-on-validation-error)
   Examples
     $ ace -o out ~/Documents/book.epub
 

tests/__tests__/cli.test.js

@@ -100,13 +100,17 @@ describe('Running the CLI', () => {
       });
       const log = stripAnsi(stdout);
       expect(/^warn:/m.test(log)).toBe(false);
+
+      expect(status).toBe(0);
     });
     test('prints the "Done" info', () => {
       const { stdout, stderr, status } = ace(['base-epub-30'], {
         cwd: path.resolve(__dirname, '../data'),
       });
       const log = stripAnsi(stdout);
       expect(/^info:\s+Done/m.test(log)).toBe(true);
+
+      expect(status).toBe(0);
     });
   });
 
@@ -120,6 +124,8 @@ describe('Running the CLI', () => {
       // The Electron-based Axe runner handles .xml files just fine
       const condition = process.env.AXE_ELECTRON_RUNNER ? true : /^warn:\s+Copying document with extension/m.test(log);
       expect(condition).toBe(true);
+
+      expect(status).toBe(0);
     });
 
     test('when the EPUB contains SVG Content Documents', () => {
@@ -128,6 +134,8 @@ describe('Running the CLI', () => {
       });
       const log = stripAnsi(stdout);
       expect(/^warn:\s+The SVG Content Documents in this EPUB will be ignored\./m.test(log)).toBe(true);
+
+      expect(status).toBe(0);
     });
   });
 
@@ -138,14 +146,27 @@ describe('Running the CLI', () => {
       });
       const log = stripAnsi(stdout);
       expect(/^warn:\s+\[xmldom error\]	entity not found/m.test(log)).toBe(false);
+
+      expect(status).toBe(0);
     });
   });
 
-  /*test('with return-2-on-validation-error set to true should exit with return code 2', () => {
-    // TODO this test won't work until we can specify the CLI option to enable returning 2 on violation(s)
+  test('without return-2-on-validation-error set to true should exit with return code 0', () => {
     const { stdout, stderr, status } = ace(['has-violations'], {
       cwd: path.resolve(__dirname, '../data')
     });
+    expect(status).toBe(0);
+  });
+  test('with return-2-on-validation-error alias set to true should exit with return code 2', () => {
+    const { stdout, stderr, status } = ace(['-E', 'has-violations'], {
+      cwd: path.resolve(__dirname, '../data')
+    });
     expect(status).toBe(2);
-  });*/
+  });
+  test('with return-2-on-validation-error set to true should exit with return code 2', () => {
+    const { stdout, stderr, status } = ace(['--exiterror2', 'has-violations'], {
+      cwd: path.resolve(__dirname, '../data')
+    });
+    expect(status).toBe(2);
+  });
 });

tests/runAceCLI.js

@@ -4,7 +4,7 @@
 
 const path = require('path');
 const spawn = require('cross-spawn');
-const spawnSync = require('child_process').spawnSync;
+// const spawnSync = require('child_process').spawnSync;
 
 const EXE_PATH_ELECTRON = path.resolve(__dirname, '../packages/ace-axe-runner-electron/lib/cli.js');
 

website/content/docs/cli.md

@@ -76,6 +76,8 @@ Ace by DAISY, an Accessibility Checker for EPUB
     -l, --lang  <language> language code for localized messages (e.g. "fr"), default is "en"
 
     -T, --timeout <milliseconds> (default is 240000 per document)
+
+    -E, --exiterror2 exit process with code 2 when fail (return-2-on-validation-error)
   Examples
     $ ace -o out ~/Documents/book.epub
 

website/content/docs/config.md

@@ -30,9 +30,6 @@ The defaults for Ace configuration are:
   },
   "report": {
     "use-relative-paths": true
-  },
-  "runtime": {
-    "concurrencyLevel": 4
   }
 }
 ```
@@ -73,9 +70,3 @@ Indicates whether to save reports in a subdirectory of the output directory. If
 Default: "`true`"
 
 Indicates whether the report uses relative or absolute paths.
-
-### `runtime.concurrencyLevel`
-
-Default: "`4`"
-
-The number of concurrent headless browser instances used to check HTML content.

website/content/docs/http-api.md

@@ -32,6 +32,8 @@ Ace by DAISY, an Accessibility Checker for EPUB
     -l, --lang  <language> language code for localized messages (e.g. "fr"), default is "en"
 
     -T, --timeout <milliseconds> (default is 240000 per document)
+
+    -E, --exiterror2 exit process with code 2 when fail (return-2-on-validation-error)
   Examples
     $ ace-http -p 3000
  ```