Merge pull request #24 from fundakol/write-more-tests

Fix outline scenario status

.github/workflows/main.yml

@@ -15,7 +15,7 @@ jobs:
     needs: lint
     strategy:
       matrix:
-        name: [ "3.7", "3.8", "3.9", "3.10" ]
+        name: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
         include:
         - name: "3.7"
           python: "3.7"
@@ -30,6 +30,9 @@ jobs:
           python: "3.10"
           tox_env: "py310"
           use_coverage: true
+        - name: "3.11"
+          python: "3.11"
+          tox_env: "py311"
     steps:
       - uses: actions/checkout@v3
       - name: Setup Python ${{ matrix.python }}

src/behave_xray/formatter.py

@@ -1,15 +1,16 @@
+import importlib
+import json
+import sys
 from collections import defaultdict
 from dataclasses import dataclass, field
 from enum import Enum, auto
-import importlib
 from os import environ, getenv
 from typing import AnyStr, Dict, List, Optional, Tuple, Union
 
 import pluggy
 from behave.formatter.base import Formatter
 from behave.model import Feature as BehaveFeature
 from behave.model import Scenario as BehaveScenario
-from behave.model import ScenarioOutline as BehaveScenarioOutline
 from behave.model import Status
 
 from behave_xray import hookspecs
@@ -79,6 +80,7 @@ class _XrayFormatterBase(Formatter):
 
     def __init__(self, stream, config, publisher: XrayPublisher):
         super().__init__(stream, config)
+        self.stream = self.open()
         self.pm = self._get_plugin_manager()
         self._register_user_hook()
         self.xray_publisher = publisher
@@ -120,7 +122,7 @@ def _get_auth(jira_config: JiraConfig) -> Union[Tuple[str, str], AuthBase]:
         elif jira_config.auth_method == AuthType.token:
             return PersonalAccessTokenAuth(token=jira_config.token)
         else:  # basic
-            return (jira_config.user_name, jira_config.user_password)
+            return jira_config.user_name, jira_config.user_password
 
     def _get_summary(self) -> str:
         return self.config.userdata.get('xray.summary', '')
@@ -156,7 +158,7 @@ def feature(self, feature):
                 self.test_execution.test_plan_key = test_plan_key
 
     def is_scenario_outline(self):
-        return isinstance(self.current_scenario, BehaveScenarioOutline)
+        return True if 'Scenario Outline' in self.current_scenario.keyword else False
 
     def scenario(self, scenario):
         self.current_scenario = scenario
@@ -180,14 +182,14 @@ def get_verdict(self, step) -> Verdict:
         verdict = Verdict(self.current_scenario.status, '')
         if step.status == Status.failed:
             verdict.message = step.error_message
-        if step.status == Status.untested:
+        elif step.status == Status.untested:
             verdict.message = 'Untested'
-        if step.status == Status.skipped:
+        elif step.status == Status.skipped:
             verdict.message = self.current_scenario.skip_reason
         return verdict
 
     @property
-    def current_test_case(self) -> ScenarioResult:
+    def current_test_case(self) -> Optional[ScenarioResult]:
         try:
             return self.testcases[self.current_test_key]
         except KeyError:
@@ -221,6 +223,9 @@ def eof(self) -> None:
         self.collect_tests()
         if self.test_execution.tests:
             self.xray_publisher.publish(self.test_execution.as_dict())
+            if self.stream != sys.stdout:
+                self.stream.write(json.dumps(self.test_execution.as_dict(), indent=4))
+                self.stream.flush()
         self.test_execution.flush()
         self.reset()
 

tests/features/calculator.feature

@@ -22,3 +22,13 @@ Feature: Calculator
   Scenario: Add two numbers without jira id
     When I add 5 and 7
     Then result is 12
+
+  @jira.testcase('JIRA-34')
+  Scenario Outline: Add two numbers: <first> and <second>
+    When I add <first> and <second>
+    Then result is <result>
+
+    Examples:
+      | first | second | result |
+      | 1     | 2      | 3      |
+      | 2     | 5      | 8      |

tests/integration_test.py

@@ -1,3 +1,4 @@
+import json
 import os
 import subprocess
 
@@ -24,4 +25,52 @@ def test_if_xray_formatter_publishes_results(formatter, auth_type, auth):
     )
     assert not process.stderr
     assert 'Uploaded results to JIRA XRAY Test Execution: JIRA-1000' in process.stdout, process.stdout
-    assert '3 scenarios passed, 1 failed, 0 skipped' in process.stdout, process.stdout
+    assert '4 scenarios passed, 2 failed, 0 skipped' in process.stdout, process.stdout
+
+
+def test_if_xray_formatter_results_matches_expected_format(auth, tmp_path):
+    report_path = tmp_path / 'xray.json'
+    env = dict(os.environ).copy()
+    env.update(auth('basic'))
+
+    process = subprocess.run(
+        ['python', '-m', 'behave', 'tests', '-f', 'behave_xray:XrayFormatter', '-o', report_path.name],
+        capture_output=True,
+        text=True,
+        env=env
+    )
+    print(process.stdout)
+    assert not process.stderr
+    assert 'Uploaded results to JIRA XRAY Test Execution: JIRA-1000' in process.stdout
+    assert '4 scenarios passed, 2 failed, 0 skipped' in process.stdout
+
+    with open(report_path.name, 'r') as f:
+        report = json.load(f)
+
+    assert 'tests' in report
+    assert report['tests'] == [
+            {
+                'testKey': 'JIRA-31',
+                'status': 'PASS',
+                'comment': '',
+                'examples': []
+            },
+            {
+                'testKey': 'JIRA-32',
+                'status': 'FAIL',
+                'comment': 'Assertion Failed: Not equal',
+                'examples': []
+            },
+            {
+                'testKey': 'JIRA-33',
+                'status': 'PASS',
+                'comment': '',
+                'examples': []
+            },
+            {
+                'testKey': 'JIRA-34',
+                'status': 'FAIL',
+                'comment': '',  # FIXME: missing assertion message
+                'examples': ['PASS', 'FAIL']
+            }
+        ]

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py36,py37,py38,py39,py310,flake8
+envlist = py36,py37,py38,py39,py310,311,flake8
 isolated_build = True
 minversion = 3.20.0
 distshare = {homedir}/.tox/distshare