Fix version was missing for certain packages for osv (#136)

Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>

.gitignore

@@ -136,3 +136,4 @@ dmypy.json
 .pyre/
 .coverage
 reports/
+.mise.toml

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "appthreat-vulnerability-db"
-version = "5.6.7"
+version = "5.6.8"
 description = "AppThreat's vulnerability database and package search library with a built-in file based storage. OSV, CVE, GitHub, npm are the primary sources of vulnerabilities."
 authors = [
     {name = "Team AppThreat", email = "cloud@appthreat.com"},

test/data/osv-pypi-ujson-bug.json

@@ -0,0 +1,110 @@
+{
+  "id": "GHSA-wpqr-jcpx-745r",
+  "summary": "Incorrect handling of invalid surrogate pair characters",
+  "details": "### Impact\n_What kind of vulnerability is it? Who is impacted?_\n\nAnyone parsing JSON from an untrusted source is vulnerable.\n\nJSON strings that contain escaped surrogate characters not part of a proper surrogate pair were decoded incorrectly. Besides corrupting strings, this allowed for potential key confusion and value overwriting in dictionaries.\n\nExamples:\n\n```python\n# An unpaired high surrogate character is ignored.\n>>> ujson.loads(r'\"\\uD800\"')\n''\n>>> ujson.loads(r'\"\\uD800hello\"')\n'hello'\n\n# An unpaired low surrogate character is preserved.\n>>> ujson.loads(r'\"\\uDC00\"')\n'\\udc00'\n\n# A pair of surrogates with additional non surrogate characters pair up in spite of being invalid.\n>>> ujson.loads(r'\"\\uD800foo bar\\uDC00\"')\n'foo bar\ud800\udc00'\n```\n\n### Patches\n_Has the problem been patched? What versions should users upgrade to?_\n\nUsers should upgrade to UltraJSON 5.4.0.\n\nFrom version 5.4.0, UltraJSON decodes lone surrogates in the same way as the standard library's `json` module does, preserving them in the parsed output:\n\n```python3\n>>> ujson.loads(r'\"\\uD800\"')\n'\\ud800'\n>>> ujson.loads(r'\"\\uD800hello\"')\n'\\ud800hello'\n>>> ujson.loads(r'\"\\uDC00\"')\n'\\udc00'\n>>> ujson.loads(r'\"\\uD800foo bar\\uDC00\"')\n'\\ud800foo bar\\udc00'\n```\n\n### Workarounds\n_Is there a way for users to fix or remediate the vulnerability without upgrading?_\n\nShort of switching to an entirely different JSON library, there are no safe alternatives to upgrading.\n\n### For more information\nIf you have any questions or comments about this advisory:\n* Open an issue in [UltraJSON](http://github.com/ultrajson/ultrajson/issues)\n",
+  "aliases": [
+    "CVE-2022-31116"
+  ],
+  "modified": "2024-02-21T05:24:44.762164Z",
+  "published": "2022-07-05T21:06:00Z",
+  "database_specific": {
+    "nvd_published_at": "2022-07-05T18:15:00Z",
+    "cwe_ids": [
+      "CWE-670"
+    ],
+    "severity": "HIGH",
+    "github_reviewed": true,
+    "github_reviewed_at": "2022-07-05T21:06:00Z"
+  },
+  "references": [
+    {
+      "type": "WEB",
+      "url": "https://github.com/ultrajson/ultrajson/security/advisories/GHSA-wpqr-jcpx-745r"
+    },
+    {
+      "type": "ADVISORY",
+      "url": "https://nvd.nist.gov/vuln/detail/CVE-2022-31116"
+    },
+    {
+      "type": "WEB",
+      "url": "https://github.com/ultrajson/ultrajson/commit/67ec07183342589d602e0fcf7bb1ff3e19272687"
+    },
+    {
+      "type": "PACKAGE",
+      "url": "https://github.com/ultrajson/ultrajson"
+    },
+    {
+      "type": "WEB",
+      "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/NAU5N4A7EUK2AMUCOLYDD5ARXAJYZBD2"
+    },
+    {
+      "type": "WEB",
+      "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/OPPU5FZP3LCTXYORFH7NHUMYA5X66IA7"
+    }
+  ],
+  "affected": [
+    {
+      "package": {
+        "name": "ujson",
+        "ecosystem": "PyPI",
+        "purl": "pkg:pypi/ujson"
+      },
+      "ranges": [
+        {
+          "type": "ECOSYSTEM",
+          "events": [
+            {
+              "introduced": "0"
+            },
+            {
+              "fixed": "5.4.0"
+            }
+          ]
+        }
+      ],
+      "versions": [
+        "1.15",
+        "1.18",
+        "1.19",
+        "1.21",
+        "1.22",
+        "1.23",
+        "1.30",
+        "1.33",
+        "1.34",
+        "1.35",
+        "1.4",
+        "1.6",
+        "1.8",
+        "1.9",
+        "2.0.0",
+        "2.0.1",
+        "2.0.2",
+        "2.0.3",
+        "3.0.0",
+        "3.1.0",
+        "3.2.0",
+        "4.0.0",
+        "4.0.1",
+        "4.0.2",
+        "4.1.0",
+        "4.2.0",
+        "4.3.0",
+        "5.0.0",
+        "5.1.0",
+        "5.2.0",
+        "5.3.0"
+      ],
+      "database_specific": {
+        "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2022/07/GHSA-wpqr-jcpx-745r/GHSA-wpqr-jcpx-745r.json"
+      }
+    }
+  ],
+  "schema_version": "1.6.0",
+  "severity": [
+    {
+      "type": "CVSS_V3",
+      "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
+    }
+  ]
+}

test/test_source.py

@@ -26,6 +26,7 @@ def test_cve_wconfig_json():
     with open(test_cve_data, "r") as fp:
         return json.loads(fp.read())
 
+
 @pytest.fixture
 def test_nvd_api_json1():
     test_cve_data = os.path.join(
@@ -34,6 +35,7 @@ def test_nvd_api_json1():
     with open(test_cve_data, "r") as fp:
         return json.loads(fp.read())
 
+
 @pytest.fixture
 def test_nvd_api_json2():
     test_cve_data = os.path.join(
@@ -42,6 +44,7 @@ def test_nvd_api_json2():
     with open(test_cve_data, "r") as fp:
         return json.loads(fp.read())
 
+
 @pytest.fixture
 def test_nvd_api_json3():
     test_cve_data = os.path.join(
@@ -50,6 +53,7 @@ def test_nvd_api_json3():
     with open(test_cve_data, "r") as fp:
         return json.loads(fp.read())
 
+
 @pytest.fixture
 def test_osv_rust_json():
     test_cve_data = os.path.join(
@@ -104,6 +108,15 @@ def test_osv_pypi2_json():
         return json.loads(fp.read())
 
 
+@pytest.fixture
+def test_osv_pypi3_json():
+    test_cve_data = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)), "data", "osv-pypi-ujson-bug.json"
+    )
+    with open(test_cve_data, "r") as fp:
+        return json.loads(fp.read())
+
+
 @pytest.fixture
 def test_aqua_alsa_json():
     test_cve_data = os.path.join(
@@ -441,8 +454,11 @@ def test_osv_convert(
     test_osv_go_json,
     test_osv_pypi_json,
     test_osv_pypi2_json,
+    test_osv_pypi3_json
 ):
     osvlatest = OSVSource()
+    cve_data = osvlatest.convert(test_osv_pypi3_json)
+    assert cve_data
     cve_data = osvlatest.convert(test_osv_rust_json)
     assert cve_data
     cve_data = osvlatest.convert(test_osv_mvn_json)

vdb/lib/osv.py

@@ -249,15 +249,15 @@ def to_vuln(self, cve_data):
             # Problem 2: The versions_list may be unsorted and trying to sort based on semantic versions can fail
             # Solution: We do our best to sort the versions_list. If it fails, we assume the input is sorted and store the first and last entry as min and max.
             # The assumption seems to be working as of today, but could result in false positives or false negatives in the future.
-            needs_version_backup = True
+            needs_version_backup = False
+            possible_fix_version = ""
             for r in ranges:
                 events = r.get("events")
                 for ev in events:
                     if ev.get("introduced", "") in (0, "0", "0.0.0"):
-                        break
-                    if ev.get("fixed") or ev.get("last_affected"):
-                        needs_version_backup = False
-                        break
+                        needs_version_backup = True
+                    if ev.get("fixed"):
+                        possible_fix_version = ev.get("fixed")
             if needs_version_backup and len(versions_list) > 1:
                 try:
                     min_ver = min(versions_list, key=Version.parse)
@@ -281,7 +281,7 @@ def to_vuln(self, cve_data):
                         version_end_including=max_ver,
                         version_start_excluding="",
                         version_end_excluding="",
-                        fix_version_start_including="",
+                        fix_version_start_including=possible_fix_version,
                         fix_version_end_including="",
                         fix_version_start_excluding=max_ver,
                         fix_version_end_excluding="",