Fix: reduce workflow dependencies (#6)

.github/workflows/build-meson-tst.yml

@@ -21,6 +21,8 @@ jobs:
         path: blint-db
     - name: Install dependencies
       run: |
+        [ -d venv ] || python3 -m venv venv
+        source venv/bin/activate
         python -m pip install --upgrade pip
         pip install setuptools wheel twine build
         cd blint-db && python -m pip install .
@@ -32,8 +34,9 @@ jobs:
     - name: Build and upload db
       run: |
         mkdir -p temp
+        source venv/bin/activate
         cd blint-db/
-        python blint_db/cli.py -f -Z1
+        python blint_db/cli.py --clean-start -f -Z1
         python ./.oras/orasclient.py
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/build-meson.yml

@@ -21,6 +21,8 @@ jobs:
         path: blint-db
     - name: Install dependencies
       run: |
+        [ -d venv ] || python3 -m venv venv
+        source venv/bin/activate
         python -m pip install --upgrade pip
         pip install setuptools wheel twine build
         cd blint-db && python -m pip install .
@@ -32,8 +34,9 @@ jobs:
     - name: Build and upload db
       run: |
         mkdir -p temp
+        source venv/bin/activate
         cd blint-db/
-        python blint_db/cli.py -Z1
+        python blint_db/cli.py --clean-start -Z1
         python ./.oras/orasclient.py
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/build-vcpkg-tst.yml

@@ -0,0 +1,43 @@
+name: Build and Upload vcpkg blint-db 0.1.0
+
+on:
+  schedule:
+  - cron: "0 */12 * * *"
+  workflow_dispatch:
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: appthreat/blintdb
+
+jobs:
+  builder-vcpkg-tst:
+    runs-on: ['self-hosted', 'ubuntu', 'arm64']
+    permissions:
+      contents: write
+      packages: write
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        repository: AppThreat/blint-db
+        path: blint-db
+    - name: Install dependencies
+      run: |
+        [ -d venv ] || python3 -m venv venv
+        source venv/bin/activate
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine build
+        cd blint-db && python -m pip install .
+    - name: Clean up previous
+      run: |
+        cd blint-db
+        rm -f info.log
+        rm -f blint.db
+    - name: Build and upload db
+      run: |
+        mkdir -p temp
+        source venv/bin/activate
+        cd blint-db/
+        python3 blint-db/blint_db/cli.py --clean-start -f -Z2
+        python3 ./.oras/orasclient.py
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        GITHUB_USERNAME: ${{ github.actor }}

.github/workflows/build-vcpkg.yml

@@ -21,6 +21,8 @@ jobs:
         path: blint-db
     - name: Install dependencies
       run: |
+        [ -d venv ] || python3 -m venv venv
+        source venv/bin/activate
         python -m pip install --upgrade pip
         pip install setuptools wheel twine build
         cd blint-db && python -m pip install .
@@ -32,9 +34,10 @@ jobs:
     - name: Build and upload db
       run: |
         mkdir -p temp
+        source venv/bin/activate
         cd blint-db/
-        python blint-db/blint_db/cli.py -Z2
-        python ./.oras/orasclient.py
+        python3 blint-db/blint_db/cli.py --clean-start -Z2
+        python3 ./.oras/orasclient.py
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         GITHUB_USERNAME: ${{ github.actor }}

.oras/orasclient.py

@@ -3,7 +3,7 @@
 
 client = oras.client.OrasClient()
 
-token = os.getenv("GITHUB_USERNAME", "")
+token = os.getenv("GITHUB_TOKEN", "")
 username = os.getenv("GITHUB_USERNAME", "")
 
 

.vscode/launch.json

@@ -19,7 +19,8 @@
             "request": "launch",
             "program": "${workspaceFolder}/blint_db/cli.py",
             "console": "integratedTerminal",
-            "args": "-Z1"
+            // "preLaunchTask": "Run Cleanup Script",
+            "args": ["-f", "-Z1"]
         },
         {
             "name": "test nodejs sbom deep",
@@ -38,7 +39,7 @@
             "request": "launch",
             "program": "${workspaceFolder}/blint_db/cli.py",
             "console": "integratedTerminal",
-            "preLaunchTask": "Run Cleanup Script",
+            // "preLaunchTask": "Run Cleanup Script",
             "args": "-Z2"
         }
     ]

blint_db/cli.py

@@ -1,5 +1,6 @@
 import os
 import argparse
+import sqlite3
 from concurrent import futures
 
 from blint_db import BLINTDB_LOCATION, COMMON_CONNECTION
@@ -60,7 +61,6 @@ def arguments_parser():
     parser.add_argument(
         "--clean-start",
         dest="clean",
-        default=False,
         action="store_true",
         help="Resets the database before starting a new build",
     )
@@ -69,9 +69,17 @@ def arguments_parser():
         "--few-packages",
         dest="test_mode",
         action="store_true",
-        help="Set meson to build fewer projects, helpful for debugging",
+        help="Set pkg managers to build fewer projects, helpful for debugging",
     )
 
+    # parser.add_argument(
+    #     "-R",
+    #     "--reuse-old-db",
+    #     dest="reuse",
+    #     action="store_true",
+    #     help="when set does not create a new database"
+    # )
+
     return parser.parse_args()
 
 def reset_and_backup():
@@ -96,8 +104,10 @@ def meson_add_blint_bom_process(test_mode=False):
             print(f"Ran complete for {project_name} and we found {len(executables)}")
 
 
-def vcpkg_add_blint_bom_process():
+def vcpkg_add_blint_bom_process(test_mode=False):
     projects_list = get_vcpkg_projects()
+    if test_mode:
+        projects_list = projects_list[:10]
     count = 0
     for project_name in projects_list:
         executables = mt_vcpkg_blint_db_build(project_name)
@@ -108,12 +118,6 @@ def vcpkg_add_blint_bom_process():
             reset_and_backup()
             count = 0
 
-    # with futures.ProcessPoolExecutor(max_workers=1) as executor:
-    #     for project_name, executables in zip(
-    #         projects_list, executor.map(mt_vcpkg_blint_db_build, projects_list)
-    #     ):
-    #         print(f"Ran complete for {project_name} and we found {len(executables)}")
-
 
 def main():
 
@@ -127,10 +131,11 @@ def main():
         meson_add_blint_bom_process(args["test_mode"])
 
     if args["vcpkg"]:
-        vcpkg_add_blint_bom_process()
+        vcpkg_add_blint_bom_process(args["test_mode"])
 
     if COMMON_CONNECTION:
         reset_and_backup()
+        print("Build Completed Saved Database")
 
 
 if __name__ == "__main__":

blint_db/config.py

@@ -10,7 +10,7 @@
 
 DELIMETER_BOM = "~~"
 # variables
-DEBUG_MODE = True
+DEBUG_MODE = False
 # constants
 TEMP_PATH = Path.cwd() / "temp"
 WRAPDB_LOCATION = TEMP_PATH / "wrapdb"
@@ -28,5 +28,5 @@
 
 SQLITE_TIMEOUT = 20.0
 
-# COMMON_CONNECTION = None
-COMMON_CONNECTION = sqlite3.connect(":memory:")
+COMMON_CONNECTION = None
+# COMMON_CONNECTION = sqlite3.connect(":memory:")

blint_db/handlers/sqlite_handler.py

@@ -8,6 +8,7 @@
                       SQLITE_TIMEOUT)
 
 
+
 def use_existing_connection(connection=None):  # Decorator now accepts connection
     """
     Decorator to use an existing connection when BLINTDB_LOCATION is ':memory:'.
@@ -18,7 +19,7 @@ def decorator(func):
         def wrapper(*args, **kwargs):
             if connection:
                 with closing(connection.cursor()) as c:
-                    if len(args) > 1:  # Statement and arguments provided
+                    if len(args) > 1:
                         c.execute(args[0], args[1])
                     else:
                         c.execute(args[0])
@@ -117,7 +118,9 @@ def create_database():
 
 
 def clear_sqlite_database():
-    os.remove(BLINTDB_LOCATION)
+    if os.path.exists(BLINTDB_LOCATION):
+        if os.path.isfile(BLINTDB_LOCATION):
+            os.remove(BLINTDB_LOCATION)
 
 
 def store_sbom_in_sqlite(purl, sbom):
@@ -196,7 +199,3 @@ def _fetch_infunc_row(infunc):
     execute_statement(
         "INSERT INTO BinariesExports (bid, eid) VALUES (?, ?)", (bid, eid)
     )
-
-
-if not os.path.exists(BLINTDB_LOCATION):
-    create_database()