diff --git a/.github/workflows/build-meson-tst.yml b/.github/workflows/build-meson-tst.yml index 0e3ae1f..e1bec59 100644 --- a/.github/workflows/build-meson-tst.yml +++ b/.github/workflows/build-meson-tst.yml @@ -21,6 +21,8 @@ jobs: path: blint-db - name: Install dependencies run: | + [ -d venv ] || python3 -m venv venv + source venv/bin/activate python -m pip install --upgrade pip pip install setuptools wheel twine build cd blint-db && python -m pip install . @@ -32,8 +34,9 @@ jobs: - name: Build and upload db run: | mkdir -p temp + source venv/bin/activate cd blint-db/ - python blint_db/cli.py -f -Z1 + python blint_db/cli.py --clean-start -f -Z1 python ./.oras/orasclient.py env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-meson.yml b/.github/workflows/build-meson.yml index 41b223c..b9e8587 100644 --- a/.github/workflows/build-meson.yml +++ b/.github/workflows/build-meson.yml @@ -21,6 +21,8 @@ jobs: path: blint-db - name: Install dependencies run: | + [ -d venv ] || python3 -m venv venv + source venv/bin/activate python -m pip install --upgrade pip pip install setuptools wheel twine build cd blint-db && python -m pip install . @@ -32,8 +34,9 @@ jobs: - name: Build and upload db run: | mkdir -p temp + source venv/bin/activate cd blint-db/ - python blint_db/cli.py -Z1 + python blint_db/cli.py --clean-start -Z1 python ./.oras/orasclient.py env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-vcpkg-tst.yml b/.github/workflows/build-vcpkg-tst.yml new file mode 100644 index 0000000..debc5b0 --- /dev/null +++ b/.github/workflows/build-vcpkg-tst.yml @@ -0,0 +1,43 @@ +name: Build and Upload vcpkg blint-db 0.1.0 + +on: + schedule: + - cron: "0 */12 * * *" + workflow_dispatch: +env: + REGISTRY: ghcr.io + IMAGE_NAME: appthreat/blintdb + +jobs: + builder-vcpkg-tst: + runs-on: ['self-hosted', 'ubuntu', 'arm64'] + permissions: + contents: write + packages: write + steps: + - uses: actions/checkout@v4 + with: + repository: AppThreat/blint-db + path: blint-db + - name: Install dependencies + run: | + [ -d venv ] || python3 -m venv venv + source venv/bin/activate + python -m pip install --upgrade pip + pip install setuptools wheel twine build + cd blint-db && python -m pip install . + - name: Clean up previous + run: | + cd blint-db + rm -f info.log + rm -f blint.db + - name: Build and upload db + run: | + mkdir -p temp + source venv/bin/activate + cd blint-db/ + python3 blint-db/blint_db/cli.py --clean-start -f -Z2 + python3 ./.oras/orasclient.py + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} diff --git a/.github/workflows/build-vcpkg.yml b/.github/workflows/build-vcpkg.yml index 707b5db..970de48 100644 --- a/.github/workflows/build-vcpkg.yml +++ b/.github/workflows/build-vcpkg.yml @@ -21,6 +21,8 @@ jobs: path: blint-db - name: Install dependencies run: | + [ -d venv ] || python3 -m venv venv + source venv/bin/activate python -m pip install --upgrade pip pip install setuptools wheel twine build cd blint-db && python -m pip install . @@ -32,9 +34,10 @@ jobs: - name: Build and upload db run: | mkdir -p temp + source venv/bin/activate cd blint-db/ - python blint-db/blint_db/cli.py -Z2 - python ./.oras/orasclient.py + python3 blint-db/blint_db/cli.py --clean-start -Z2 + python3 ./.oras/orasclient.py env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} diff --git a/.oras/orasclient.py b/.oras/orasclient.py index 9955464..1feb33d 100644 --- a/.oras/orasclient.py +++ b/.oras/orasclient.py @@ -3,7 +3,7 @@ client = oras.client.OrasClient() -token = os.getenv("GITHUB_USERNAME", "") +token = os.getenv("GITHUB_TOKEN", "") username = os.getenv("GITHUB_USERNAME", "") diff --git a/.vscode/launch.json b/.vscode/launch.json index a13aa40..e0ad65b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -19,7 +19,8 @@ "request": "launch", "program": "${workspaceFolder}/blint_db/cli.py", "console": "integratedTerminal", - "args": "-Z1" + // "preLaunchTask": "Run Cleanup Script", + "args": ["-f", "-Z1"] }, { "name": "test nodejs sbom deep", @@ -38,7 +39,7 @@ "request": "launch", "program": "${workspaceFolder}/blint_db/cli.py", "console": "integratedTerminal", - "preLaunchTask": "Run Cleanup Script", + // "preLaunchTask": "Run Cleanup Script", "args": "-Z2" } ] diff --git a/blint_db/cli.py b/blint_db/cli.py index 2786f44..cdf4747 100644 --- a/blint_db/cli.py +++ b/blint_db/cli.py @@ -1,5 +1,6 @@ import os import argparse +import sqlite3 from concurrent import futures from blint_db import BLINTDB_LOCATION, COMMON_CONNECTION @@ -60,7 +61,6 @@ def arguments_parser(): parser.add_argument( "--clean-start", dest="clean", - default=False, action="store_true", help="Resets the database before starting a new build", ) @@ -69,9 +69,17 @@ def arguments_parser(): "--few-packages", dest="test_mode", action="store_true", - help="Set meson to build fewer projects, helpful for debugging", + help="Set pkg managers to build fewer projects, helpful for debugging", ) + # parser.add_argument( + # "-R", + # "--reuse-old-db", + # dest="reuse", + # action="store_true", + # help="when set does not create a new database" + # ) + return parser.parse_args() def reset_and_backup(): @@ -96,8 +104,10 @@ def meson_add_blint_bom_process(test_mode=False): print(f"Ran complete for {project_name} and we found {len(executables)}") -def vcpkg_add_blint_bom_process(): +def vcpkg_add_blint_bom_process(test_mode=False): projects_list = get_vcpkg_projects() + if test_mode: + projects_list = projects_list[:10] count = 0 for project_name in projects_list: executables = mt_vcpkg_blint_db_build(project_name) @@ -108,12 +118,6 @@ def vcpkg_add_blint_bom_process(): reset_and_backup() count = 0 - # with futures.ProcessPoolExecutor(max_workers=1) as executor: - # for project_name, executables in zip( - # projects_list, executor.map(mt_vcpkg_blint_db_build, projects_list) - # ): - # print(f"Ran complete for {project_name} and we found {len(executables)}") - def main(): @@ -127,10 +131,11 @@ def main(): meson_add_blint_bom_process(args["test_mode"]) if args["vcpkg"]: - vcpkg_add_blint_bom_process() + vcpkg_add_blint_bom_process(args["test_mode"]) if COMMON_CONNECTION: reset_and_backup() + print("Build Completed Saved Database") if __name__ == "__main__": diff --git a/blint_db/config.py b/blint_db/config.py index 0f2a657..852a565 100644 --- a/blint_db/config.py +++ b/blint_db/config.py @@ -10,7 +10,7 @@ DELIMETER_BOM = "~~" # variables -DEBUG_MODE = True +DEBUG_MODE = False # constants TEMP_PATH = Path.cwd() / "temp" WRAPDB_LOCATION = TEMP_PATH / "wrapdb" @@ -28,5 +28,5 @@ SQLITE_TIMEOUT = 20.0 -# COMMON_CONNECTION = None -COMMON_CONNECTION = sqlite3.connect(":memory:") +COMMON_CONNECTION = None +# COMMON_CONNECTION = sqlite3.connect(":memory:") diff --git a/blint_db/handlers/sqlite_handler.py b/blint_db/handlers/sqlite_handler.py index 26b5ec2..0621a03 100644 --- a/blint_db/handlers/sqlite_handler.py +++ b/blint_db/handlers/sqlite_handler.py @@ -8,6 +8,7 @@ SQLITE_TIMEOUT) + def use_existing_connection(connection=None): # Decorator now accepts connection """ Decorator to use an existing connection when BLINTDB_LOCATION is ':memory:'. @@ -18,7 +19,7 @@ def decorator(func): def wrapper(*args, **kwargs): if connection: with closing(connection.cursor()) as c: - if len(args) > 1: # Statement and arguments provided + if len(args) > 1: c.execute(args[0], args[1]) else: c.execute(args[0]) @@ -117,7 +118,9 @@ def create_database(): def clear_sqlite_database(): - os.remove(BLINTDB_LOCATION) + if os.path.exists(BLINTDB_LOCATION): + if os.path.isfile(BLINTDB_LOCATION): + os.remove(BLINTDB_LOCATION) def store_sbom_in_sqlite(purl, sbom): @@ -196,7 +199,3 @@ def _fetch_infunc_row(infunc): execute_statement( "INSERT INTO BinariesExports (bid, eid) VALUES (?, ?)", (bid, eid) ) - - -if not os.path.exists(BLINTDB_LOCATION): - create_database()