Onnx build (#4)

embedded the ONNX runtime into the build and defined the `surml` layout.

.github/workflows/surrealml_core_deployment.yml

@@ -4,9 +4,6 @@ on:
   push:
     branches:
       - main
-  pull_request:
-    branches:
-      - main
 
 jobs:
   post_merge_job:

.github/workflows/surrealml_core_test.yml

@@ -1,10 +1,11 @@
 name: Rust Test for surrealml-core on Pull Request
 
 on:
-  pull_request: {}
+  pull_request:
+    types: [opened, reopened, synchronize]
 
 jobs:
-  build_and_test:
+  test_core:
     runs-on: ubuntu-latest
 
     steps:

.github/workflows/surrealml_deployment.yml

@@ -0,0 +1,186 @@
+name: cross-build
+
+on:
+  push:
+    branches:
+      - main
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+
+  wait-for-other-workflow:
+    name: Wait for Other Workflow
+    runs-on: ubuntu-latest
+    steps:
+      - name: Wait for Other Workflow to Complete
+        run: "echo 'Waiting for other workflow to complete...'"
+
+  build: # Workflow credit to https://github.com/samuelcolvin/rtoml/blob/main/.github/workflows/ci.yml
+    if: github.event.pull_request.merged == true
+    name: >
+      build ${{ matrix.python-version }} on ${{ matrix.platform || matrix.os }}
+      (${{ matrix.alt_arch_name || matrix.arch }})
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu, macos, windows]
+        python-version: ["cp310", "pp37", "pp38", "pp39"]
+        arch: [main, alt]
+        include:
+          - os: ubuntu
+            platform: linux
+          - os: windows
+            ls: dir
+          - os: macos
+            arch: alt
+            alt_arch_name: "arm64 universal2"
+        exclude:
+          - os: macos
+            python-version: "pp37"
+            arch: alt
+          - os: macos
+            python-version: "pp38"
+            arch: alt
+          - os: macos
+            python-version: "pp39"
+            arch: alt
+    runs-on: ${{ format('{0}-latest', matrix.os) }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: set up python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+
+      - name: set up rust
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: stable
+
+      - name: update pip version
+        run: |
+          pip install requests
+          python get_latest_version.py
+
+      - name: Setup Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          key: ${{ matrix.alt_arch_name }}
+
+      - run: rustup target add aarch64-apple-darwin
+        if: matrix.os == 'macos'
+
+      - run: rustup toolchain install stable-i686-pc-windows-msvc
+        if: matrix.os == 'windows'
+
+      - run: rustup target add i686-pc-windows-msvc
+        if: matrix.os == 'windows'
+
+      - name: Get pip cache dir
+        id: pip-cache
+        if: matrix.os != 'windows'
+        run: |
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+
+      - name: Get pip cache dir
+        id: pip-cache-win
+        if: matrix.os == 'windows'
+        run: |
+          "dir=$(pip cache dir)" >> $env:GITHUB_OUTPUT
+
+      - name: Cache python dependencies
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.pip-cache.outputs.dir || steps.pip-cache-win.outputs.dir }}
+          key: ${{ runner.os }}-pip-${{ matrix.python-version }}
+
+      - name: install python dependencies
+        run: pip install -U setuptools wheel twine cibuildwheel platformdirs
+
+      - name: Display cibuildwheel cache dir
+        id: cibuildwheel-cache
+        run: |
+          from platformdirs import user_cache_path
+          import os
+          with open(os.getenv('GITHUB_OUTPUT'), 'w') as f:
+            f.write(f"dir={str(user_cache_path(appname='cibuildwheel', appauthor='pypa'))}")
+        shell: python
+
+      - name: Cache cibuildwheel tools
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.cibuildwheel-cache.outputs.dir }}
+          key: ${{ runner.os }}-cibuildwheel-${{ matrix.python-version }}
+
+      - name: Install LLVM and Clang # required for bindgen to work, see https://github.com/rust-lang/rust-bindgen/issues/1797
+        uses: KyleMayes/install-llvm-action@v1
+        if: runner.os == 'Windows'
+        with:
+          version: "11.0"
+          directory: ${{ runner.temp }}/llvm
+
+      - name: Set LIBCLANG_PATH
+        run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV
+        if: runner.os == 'Windows'
+
+      - name: build_sdist
+        if: matrix.os == 'ubuntu' && matrix.python-version == 'cp310'
+        run: |
+          pip install maturin build
+          python -m build --sdist -o wheelhouse
+      - name: build ${{ matrix.platform || matrix.os }} binaries
+        run: cibuildwheel --output-dir wheelhouse
+        env:
+          CIBW_BUILD: "${{ matrix.python-version }}-*"
+          # rust doesn't seem to be available for musl linux on i686
+          CIBW_SKIP: "*-musllinux_i686"
+          # we build for "alt_arch_name" if it exists, else 'auto'
+          CIBW_ARCHS: ${{ matrix.alt_arch_name || 'auto' }}
+          CIBW_ENVIRONMENT: 'PATH="$HOME/.cargo/bin:$PATH" CARGO_TERM_COLOR="always"'
+          CIBW_ENVIRONMENT_WINDOWS: 'PATH="$UserProfile\.cargo\bin;$PATH"'
+          CIBW_BEFORE_BUILD: rustup show
+          CIBW_BEFORE_BUILD_LINUX: >
+            curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain=stable --profile=minimal -y &&
+            rustup show
+          CIBW_BUILD_VERBOSITY: 1
+
+      - run: ${{ matrix.ls || 'ls -lh' }} wheelhouse/
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: wheelhouse
+
+  release:
+    if: github.event.pull_request.merged == true
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: wheels
+          path: wheelhouse
+
+      - name: Install twine
+        run: python -m pip install --upgrade twine
+
+      - name: Create pypirc file
+        run: |
+          echo "[distutils]" > ~/.pypirc
+          echo "index-servers =" >> ~/.pypirc
+          echo "    pypi" >> ~/.pypirc
+          echo "" >> ~/.pypirc
+          echo "[pypi]" >> ~/.pypirc
+          echo "username: __token__" >> ~/.pypirc
+          echo "password: \${{ secrets.PYPI_TOKEN }}" >> ~/.pypirc
+        env:
+          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+
+      - name: Publish to PyPI
+        run: twine upload wheelhouse/*
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}

.github/workflows/surrealml_test.yml

@@ -1,10 +1,11 @@
 name: Rust Test for surrealml on Pull Request
 
 on:
-  pull_request: {}
+  pull_request:
+    types: [opened, reopened, synchronize]
 
 jobs:
-  build_and_test:
+  test_transport:
     runs-on: ubuntu-latest
 
     steps:

Cargo.toml

@@ -2,6 +2,7 @@
 name = "surrealml"
 version = "0.1.0"
 edition = "2021"
+
 description = "A machine learning python library written in Rust for SurrealDB"
 license-file = "LICENSE"
 
@@ -13,16 +14,16 @@ uuid = { version = "1.4.1", features = ["v4"] }
 once_cell = "1.18.0"
 ndarray = "0.15.6"
 hyper = { version = "0.14.27", features = ["full"] }
-tokio = { version = "1.32.0", features = ["full"] }
-futures-util = "0.3.28"
-
+tokio = { version = "1.34.0", features = ["full"] }
+base64 = "0.13"
 surrealml-core = { path = "./modules/utils" }
 
 [dev-dependencies]
 axum = "0.6.20"
 serde = "1.0.183"
 serde_json = "1.0.105"
 actix-web = "4.3.1"
+futures-util = "0.3.29"
 
 [lib]
 name = "surrealml"

README.md

@@ -209,6 +209,64 @@ print(new_file.buffered_compute({
 }))
 ```
 
+### Uploading our model to SurrealDB
+
+We can upload our trained model with the following code:
+
+```python
+url = "http://0.0.0.0:8000/ml/import"
+SurMlFile.upload(
+    path="./linear_test.surml", 
+    url=url, 
+    chunk_size=36864, 
+    namespace="test", 
+    database="test", 
+    username="root", 
+    password="root"
+)
+```
+
+### Running SurrealQL operations against our trained model
+
+With this, we can perform SQL statements in our database. To test this, we can create the following rows:
+
+```sql
+CREATE house_listing SET squarefoot_col = 500.0, num_floors_col = 1.0;
+CREATE house_listing SET squarefoot_col = 1000.0, num_floors_col = 2.0;
+CREATE house_listing SET squarefoot_col = 1500.0, num_floors_col = 3.0;
+
+SELECT * FROM (
+		SELECT 
+			*, 
+			ml::house-price-prediction<0.0.1>({ 
+				squarefoot: squarefoot_col, 
+				num_floors: num_floors_col 
+			}) AS price_prediction 
+		FROM house_listing
+	) 
+	WHERE price_prediction > 177206.21875;
+```
+
+What is happening here is that we are feeding the columns from the table `house_listing` into a model we uploaded 
+called `house-price-prediction` with a version of `0.0.1`. We then get the results of that trained ML model as the column 
+`price_prediction`. We then use the calculated predictions to filter the rows giving us the following result:
+
+```json
+[
+	{
+		"id": "house_listing:7bo0f35tl4hpx5bymq5d",
+		"num_floors_col": 3,
+		"price_prediction": 406534.75,
+		"squarefoot_col": 1500
+	},
+	{
+		"id": "house_listing:8k2ttvhp2vh8v7skwyie",
+		"num_floors_col": 2,
+		"price_prediction": 291870.5,
+		"squarefoot_col": 1000
+	}
+]
+```
 
 ### Loading our `.surml` file in Rust
 

modules/utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "surrealml-core"
-version = "0.0.1"
+version = "0.0.2"
 edition = "2021"
 build = "./build.rs"
 description = "The core machine learning library for SurrealML that enables SurrealDB to store and load ML models"

modules/utils/build.rs

@@ -15,13 +15,6 @@ fn main() {
         .status()
         .expect("failed to execute process");
 
-    // move the onnx lib to the right place
-    // let _ = Command::new("sh")
-    //     .arg("-c")
-    //     .arg("cd onnx_driver && cargo build")
-    //     .status()
-    //     .expect("failed to execute process");
-
     // let _ = Command::new("sh")
     //     .arg("-c")
     //     .arg("cd ../onnx_driver && cargo build")

modules/utils/src/execution/onnx_environment.rs

@@ -5,19 +5,19 @@ use std::sync::Arc;
 
 
 // Compiles the ONNX module into the rust binary.
-
-// #[cfg(target_os = "macos")]
-// pub static LIB_BYTES: &'static [u8] = include_bytes!("../../../onnx_driver/target/debug/libonnxruntime.dylib");
-
-#[cfg(target_os = "macos")]
+#[cfg(all(target_os = "macos", not(doc)))]
 pub static LIB_BYTES: &'static [u8] = include_bytes!("../../onnx_driver/target/debug/libonnxruntime.dylib");
 
-#[cfg(any(target_os = "linux", target_os = "android"))]
+#[cfg(all(any(target_os = "linux", target_os = "android"), not(doc)))]
 pub static LIB_BYTES: &'static [u8] = include_bytes!("../../onnx_driver/target/debug/libonnxruntime.so");
 
-#[cfg(target_os = "windows")]
+#[cfg(all(target_os = "windows", not(doc)))]
 pub static LIB_BYTES: &'static [u8] = include_bytes!("../../onnx_driver/target/debug/libonnxruntime.dll");
 
+// Fallback for documentation and other targets
+#[cfg(any(doc, not(any(target_os = "macos", target_os = "linux", target_os = "android", target_os = "windows"))))]
+pub static LIB_BYTES: &'static [u8] = &[];
+
 
 // the ONNX environment which loads the library
 pub static ENVIRONMENT: Lazy<Arc<Environment>> = Lazy::new(|| {