Python engine adapters

.github/workflows/surrealml_core_deployment.yml

@@ -4,6 +4,10 @@ on:
   push:
     branches:
       - main
+    paths:
+      - 'src/**'
+      - 'build.rs'
+      - 'Cargo.toml'
 
 jobs:
   post_merge_job:
@@ -15,6 +19,6 @@ jobs:
 
     - uses: katyo/publish-crates@v2
       with:
-          path: './modules/utils'
+          path: './modules/core'
           args: --no-verify
           registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}

.github/workflows/surrealml_core_test.yml

@@ -1,4 +1,4 @@
-name: Rust Test for surrealml-core on Pull Request
+name: Run tests on Pull Request
 
 on:
   pull_request:
@@ -20,5 +20,38 @@ jobs:
         toolchain: stable
         override: true
 
-    - name: Run Unit Tests
-      run: cd modules/utils && cargo test
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.11'
+
+    - name: Pre-test Setup
+      run: |
+        python3 -m venv venv
+        source venv/bin/activate
+        pip install --upgrade pip
+        pip install -r requirements.txt
+
+        # build the local version of the core module to be loaded into python
+        echo "Building local version of core module"
+
+        pip install .
+
+        python ./tests/scripts/ci_local_build.py
+        echo "Local build complete"
+
+        # train the models for the tests
+        python ./tests/scripts/build_assets.py
+        deactivate
+
+    - name: Run Python Unit Tests
+      run: |
+        source venv/bin/activate
+        python -m unittest discover
+        deactivate
+
+    - name: Run Core Unit Tests
+      run: cd modules/core && cargo test
+
+    - name: Run HTTP Transfer Tests
+      run: cargo test

.gitignore

@@ -6,13 +6,18 @@ Cargo.lock
 
 build/
 dist/
+__pycache__/
 venv/
 .idea/
 surrealml.egg-info/
 .vscode/
 ./modules/utils/target/
-modules/utils/target/
+modules/core/target/
 ./modules/onnx_driver/target/
 modules/onnx_driver/target/
 surrealdb_build/
-modules/utils/onnx_driver/
+modules/core/onnx_driver/
+*.so
+surrealml/rust_surrealml.cpython-310-darwin.so
+.surmlcache
+modules/core/model_stash/

Cargo.toml

@@ -16,7 +16,7 @@ ndarray = "0.15.6"
 hyper = { version = "0.14.27", features = ["full"] }
 tokio = { version = "1.34.0", features = ["full"] }
 base64 = "0.13"
-surrealml-core = { path = "./modules/utils" }
+surrealml-core = { path = "./modules/core" }
 
 [dev-dependencies]
 axum = "0.6.20"

README.md

@@ -16,12 +16,24 @@ SurrealML is a feature that allows you to store trained machine learning models
 
 ## Installation
 
-To install SurrealML, make sure you have Python installed. Then, install the SurrealML library and either PyTorch or SKLearn, based on your model choice. You can install these using pip:
+To install SurrealML, make sure you have Python installed. Then, install the `SurrealML` library and either `PyTorch` or 
+`SKLearn`, based on your model choice. You can install the package with both `PyTorch` and `SKLearn` with the command
+below:
 
 ```
-pip install surrealml
-pip install torch  # If using PyTorch
-pip install scikit-learn  # If using SKLearn
+pip install "git+https://github.com/surrealdb/surrealml#egg=surrealml[sklearn,torch]"
+```
+
+If you want to use `SurrealML` with `sklearn` you will need the following installation:
+
+```bash
+pip install "git+https://github.com/surrealdb/surrealml#egg=surrealml[sklearn]"
+```
+
+For `PyTorch`:
+
+```bash
+pip install "git+https://github.com/surrealdb/surrealml#egg=surrealml[torch]"
 ```
 
 After that, you can train your model and save it in the SurrealML format.
@@ -47,32 +59,38 @@ will also be able to load your sk-learn models in Rust and run them meaning you
 Saving a model is as simple as the following:
 
 ```python
-import numpy as np
-from sklearn.ensemble import RandomForestClassifier
-from surrealml import SurMlFile
-
-num_classes = 2
-X = np.random.rand(100, 28)
-y = np.random.randint(num_classes, size=100)
-
-skl_model = RandomForestClassifier(n_estimators=10, max_depth=10)
-skl_model.fit(X, y)
-test_file = SurMlFile(model=skl_model, name="random forrest classifier", inputs=X, sklearn=True)
-test_file.save("./test_forrest.surml")
-
-# load model and execute a calculation
-random_floats = list(np.random.rand(28))
-test_load = SurMlFile.load("./test_forrest.surml")
-print(test_load.raw_compute(random_floats, [1, -1]))
-```
+from sklearn.linear_model import LinearRegression
+from surrealml import SurMlFile, Engine
+from surrealml.model_templates.datasets.house_linear import HOUSE_LINEAR # click on this HOUSE_LINEAR to see the data
 
-## Python tutorial using Pytorch
+# train the model
+model = LinearRegression()
+model.fit(HOUSE_LINEAR["inputs"], HOUSE_LINEAR["outputs"])
+
+# package and save the model
+file = SurMlFile(model=model, name="linear", inputs=HOUSE_LINEAR["inputs"], engine=Engine.SKLEARN)
+
+# add columns in the order of the inputs to map dictionaries passed in to the model
+file.add_column("squarefoot")
+file.add_column("num_floors")
+
+# add normalisers for the columns
+file.add_normaliser("squarefoot", "z_score", HOUSE_LINEAR["squarefoot"].mean(), HOUSE_LINEAR["squarefoot"].std())
+file.add_normaliser("num_floors", "z_score", HOUSE_LINEAR["num_floors"].mean(), HOUSE_LINEAR["num_floors"].std())
+file.add_output("house_price", "z_score", HOUSE_LINEAR["outputs"].mean(), HOUSE_LINEAR["outputs"].std())
 
-To carry out this example we need the following:
+# save the file
+file.save(path="./linear.surml")
 
-- pytorch (pip installed for python)
-- numpy
-- surrealml
+# load the file
+new_file = SurMlFile.load(path="./linear.surml", engine=Engine.SKLEARN)
+
+# Make a prediction (both should be the same due to the perfectly correlated example data)
+print(new_file.buffered_compute(value_map={"squarefoot": 5, "num_floors": 6}))
+print(new_file.raw_compute(input_vector=[5, 6]))
+```
+
+## Python tutorial using Pytorch
 
 First we need to have one script where we create and store the model. In this example we will merely do a linear regression model
 to predict the house price using the number of floors and the square feet.
@@ -181,9 +199,9 @@ test_inputs = torch.stack([test_squarefoot, test_num_floors], dim=1)
 We can now wrap our model in the `SurMlFile` object with the following code:
 
 ```python
-from surrealml import SurMlFile
+from surrealml import SurMlFile, Engine
 
-file = SurMlFile(model=model, name="House Price Prediction", inputs=test_inputs)
+file = SurMlFile(model=model, name="linear", inputs=inputs[:1], engine=Engine.PYTORCH)
 ```
 
 The name is optional but the inputs and model are essential. We can now add some meta data to the file such as our inputs and outputs with the following code, however meta data is not essential, it just helps with some types of computation:
@@ -215,29 +233,13 @@ file.save("./test.surml")
 If you have followed the previous steps you should have a `.surml` file saved with all our meta data. We load it with the following code:
 
 ```python
-from surrealml import SurMlFile
+from surrealml import SurMlFile, Engine
 
-new_file = SurMlFile.load("./test.surml")
+new_file = SurMlFile.load("./test.surml", engine=Engine.PYTORCH)
 ```
 
 Our model is now loaded. We can now perform computations.
 
-### Raw computation in Python
-
-If you haven't put any meta data into the file then don't worry, we can just perform a raw computation with the following command:
-
-```python
-print(new_file.raw_compute([1.0, 2.0]))
-```
-
-This will just give you the outcome from the model. If you have put in the metadata then we can perform a buffered computation.
-We can also input dimensions for the raw compute which will perform a batch computation. This can be done with the
-following code:
-
-```python
-print(new_file.raw_compute([1.0, 2.0, 3.0, 4.0]), dims=[2, 2])
-```
-
 ### Buffered computation in Python
 
 This is where the computation utilises the data in the header. We can do this by merely passing in a dictionary as seen below:

build.rs

@@ -22,16 +22,16 @@ fn main() {
 
 	// remove ./modules/utils/target folder if there
 	let _ =
-		std::fs::remove_dir_all(Path::new("modules").join("utils").join("target")).unwrap_or(());
+		std::fs::remove_dir_all(Path::new("modules").join("core").join("target")).unwrap_or(());
 
 	// create the target module folder for the utils module
-	let _ = std::fs::create_dir(Path::new("modules").join("utils").join("target"));
-	let _ = std::fs::create_dir(Path::new("modules").join("utils").join("target").join(profile));
+	let _ = std::fs::create_dir(Path::new("modules").join("core").join("target"));
+	let _ = std::fs::create_dir(Path::new("modules").join("core").join("target").join(profile));
 
 	// copy target folder to modules/utils/target profile for the utils modules
 	std::fs::copy(
 		Path::new("target").join(profile).join(target_lib),
-		Path::new("modules").join("utils").join("target").join(profile).join(target_lib),
+		Path::new("modules").join("core").join("target").join(profile).join(target_lib),
 	)
 	.unwrap();
 }

modules/utils/Cargo.toml → modules/core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "surrealml-core"
-version = "0.0.7"
+version = "0.0.8"
 edition = "2021"
 build = "./build.rs"
 description = "The core machine learning library for SurrealML that enables SurrealDB to store and load ML models"