From 1ba2c2ce99e63cd844b4157d5b9dd9d0a39323f7 Mon Sep 17 00:00:00 2001 From: FL03 Date: Thu, 8 Dec 2022 15:35:50 -0600 Subject: [PATCH 001/118] Update Signed-off-by: FL03 --- concision/Cargo.toml | 2 +- derive/Cargo.toml | 6 +++--- macros/Cargo.toml | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 66b12381..34c7fb39 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -8,7 +8,7 @@ keywords = ["data-science", "scsys", "toolkit"] license = "Apache-2.0" name = "concision" repository = "https://github.com/scattered-systems/concision" -version = "0.1.11" # TODO - Update the cargo package version +version = "0.1.12" # TODO - Update the cargo package version [features] default = ["core"] diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 67252d46..38eed446 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -1,14 +1,14 @@ [package] -authors = ["FL03 "] +authors = ["FL03 (https://github.com/FL03)", "Scattered-Systems (https://github.com/scattered-systems)"] categories = [] description = "Derive macros created for Consision, a complete data-science toolkit written in Rust" edition = "2021" homepage = "https://scattered-systems.github.io/concision" -keywords = ["derive", "macros"] +keywords = ["derive", "macros", "scsys"] license = "Apache-2.0" name = "concision-derive" repository = "https://github.com/scattered-systems/concision" -version = "0.1.11" # TODO - Update the cargo package version +version = "0.1.12" # TODO - Update the cargo package version [lib] proc-macro = true diff --git a/macros/Cargo.toml b/macros/Cargo.toml index 75854d59..fc1165de 100644 --- a/macros/Cargo.toml +++ b/macros/Cargo.toml @@ -1,14 +1,14 @@ [package] -authors = ["FL03 "] +authors = ["FL03 (https://github.com/FL03)", "Scattered-Systems (https://github.com/scattered-systems)"] categories = [] description = "Procedural Macros created for Concision, a complete data-science toolkit written in Rust" edition = "2021" homepage = "https://scattered-systems.github.io/concision" -keywords = ["macros"] +keywords = ["macros", "scsys"] license = "Apache-2.0" name = "concision-macros" repository = "https://github.com/scattered-systems/concision" -version = "0.1.11" # TODO - Update the cargo package version +version = "0.1.12" # TODO - Update the cargo package version [lib] crate-type = ["cdylib", "rlib"] From 5a093c8cc3733f7ab36edd74982cd0b5b98e149c Mon Sep 17 00:00:00 2001 From: FL03 Date: Thu, 8 Dec 2022 15:42:31 -0600 Subject: [PATCH 002/118] Update Signed-off-by: FL03 --- concision/Cargo.toml | 4 ++-- concision/examples/simple.rs | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 34c7fb39..79619a63 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -31,5 +31,5 @@ serde = { features = ["derive"], version = "1.0.149" } serde_json = "1.0.89" strum = { features = ["derive"], version = "0.24.1" } -concision-derive = { features = [], optional = true, path = "../derive", version = "0.1.11" } -concision-macros = { features = [], optional = true, path = "../macros", version = "0.1.11" } +concision-derive = { features = [], optional = true, path = "../derive", version = "0.1.12" } +concision-macros = { features = [], optional = true, path = "../macros", version = "0.1.12" } diff --git a/concision/examples/simple.rs b/concision/examples/simple.rs index ecc8b1fc..f1604390 100644 --- a/concision/examples/simple.rs +++ b/concision/examples/simple.rs @@ -1,7 +1,15 @@ -use concision as cnc; +use concision::num::complex::{C, Complex}; +use std::ops::Mul; -fn main() { - let a = ""; +// Define a holomorphic function that squares its input. +fn square + Clone>(z: T) -> T { + z.clone() * z +} - println!("{:?}", a); +fn main() { + let c = C::from((1.0, 1.0)); + let res = square(c); + assert_eq!(res.clone(), C::from((0.0, 2.0))); + + println!("{:?}", res); } From 7aa4eede6d9a191c644b789c518f67cb6b56fe3f Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 25 Feb 2023 09:43:05 -0600 Subject: [PATCH 003/118] Update --- Cargo.toml | 12 ++++++++ concision/Cargo.toml | 28 +++++++++-------- concision/examples/simple.rs | 9 ++++-- concision/src/lib.rs | 16 ++++++++++ concision/src/num/complex.rs | 58 +++++++++++++++++++----------------- concision/src/num/mod.rs | 28 ++++------------- derive/Cargo.toml | 27 +++++++++-------- macros/Cargo.toml | 19 ++++++------ macros/src/lib.rs | 4 +-- 9 files changed, 111 insertions(+), 90 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 19128ea3..941b72d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,15 @@ +[workspace.package] +authors = ["FL03 (https://github.com/FL03)", "Scattered-Systems (https://github.com/scattered-systems)"] +categories = [] +description = "Concision is a complete data-science toolkit written in Rust" +edition = "2021" +homepage = "https://github.com/FL03/concision/wiki" +keywords = ["data-science", "scsys", "toolkit"] +license = "Apache-2.0" +readme = "README.md" +repository = "https://github.com/FL03/concision" +version = "0.1.12" # TODO - Update the cargo package version + [workspace] default-members = [ "concision" diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 79619a63..55ec7a55 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -1,14 +1,15 @@ [package] -authors = ["FL03 (https://github.com/FL03)", "Scattered-Systems (https://github.com/scattered-systems)"] -categories = [] -description = "Concision is a complete data-science toolkit written in Rust" -edition = "2021" -homepage = "https://scattered-systems.github.io/concision" -keywords = ["data-science", "scsys", "toolkit"] -license = "Apache-2.0" +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true name = "concision" -repository = "https://github.com/scattered-systems/concision" -version = "0.1.12" # TODO - Update the cargo package version +readme.workspace = true +repository.workspace = true +version.workspace = true [features] default = ["core"] @@ -26,10 +27,11 @@ test = true [build-dependencies] [dependencies] -scsys = { features = [], version = "0.1.36" } -serde = { features = ["derive"], version = "1.0.149" } -serde_json = "1.0.89" -strum = { features = ["derive"], version = "0.24.1" } +scsys = { features = [], version = "0.1.41" } +serde = { features = ["derive"], version = "1" } +serde_json = "1" +smart-default = "0.6" +strum = { features = ["derive"], version = "0.24" } concision-derive = { features = [], optional = true, path = "../derive", version = "0.1.12" } concision-macros = { features = [], optional = true, path = "../macros", version = "0.1.12" } diff --git a/concision/examples/simple.rs b/concision/examples/simple.rs index f1604390..536472e0 100644 --- a/concision/examples/simple.rs +++ b/concision/examples/simple.rs @@ -1,8 +1,11 @@ -use concision::num::complex::{C, Complex}; -use std::ops::Mul; + +extern crate concision; + +use concision::num::complex::C; +use concision::Numerical; // Define a holomorphic function that squares its input. -fn square + Clone>(z: T) -> T { +fn square(z: C) -> C { z.clone() * z } diff --git a/concision/src/lib.rs b/concision/src/lib.rs index 8bdef2a9..c113f60f 100644 --- a/concision/src/lib.rs +++ b/concision/src/lib.rs @@ -16,3 +16,19 @@ pub mod num; pub(crate) mod primitives; pub(crate) mod utils; + +use std::ops::{Add, Div, Mul, Sub}; + +/// [Numerical] is a basic trait describing numerical objects +pub trait Numerical: + Add + Div + Mul + Sub + Clone + Copy + Sized +{ +} + +impl Numerical for f32 {} + +impl Numerical for f64 {} + +impl Numerical for i64 {} + +impl Numerical for usize {} diff --git a/concision/src/num/complex.rs b/concision/src/num/complex.rs index e35ec9aa..f2a40a39 100644 --- a/concision/src/num/complex.rs +++ b/concision/src/num/complex.rs @@ -3,6 +3,7 @@ Contrib: FL03 Description: ... Summary ... */ +use crate::Numerical; use serde::{Deserialize, Serialize}; use std::{ convert::From, @@ -10,15 +11,15 @@ use std::{ }; // Define a trait for complex numbers. -pub trait Complex { - fn new(re: f64, im: f64) -> Self; - fn re(&self) -> f64; - fn im(&self) -> f64; +pub trait Complex { + fn build(re: T, im: T) -> Self; + fn re(&self) -> T; + fn im(&self) -> T; } // Implement the Complex trait for the tuple (f64, f64). -impl Complex for (f64, f64) { - fn new(re: f64, im: f64) -> Self { +impl Complex for (f64, f64) { + fn build(re: f64, im: f64) -> Self { (re, im) } @@ -32,51 +33,54 @@ impl Complex for (f64, f64) { } #[derive(Clone, Debug, Default, Deserialize, PartialEq, PartialOrd, Serialize)] -pub struct C { - pub re: f64, - pub im: f64, +pub struct C(T, T); + +impl C { + pub fn new(a: T, b: T) -> Self { + Self(a, b) + } } -impl Complex for C { - fn new(re: f64, im: f64) -> Self { - Self { re, im } +impl Complex for C { + fn build(re: T, im: T) -> Self { + Self(re, im) } - fn re(&self) -> f64 { - self.re + fn re(&self) -> T { + self.0 } - fn im(&self) -> f64 { - self.im + fn im(&self) -> T { + self.1 } } -impl From<(f64, f64)> for C { - fn from(data: (f64, f64)) -> Self { +impl From<(T, T)> for C { + fn from(data: (T, T)) -> Self { Self::new(data.0, data.1) } } -impl From for (f64, f64) { - fn from(data: C) -> (f64, f64) { - (data.re, data.im) +impl From> for (T, T) { + fn from(data: C) -> (T, T) { + (data.re(), data.im()) } } // Implement the Add and Mul traits for complex numbers. -impl Add for C { +impl Add for C { type Output = Self; fn add(self, other: Self) -> Self { - Self::from((self.re + other.re, self.im + other.im)) + Self::from((self.re() + other.re(), self.im() + other.im())) } } -impl Mul for C { +impl Mul for C { type Output = Self; fn mul(self, other: Self) -> Self { Self::from(( - self.re * other.re - self.im * other.im, - self.re * other.im + self.im * other.re, + self.re() * other.re() - self.im() * other.im(), + self.re() * other.im() + self.im() * other.re(), )) } } @@ -85,7 +89,7 @@ impl Mul for C { mod tests { use super::*; // Define a holomorphic function that squares its input. - fn square + Clone>(z: T) -> T { + fn square(z: C) -> C { z.clone() * z } diff --git a/concision/src/num/mod.rs b/concision/src/num/mod.rs index edc487f5..75c4ac2c 100644 --- a/concision/src/num/mod.rs +++ b/concision/src/num/mod.rs @@ -6,38 +6,20 @@ pub mod complex; -use std::ops::{Add, Div, Mul, Sub}; +use crate::Numerical; -pub enum Z { - Signed, - Unsigned, -} - -pub enum Numbers { - Integer, - Float, -} - -pub struct Point(T); - -pub trait BaseObject: Clone + Sized {} - -pub trait Numerical: - Add + Div + Mul + Sub + Sized -{ -} #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] -pub struct OrdPair>((Re, Re)); +pub struct OrdPair((Re, Re)); #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] -pub struct Re>(T); +pub struct Re(T); -impl> Re { +impl Re { pub fn new(data: T) -> Self { Self(data) } } #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] -pub struct Im>(Re); +pub struct Im(Re); diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 38eed446..ec0f2715 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -1,23 +1,24 @@ [package] -authors = ["FL03 (https://github.com/FL03)", "Scattered-Systems (https://github.com/scattered-systems)"] -categories = [] -description = "Derive macros created for Consision, a complete data-science toolkit written in Rust" -edition = "2021" -homepage = "https://scattered-systems.github.io/concision" -keywords = ["derive", "macros", "scsys"] -license = "Apache-2.0" +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true name = "concision-derive" -repository = "https://github.com/scattered-systems/concision" -version = "0.1.12" # TODO - Update the cargo package version +readme.workspace = true +repository.workspace = true +version.workspace = true [lib] proc-macro = true test = false [dependencies] -proc-macro2 = "1.0.47" -quote = "1.0.21" -syn = { features = ["full"], version = "1.0.105" } +proc-macro2 = "1" +quote = "1" +syn = { features = ["full"], version = "1" } [dev-dependencies.concision] -path = "../concision" \ No newline at end of file +path = "../concision" diff --git a/macros/Cargo.toml b/macros/Cargo.toml index fc1165de..15ccb9d8 100644 --- a/macros/Cargo.toml +++ b/macros/Cargo.toml @@ -1,14 +1,15 @@ [package] -authors = ["FL03 (https://github.com/FL03)", "Scattered-Systems (https://github.com/scattered-systems)"] -categories = [] -description = "Procedural Macros created for Concision, a complete data-science toolkit written in Rust" -edition = "2021" -homepage = "https://scattered-systems.github.io/concision" -keywords = ["macros", "scsys"] -license = "Apache-2.0" +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true name = "concision-macros" -repository = "https://github.com/scattered-systems/concision" -version = "0.1.12" # TODO - Update the cargo package version +readme.workspace = true +repository.workspace = true +version.workspace = true [lib] crate-type = ["cdylib", "rlib"] diff --git a/macros/src/lib.rs b/macros/src/lib.rs index bfc6ceac..a64e07bb 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1,6 +1,6 @@ /* - Appellation: concision-derive + Appellation: concision-macros Creator: FL03 Description: - Derive macros for concision, a robust data-science toolkit for creating powerful Rust apps. + Useful macros for data-centric workloads */ From a6193191a768b4e34164ecaf5ac92f99c85fc8c5 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 25 Feb 2023 09:43:18 -0600 Subject: [PATCH 004/118] Update --- concision/examples/simple.rs | 3 +-- concision/src/lib.rs | 8 +++++++- concision/src/math/factorials.rs | 1 - concision/src/num/mod.rs | 1 - 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/concision/examples/simple.rs b/concision/examples/simple.rs index 536472e0..fd4dc389 100644 --- a/concision/examples/simple.rs +++ b/concision/examples/simple.rs @@ -1,4 +1,3 @@ - extern crate concision; use concision::num::complex::C; @@ -13,6 +12,6 @@ fn main() { let c = C::from((1.0, 1.0)); let res = square(c); assert_eq!(res.clone(), C::from((0.0, 2.0))); - + println!("{:?}", res); } diff --git a/concision/src/lib.rs b/concision/src/lib.rs index c113f60f..b6fb5d22 100644 --- a/concision/src/lib.rs +++ b/concision/src/lib.rs @@ -21,7 +21,13 @@ use std::ops::{Add, Div, Mul, Sub}; /// [Numerical] is a basic trait describing numerical objects pub trait Numerical: - Add + Div + Mul + Sub + Clone + Copy + Sized + Add + + Div + + Mul + + Sub + + Clone + + Copy + + Sized { } diff --git a/concision/src/math/factorials.rs b/concision/src/math/factorials.rs index d438a06a..0a133213 100644 --- a/concision/src/math/factorials.rs +++ b/concision/src/math/factorials.rs @@ -37,7 +37,6 @@ where } } - pub fn factorial(data: usize) -> usize { match data { 0 | 1 => 1, diff --git a/concision/src/num/mod.rs b/concision/src/num/mod.rs index 75c4ac2c..ebaf255d 100644 --- a/concision/src/num/mod.rs +++ b/concision/src/num/mod.rs @@ -8,7 +8,6 @@ pub mod complex; use crate::Numerical; - #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] pub struct OrdPair((Re, Re)); From 4fd0a879491f4ff21b109356b6d78113d6ca8cec Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 25 Feb 2023 09:46:43 -0600 Subject: [PATCH 005/118] Update --- README.md | 8 +++++--- macros/Cargo.toml | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8a4e2a97..af80b615 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ # Concision -[![Clippy](https://github.com/scattered-systems/concision/actions/workflows/clippy.yml/badge.svg)](https://github.com/scattered-systems/concision/actions/workflows/clippy.yml) -[![Rust](https://github.com/scattered-systems/concision/actions/workflows/rust.yml/badge.svg)](https://github.com/scattered-systems/concision/actions/workflows/rust.yml) +[![crates.io](https://img.shields.io/crates/v/concision.svg)](https://crates.io/crates/concision) +[![docs.rs](https://docs.rs/concision/badge.svg)](https://docs.rs/concision) +[![Clippy](https://github.com/FL03/concision/actions/workflows/clippy.yml/badge.svg)](https://github.com/FL03/concision/actions/workflows/clippy.yml) +[![Rust](https://github.com/FL03/concision/actions/workflows/rust.yml/badge.svg)](https://github.com/FL03/concision/actions/workflows/rust.yml) *** @@ -15,7 +17,7 @@ written in Rust and designed to support the creation of enterprise-grade, data d Start by cloning the repository ```bash -git clone https://github.com/scattered-systems/concision +git clone https://github.com/FL03/concision ``` ```bash diff --git a/macros/Cargo.toml b/macros/Cargo.toml index 15ccb9d8..567b2290 100644 --- a/macros/Cargo.toml +++ b/macros/Cargo.toml @@ -16,9 +16,9 @@ crate-type = ["cdylib", "rlib"] test = false [dependencies] -proc-macro2 = "1.0.47" -quote = "1.0.21" -syn = { features = ["full"], version = "1.0.105" } +proc-macro2 = "1" +quote = "1" +syn = { features = ["full"], version = "1" } [dev-dependencies.concision] path = "../concision" From c811b4c9c6a6a2847005e21d05a04ef02d46b94e Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 25 Feb 2023 15:40:40 -0600 Subject: [PATCH 006/118] Update --- concision/src/math/linalg/vs.rs | 1 - concision/src/num/complex.rs | 12 +++--------- concision/src/num/mod.rs | 16 ---------------- 3 files changed, 3 insertions(+), 26 deletions(-) diff --git a/concision/src/math/linalg/vs.rs b/concision/src/math/linalg/vs.rs index ee2f74fe..9790b147 100644 --- a/concision/src/math/linalg/vs.rs +++ b/concision/src/math/linalg/vs.rs @@ -3,7 +3,6 @@ Contrib: FL03 Description: ... Summary ... */ -use crate::num::{Im, OrdPair, Re}; #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] pub struct VectorSpace(pub Vec); diff --git a/concision/src/num/complex.rs b/concision/src/num/complex.rs index f2a40a39..d5d9b2f9 100644 --- a/concision/src/num/complex.rs +++ b/concision/src/num/complex.rs @@ -12,14 +12,14 @@ use std::{ // Define a trait for complex numbers. pub trait Complex { - fn build(re: T, im: T) -> Self; + fn new(re: T, im: T) -> Self; fn re(&self) -> T; fn im(&self) -> T; } // Implement the Complex trait for the tuple (f64, f64). impl Complex for (f64, f64) { - fn build(re: f64, im: f64) -> Self { + fn new(re: f64, im: f64) -> Self { (re, im) } @@ -35,14 +35,8 @@ impl Complex for (f64, f64) { #[derive(Clone, Debug, Default, Deserialize, PartialEq, PartialOrd, Serialize)] pub struct C(T, T); -impl C { - pub fn new(a: T, b: T) -> Self { - Self(a, b) - } -} - impl Complex for C { - fn build(re: T, im: T) -> Self { + fn new(re: T, im: T) -> Self { Self(re, im) } fn re(&self) -> T { diff --git a/concision/src/num/mod.rs b/concision/src/num/mod.rs index ebaf255d..6b89b331 100644 --- a/concision/src/num/mod.rs +++ b/concision/src/num/mod.rs @@ -6,19 +6,3 @@ pub mod complex; -use crate::Numerical; - -#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] -pub struct OrdPair((Re, Re)); - -#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] -pub struct Re(T); - -impl Re { - pub fn new(data: T) -> Self { - Self(data) - } -} - -#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] -pub struct Im(Re); From c6f89da3150855dabb556445c86af466a959d3e7 Mon Sep 17 00:00:00 2001 From: FL03 Date: Tue, 28 Mar 2023 17:07:59 -0500 Subject: [PATCH 007/118] Update --- .github/workflows/clippy.yml | 31 +++++----- .github/workflows/rust.yml | 58 +++++++++++------- Cargo.toml | 7 +++ concision/Cargo.toml | 25 +++++--- concision/benches/default.rs | 52 ++++++++++++++++ concision/src/lib.rs | 2 + concision/src/math/mod.rs | 3 +- concision/src/math/regression/linear.rs | 13 ---- concision/src/math/statistics/deviation.rs | 24 ++++++++ concision/src/math/statistics/mod.rs | 60 +++++++++++++++++++ .../src/math/statistics/regression/linear.rs | 40 +++++++++++++ .../math/{ => statistics}/regression/mod.rs | 0 concision/src/num/mod.rs | 1 - core/Cargo.toml | 41 +++++++++++++ core/src/lib.rs | 10 ++++ core/src/primitives.rs | 15 +++++ core/src/utils.rs | 5 ++ core/tests/default.rs | 7 +++ derive/Cargo.toml | 9 ++- macros/Cargo.toml | 7 ++- 20 files changed, 347 insertions(+), 63 deletions(-) create mode 100644 concision/benches/default.rs delete mode 100644 concision/src/math/regression/linear.rs create mode 100644 concision/src/math/statistics/deviation.rs create mode 100644 concision/src/math/statistics/mod.rs create mode 100644 concision/src/math/statistics/regression/linear.rs rename concision/src/math/{ => statistics}/regression/mod.rs (100%) create mode 100644 core/Cargo.toml create mode 100644 core/src/lib.rs create mode 100644 core/src/primitives.rs create mode 100644 core/src/utils.rs create mode 100644 core/tests/default.rs diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml index f5595074..3824c6c8 100644 --- a/.github/workflows/clippy.yml +++ b/.github/workflows/clippy.yml @@ -1,45 +1,44 @@ name: Clippy on: + pull_request: + branches-ignore: [ "beta*", "dev*", "next*" ] push: - branches: [ "main", "master", "prod" ] - tags: ["v*.*.*"] + branches-ignore: [ "beta*", "dev*", "next*" ] + tags: [ "nightly*", "v*.*.*" ] + release: + types: [created] schedule: - - cron: "30 9 * * 5" + - cron: "30 9 * * *" # 9:30am UTC workflow_dispatch: jobs: - rust-clippy-analyze: - name: Run rust-clippy analyzing - runs-on: ubuntu-latest + clippy: + name: Clippy permissions: + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status contents: read security-events: write - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Install Rust toolchain + - uses: actions/checkout@v3 + - name: Setup rust toolchain uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af #@v1 with: profile: minimal toolchain: stable components: clippy override: true - - name: Install required cargo run: cargo install clippy-sarif sarif-fmt - - name: Run rust-clippy run: cargo clippy --all-features --message-format=json | clippy-sarif | tee rust-clippy-results.sarif | sarif-fmt continue-on-error: true - - - name: Upload analysis results to GitHub + - name: Upload analysis uses: github/codeql-action/upload-sarif@v2 with: sarif_file: rust-clippy-results.sarif - wait-for-processing: true \ No newline at end of file + wait-for-processing: true diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 6eff7483..38b08dac 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -4,11 +4,15 @@ env: CARGO_TERM_COLOR: always on: + pull_request: + branches-ignore: [ "beta*", "dev*", "next*" ] push: - branches: [ "main", "master", "prod" ] - tags: [ "v*.*.*" ] + branches-ignore: [ "beta*", "dev*", "next*" ] + tags: [ "nightly*", "v*.*.*" ] + release: + types: [created] schedule: - - cron: "30 9 * * 5" + - cron: "30 9 * * *" # 9:30am UTC workflow_dispatch: inputs: publish: @@ -20,20 +24,36 @@ on: jobs: build: name: Build and Test - runs-on: ubuntu-latest strategy: matrix: - toolchain: - - stable - - nightly + platform: [ macos-latest, ubuntu-latest, windows-latest ] + toolchain: [ stable, nightly ] + runs-on: ${{ matrix.platform }} steps: - uses: actions/checkout@v3 - - run: rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }} - - run: cargo build -v - - run: cargo test --all --all-features -v + - name: setup (langspace) + run: | + rustup update + rustup default ${{ matrix.toolchain }} + - name: Build + run: cargo build -F full --release -v --workspace + - name: Cache build + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target/release + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - name: Test + run: cargo test --all -F full --release -v + - name: Bench + if: matrix.toolchain == 'nightly' + run: cargo bench --all -F full --release -v features: - if: ${{ github.event.inputs.publish }} - name: Features + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.event_name == 'release' && github.event.action == 'created' || github.event.inputs.publish == 'true' + name: Publish (features) + needs: build runs-on: ubuntu-latest strategy: matrix: @@ -43,17 +63,13 @@ jobs: steps: - uses: actions/checkout@v3 - name: Publish (${{matrix.package}}) - run: cargo publish --all-features --package ${{ matrix.package }} --verbose --token ${{ secrets.CARGO_REGISTRY_TOKEN }} + run: cargo publish --all-features -v -p ${{ matrix.package }} --token ${{ secrets.CARGO_REGISTRY_TOKEN }} publish: - if: ${{ github.event.inputs.publish }} - name: Publish + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.event_name == 'release' && github.event.action == 'created' || github.event.inputs.publish == 'true' + name: Publish (sdk) needs: features runs-on: ubuntu-latest - strategy: - matrix: - package: - - concision steps: - uses: actions/checkout@v3 - - name: Publish (${{matrix.package}}) - run: cargo publish --all-features --package ${{ matrix.package }} --verbose --token ${{ secrets.CARGO_REGISTRY_TOKEN }} \ No newline at end of file + - name: Publish (sdk) + run: cargo publish --all-features -v -p concision --token ${{ secrets.CARGO_REGISTRY_TOKEN }} \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 941b72d6..d54dfe94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,12 @@ readme = "README.md" repository = "https://github.com/FL03/concision" version = "0.1.12" # TODO - Update the cargo package version +[workspace.dependencies] +serde = { features = ["derive"], version = "1" } +serde_json = "1" +smart-default = "0.6" +strum = { features = ["derive"], version = "0.24" } + [workspace] default-members = [ "concision" @@ -17,6 +23,7 @@ default-members = [ members = [ "concision", + "core", "derive", "macros" ] diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 55ec7a55..932463c2 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -15,23 +15,34 @@ version.workspace = true default = ["core"] full = ["core", "derive", "macros"] -core = [] +core = ["concision-core/default"] derive = ["concision-derive"] macros = ["concision-macros"] [lib] -bench = false crate-type = ["cdylib", "rlib"] test = true [build-dependencies] [dependencies] -scsys = { features = [], version = "0.1.41" } -serde = { features = ["derive"], version = "1" } -serde_json = "1" -smart-default = "0.6" -strum = { features = ["derive"], version = "0.24" } +ndarray = { features = ["blas", "serde-1"], version = "0.15" } +num = { features = [], version = "0.4" } +serde.workspace = true +serde_json.workspace = true +smart-default.workspace = true +strum.workspace = true +concision-core = { features = [], optional = true, path = "../core", version = "0.1.12" } concision-derive = { features = [], optional = true, path = "../derive", version = "0.1.12" } concision-macros = { features = [], optional = true, path = "../macros", version = "0.1.12" } + +[dev-dependencies] + +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] + +[target.wasm32-unknown-unknown] + +[target.wasm32-wasi] \ No newline at end of file diff --git a/concision/benches/default.rs b/concision/benches/default.rs new file mode 100644 index 00000000..937f2387 --- /dev/null +++ b/concision/benches/default.rs @@ -0,0 +1,52 @@ +// bench.rs +#![feature(test)] + +extern crate test; + +use std::mem::replace; +use test::Bencher; + +// bench: find the `BENCH_SIZE` first terms of the fibonacci sequence +static BENCH_SIZE: usize = 20; + +// recursive fibonacci +fn fibonacci(n: usize) -> u32 { + if n < 2 { + 1 + } else { + fibonacci(n - 1) + fibonacci(n - 2) + } +} + +// iterative fibonacci +struct Fibonacci { + curr: u32, + next: u32, +} + +impl Iterator for Fibonacci { + type Item = u32; + fn next(&mut self) -> Option { + let new_next = self.curr + self.next; + let new_curr = replace(&mut self.next, new_next); + + Some(replace(&mut self.curr, new_curr)) + } +} + +fn fibonacci_sequence() -> Fibonacci { + Fibonacci { curr: 1, next: 1 } +} + +// function to benchmark must be annotated with `#[bench]` +#[bench] +fn recursive_fibonacci(b: &mut Bencher) { + // exact code to benchmark must be passed as a closure to the iter + // method of Bencher + b.iter(|| (0..BENCH_SIZE).map(fibonacci).collect::>()) +} + +#[bench] +fn iterative_fibonacci(b: &mut Bencher) { + b.iter(|| fibonacci_sequence().take(BENCH_SIZE).collect::>()) +} diff --git a/concision/src/lib.rs b/concision/src/lib.rs index b6fb5d22..fc39a126 100644 --- a/concision/src/lib.rs +++ b/concision/src/lib.rs @@ -6,6 +6,8 @@ */ #[doc(inline)] pub use crate::{primitives::*, utils::*}; +#[cfg(feature = "core")] +pub use concision_core as core; #[cfg(feature = "derive")] pub use concision_derive::*; #[cfg(feature = "macros")] diff --git a/concision/src/math/mod.rs b/concision/src/math/mod.rs index 4b1a4a62..5bcaf74a 100644 --- a/concision/src/math/mod.rs +++ b/concision/src/math/mod.rs @@ -7,5 +7,6 @@ pub use self::factorials::*; pub mod calculus; pub mod linalg; +pub mod statistics; -pub(crate) mod factorials; +mod factorials; diff --git a/concision/src/math/regression/linear.rs b/concision/src/math/regression/linear.rs deleted file mode 100644 index 375a3b31..00000000 --- a/concision/src/math/regression/linear.rs +++ /dev/null @@ -1,13 +0,0 @@ -/* - Appellation: linear - Contrib: FL03 - Description: ... Summary ... -*/ - -pub struct LinearRegression; - -impl LinearRegression { - pub fn new() -> Self { - Self - } -} \ No newline at end of file diff --git a/concision/src/math/statistics/deviation.rs b/concision/src/math/statistics/deviation.rs new file mode 100644 index 00000000..d6fd8945 --- /dev/null +++ b/concision/src/math/statistics/deviation.rs @@ -0,0 +1,24 @@ +/* + Appellation: deviation + Contrib: FL03 + Description: ... Summary ... +*/ + +pub struct StandardDeviation { + pub mean: f64, + pub variance: f64, + pub deviation: f64, +} + +impl StandardDeviation { + pub fn new(x: &[f64]) -> StandardDeviation { + let mean = x.iter().sum::() / x.len() as f64; + let variance = x.iter().map(|&x| x * x).sum::() / x.len() as f64 - mean * mean; + let deviation = variance.sqrt(); + StandardDeviation { + mean, + variance, + deviation, + } + } +} diff --git a/concision/src/math/statistics/mod.rs b/concision/src/math/statistics/mod.rs new file mode 100644 index 00000000..ac475421 --- /dev/null +++ b/concision/src/math/statistics/mod.rs @@ -0,0 +1,60 @@ +/* + Appellation: statistics + Contrib: FL03 + Description: ... Summary ... +*/ +pub use self::deviation::*; + +pub mod regression; + +mod deviation; + +/// Covariance is the average of the products of the deviations from the mean. +pub fn covariance(x: Vec, y: Vec) -> f64 { + x.iter().zip(y.iter()).map(|(&x, &y)| x * y).sum::() / x.len() as f64 +} +/// Deviation is the distance from the mean. +pub fn deviation(x: &[f64], mean: f64) -> Vec { + x.iter().map(|&x| x - mean).collect() +} +/// Mean is the average of the data. +pub fn mean(x: &[f64]) -> f64 { + x.iter().sum::() / x.len() as f64 +} +/// Variance is the average of the squared deviations from the mean. +pub fn variance(x: Vec) -> f64 { + let mean = mean(&x); + let dev = deviation(&x, mean); + dev.iter().map(|&x| x * x).sum::() / dev.len() as f64 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_covariance() { + let x = vec![1.0, 2.0, 3.0, 4.0, 5.0]; + let y = vec![1.0, 2.0, 3.0, 4.0, 5.0]; + assert_eq!(covariance(x, y), 2.0); + } + + #[test] + fn test_deviation() { + let x = vec![1.0, 2.0, 3.0, 4.0, 5.0]; + let mean = mean(&x); + assert_eq!(deviation(&x, mean), vec![-2.0, -1.0, 0.0, 1.0, 2.0]); + } + + #[test] + fn test_mean() { + let x = vec![1.0, 2.0, 3.0, 4.0, 5.0]; + assert_eq!(mean(&x), 3.0); + } + + #[test] + fn test_variance() { + let x = vec![1.0, 2.0, 3.0, 4.0, 5.0]; + assert_eq!(variance(x), 2.0); + } +} diff --git a/concision/src/math/statistics/regression/linear.rs b/concision/src/math/statistics/regression/linear.rs new file mode 100644 index 00000000..2e53fdd9 --- /dev/null +++ b/concision/src/math/statistics/regression/linear.rs @@ -0,0 +1,40 @@ +/* + Appellation: linear + Contrib: FL03 + Description: ... Summary ... +*/ +use crate::math::statistics::{covariance, deviation, mean, variance}; + +pub struct LinearRegression { + pub slope: f64, + pub intercept: f64, +} + +impl LinearRegression { + pub fn new(x: &[f64], y: &[f64]) -> LinearRegression { + let (x_mean, y_mean) = (mean(x), mean(y)); + let (x_dev, y_dev) = (deviation(x, x_mean), deviation(y, y_mean)); + let slope = covariance(x_dev.clone(), y_dev) / variance(x_dev); + let intercept = y_mean - slope * x_mean; + LinearRegression { slope, intercept } + } + + pub fn predict(&self, x: &[f64]) -> Vec { + x.iter().map(|&x| self.slope * x + self.intercept).collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_linear_regression() { + let x = vec![1.0, 2.0, 3.0, 4.0, 5.0]; + let y = vec![1.0, 2.0, 3.0, 4.0, 5.0]; + let lr = LinearRegression::new(&x, &y); + assert_eq!(lr.slope, 1.0); + assert_eq!(lr.intercept, 0.0); + assert_eq!(lr.predict(&x), y); + } +} diff --git a/concision/src/math/regression/mod.rs b/concision/src/math/statistics/regression/mod.rs similarity index 100% rename from concision/src/math/regression/mod.rs rename to concision/src/math/statistics/regression/mod.rs diff --git a/concision/src/num/mod.rs b/concision/src/num/mod.rs index 6b89b331..cddd73df 100644 --- a/concision/src/num/mod.rs +++ b/concision/src/num/mod.rs @@ -5,4 +5,3 @@ */ pub mod complex; - diff --git a/core/Cargo.toml b/core/Cargo.toml new file mode 100644 index 00000000..c9544e5a --- /dev/null +++ b/core/Cargo.toml @@ -0,0 +1,41 @@ +[package] +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "concision-core" +readme.workspace = true +repository.workspace = true +version.workspace = true + +[features] +default = [] + +wasm = [] + +[lib] +crate-type = ["cdylib", "rlib"] +test = true + +[build-dependencies] + +[dependencies] +ndarray = { features = ["blas", "serde-1"], version = "0.15" } +num = { features = [], version = "0.4" } +serde.workspace = true +serde_json.workspace = true +smart-default.workspace = true +strum.workspace = true + +[dev-dependencies] + +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] + +[target.wasm32-unknown-unknown] + +[target.wasm32-wasi] diff --git a/core/src/lib.rs b/core/src/lib.rs new file mode 100644 index 00000000..71c26cec --- /dev/null +++ b/core/src/lib.rs @@ -0,0 +1,10 @@ +/* + Appellation: concision + Contrib: FL03 + Description: + Concision is a robust framework for creating powerful data-centric applications in Rust. +*/ +pub use self::{primitives::*, utils::*}; + +mod primitives; +mod utils; diff --git a/core/src/primitives.rs b/core/src/primitives.rs new file mode 100644 index 00000000..b06158b7 --- /dev/null +++ b/core/src/primitives.rs @@ -0,0 +1,15 @@ +/* + Appellation: primitives + Contrib: FL03 + Description: ... Summary ... +*/ +pub use self::{constants::*, statics::*, types::*}; + +/// Collection of constants used throughout the system +pub(crate) mod constants {} + +/// Collection of static references used throughout +pub(crate) mod statics {} + +/// Collection of types used throughout the system +pub(crate) mod types {} diff --git a/core/src/utils.rs b/core/src/utils.rs new file mode 100644 index 00000000..5a7b9ed6 --- /dev/null +++ b/core/src/utils.rs @@ -0,0 +1,5 @@ +/* + Appellation: utils + Contrib: FL03 + Description: ... Summary ... +*/ diff --git a/core/tests/default.rs b/core/tests/default.rs new file mode 100644 index 00000000..834d08a6 --- /dev/null +++ b/core/tests/default.rs @@ -0,0 +1,7 @@ +#[cfg(test)] +#[test] +fn lib_compiles() { + let f = |x: usize, y: usize| x + y; + + assert_eq!(f(10, 10), 20) +} diff --git a/derive/Cargo.toml b/derive/Cargo.toml index ec0f2715..49ef9cff 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -20,5 +20,10 @@ proc-macro2 = "1" quote = "1" syn = { features = ["full"], version = "1" } -[dev-dependencies.concision] -path = "../concision" +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] + +[target.wasm32-unknown-unknown] + +[target.wasm32-wasi] diff --git a/macros/Cargo.toml b/macros/Cargo.toml index 567b2290..8adaa70e 100644 --- a/macros/Cargo.toml +++ b/macros/Cargo.toml @@ -20,7 +20,10 @@ proc-macro2 = "1" quote = "1" syn = { features = ["full"], version = "1" } -[dev-dependencies.concision] -path = "../concision" +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] [target.wasm32-unknown-unknown] + +[target.wasm32-wasi] From fbd6358955e7e884f0a60709ec391524a088aa4d Mon Sep 17 00:00:00 2001 From: FL03 Date: Tue, 28 Mar 2023 17:08:54 -0500 Subject: [PATCH 008/118] Update --- .github/dependabot.yml | 8 ++++++-- .github/workflows/rust.yml | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1efe9c33..fff40f86 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,10 +13,14 @@ updates: schedule: interval: daily - package-ecosystem: cargo - directory: /concision-derive + directory: /core schedule: interval: daily - package-ecosystem: cargo - directory: /concision-macros + directory: /derive + schedule: + interval: daily + - package-ecosystem: cargo + directory: /macros schedule: interval: daily \ No newline at end of file diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 38b08dac..2e75b74a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -58,6 +58,7 @@ jobs: strategy: matrix: package: + - concision-core - concision-derive - concision-macros steps: From 467128e1ebe83cfb4012c536f2bf6b88163858d7 Mon Sep 17 00:00:00 2001 From: FL03 Date: Tue, 11 Apr 2023 19:25:12 -0500 Subject: [PATCH 009/118] Update --- .github/workflows/rust.yml | 15 +++--- concision/src/lib.rs | 4 +- .../src/math/calculus/derivatives/nderive.rs | 49 +++---------------- concision/src/math/statistics/mod.rs | 4 +- concision/src/primitives.rs | 6 +-- core/src/lib.rs | 7 +-- core/src/linstep.rs | 5 ++ core/src/primitives.rs | 6 +-- 8 files changed, 33 insertions(+), 63 deletions(-) create mode 100644 core/src/linstep.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 2e75b74a..f02e0992 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -49,7 +49,7 @@ jobs: run: cargo test --all -F full --release -v - name: Bench if: matrix.toolchain == 'nightly' - run: cargo bench --all -F full --release -v + run: cargo bench --all -v features: if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.event_name == 'release' && github.event.action == 'created' || github.event.inputs.publish == 'true' name: Publish (features) @@ -57,14 +57,13 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - package: - - concision-core - - concision-derive - - concision-macros + feature: [core, derive, macros ] + env: + PACKAGE_NAME: ${{ github.event.repository.name }}-${{ matrix.feature }} steps: - uses: actions/checkout@v3 - - name: Publish (${{matrix.package}}) - run: cargo publish --all-features -v -p ${{ matrix.package }} --token ${{ secrets.CARGO_REGISTRY_TOKEN }} + - name: Publish (${{env.PACKAGE_NAME}}) + run: cargo publish --all-features -v -p ${{ env.PACKAGE_NAME }} --token ${{ secrets.CARGO_REGISTRY_TOKEN }} publish: if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.event_name == 'release' && github.event.action == 'created' || github.event.inputs.publish == 'true' name: Publish (sdk) @@ -73,4 +72,4 @@ jobs: steps: - uses: actions/checkout@v3 - name: Publish (sdk) - run: cargo publish --all-features -v -p concision --token ${{ secrets.CARGO_REGISTRY_TOKEN }} \ No newline at end of file + run: cargo publish --all-features -v -p ${{ github.event.repository.name }} --token ${{ secrets.CARGO_REGISTRY_TOKEN }} \ No newline at end of file diff --git a/concision/src/lib.rs b/concision/src/lib.rs index fc39a126..89b213fb 100644 --- a/concision/src/lib.rs +++ b/concision/src/lib.rs @@ -16,8 +16,8 @@ pub use concision_macros::*; pub mod math; pub mod num; -pub(crate) mod primitives; -pub(crate) mod utils; +mod primitives; +mod utils; use std::ops::{Add, Div, Mul, Sub}; diff --git a/concision/src/math/calculus/derivatives/nderive.rs b/concision/src/math/calculus/derivatives/nderive.rs index cf807cd6..9049f614 100644 --- a/concision/src/math/calculus/derivatives/nderive.rs +++ b/concision/src/math/calculus/derivatives/nderive.rs @@ -3,47 +3,9 @@ Contrib: FL03 Description: ... Summary ... */ -use serde::{Deserialize, Serialize}; -use strum::{EnumString, EnumVariantNames}; -#[derive( - Clone, - Copy, - Debug, - Default, - Deserialize, - EnumString, - EnumVariantNames, - Eq, - Hash, - PartialEq, - PartialOrd, - Serialize, -)] -#[strum(serialize_all = "snake_case")] -pub enum DerivativeMode { - Backwards, - #[default] - Central, - Forwards, -} - -impl DerivativeMode { - pub fn execute(&self, data: T, func: &dyn Fn(T) -> T) { - match self { - Self::Backwards => {} - Self::Central => {} - Self::Forwards => {} - } - } -} - -pub fn central(x: Vec, f: &dyn Fn(f64) -> f64, h: f64) -> Vec { - let mut res = Vec::new(); - for i in x { - res.push((f(i.clone() + h) - f(i - h)) / h); - } - res +pub fn central(x: T, f: Box T>, h: T) -> T { + (f(x + h) - f(x)) / h } #[cfg(test)] @@ -51,8 +13,9 @@ mod tests { use super::*; #[test] - fn test() { - let f = |x: f64| x.powf(x); - assert_eq!(f(2.0), 4.0) + fn test_central() { + let f = |x: f64| x * x; + let res = central(5.0, Box::new(f), 0.001); + assert_eq!(res.round(), 10.0); } } diff --git a/concision/src/math/statistics/mod.rs b/concision/src/math/statistics/mod.rs index ac475421..c42b0cae 100644 --- a/concision/src/math/statistics/mod.rs +++ b/concision/src/math/statistics/mod.rs @@ -11,7 +11,9 @@ mod deviation; /// Covariance is the average of the products of the deviations from the mean. pub fn covariance(x: Vec, y: Vec) -> f64 { - x.iter().zip(y.iter()).map(|(&x, &y)| x * y).sum::() / x.len() as f64 + let dx = deviation(&x, mean(&x)); + let dy = deviation(&y, mean(&y)); + dx.iter().zip(dy.iter()).map(|(&x, &y)| x * y).sum::() / dx.len() as f64 } /// Deviation is the distance from the mean. pub fn deviation(x: &[f64], mean: f64) -> Vec { diff --git a/concision/src/primitives.rs b/concision/src/primitives.rs index b06158b7..43288c57 100644 --- a/concision/src/primitives.rs +++ b/concision/src/primitives.rs @@ -6,10 +6,10 @@ pub use self::{constants::*, statics::*, types::*}; /// Collection of constants used throughout the system -pub(crate) mod constants {} +mod constants {} /// Collection of static references used throughout -pub(crate) mod statics {} +mod statics {} /// Collection of types used throughout the system -pub(crate) mod types {} +mod types {} diff --git a/core/src/lib.rs b/core/src/lib.rs index 71c26cec..61d16121 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -1,10 +1,11 @@ /* - Appellation: concision + Appellation: core Contrib: FL03 - Description: - Concision is a robust framework for creating powerful data-centric applications in Rust. + Description: Implements the core functionality of the concision. Concision is an advanced data-science and machine-learning crate written in pure Rust and optimized for WASM environments. */ pub use self::{primitives::*, utils::*}; mod primitives; mod utils; + +pub mod linstep; diff --git a/core/src/linstep.rs b/core/src/linstep.rs new file mode 100644 index 00000000..ca2e26af --- /dev/null +++ b/core/src/linstep.rs @@ -0,0 +1,5 @@ +/* + Appellation: linstep + Contrib: FL03 + Description: ... Summary ... +*/ diff --git a/core/src/primitives.rs b/core/src/primitives.rs index b06158b7..43288c57 100644 --- a/core/src/primitives.rs +++ b/core/src/primitives.rs @@ -6,10 +6,10 @@ pub use self::{constants::*, statics::*, types::*}; /// Collection of constants used throughout the system -pub(crate) mod constants {} +mod constants {} /// Collection of static references used throughout -pub(crate) mod statics {} +mod statics {} /// Collection of types used throughout the system -pub(crate) mod types {} +mod types {} From 525cbb92f7e3d0752f8837d01e437f62f5d770a3 Mon Sep 17 00:00:00 2001 From: FL03 Date: Tue, 6 Jun 2023 09:11:32 -0500 Subject: [PATCH 010/118] update Signed-off-by: FL03 --- .../license/{APACHE.LICENSE => APACHE.SCSYS.LICENSE} | 2 +- .artifacts/license/MIT.LICENSE | 2 +- .github/FUNDING.yml | 1 + .github/workflows/rust.yml | 3 ++- Cargo.toml | 2 +- LICENSE | 2 +- concision/Cargo.toml | 2 ++ core/Cargo.toml | 2 ++ derive/Cargo.toml | 7 ++++++- macros/Cargo.toml | 9 +++++++-- 10 files changed, 24 insertions(+), 8 deletions(-) rename .artifacts/license/{APACHE.LICENSE => APACHE.SCSYS.LICENSE} (99%) create mode 100644 .github/FUNDING.yml diff --git a/.artifacts/license/APACHE.LICENSE b/.artifacts/license/APACHE.SCSYS.LICENSE similarity index 99% rename from .artifacts/license/APACHE.LICENSE rename to .artifacts/license/APACHE.SCSYS.LICENSE index 21e337ae..86685f9a 100644 --- a/.artifacts/license/APACHE.LICENSE +++ b/.artifacts/license/APACHE.SCSYS.LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2022 Scattered-Systems, DAO LLC + Copyright 2023 Scattered-Systems, DAO LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/.artifacts/license/MIT.LICENSE b/.artifacts/license/MIT.LICENSE index 7a3e60be..4cea9a3c 100644 --- a/.artifacts/license/MIT.LICENSE +++ b/.artifacts/license/MIT.LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2022 Scattered-Systems, DAO LLC +Copyright (c) 2023 Joe McCain III Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..c26b651b --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: [FL03] diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index f02e0992..5b7c10aa 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -10,7 +10,8 @@ on: branches-ignore: [ "beta*", "dev*", "next*" ] tags: [ "nightly*", "v*.*.*" ] release: - types: [created] + repository_dispatch: + types: [ rust-publish ] schedule: - cron: "30 9 * * *" # 9:30am UTC workflow_dispatch: diff --git a/Cargo.toml b/Cargo.toml index d54dfe94..f226a5d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ version = "0.1.12" # TODO - Update the cargo package version [workspace.dependencies] serde = { features = ["derive"], version = "1" } serde_json = "1" -smart-default = "0.6" +smart-default = "0.7" strum = { features = ["derive"], version = "0.24" } [workspace] diff --git a/LICENSE b/LICENSE index 21e337ae..f25b2b18 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2022 Scattered-Systems, DAO LLC + Copyright 2023 Joe McCain III Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 932463c2..81d27b05 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -20,7 +20,9 @@ derive = ["concision-derive"] macros = ["concision-macros"] [lib] +bench = true crate-type = ["cdylib", "rlib"] +doctest = false test = true [build-dependencies] diff --git a/core/Cargo.toml b/core/Cargo.toml index c9544e5a..e9c05502 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -17,7 +17,9 @@ default = [] wasm = [] [lib] +bench = true crate-type = ["cdylib", "rlib"] +doctest = false test = true [build-dependencies] diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 49ef9cff..f885219f 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -11,14 +11,19 @@ readme.workspace = true repository.workspace = true version.workspace = true +[features] +default = [] + [lib] +bench = false +doctest = false proc-macro = true test = false [dependencies] proc-macro2 = "1" quote = "1" -syn = { features = ["full"], version = "1" } +syn = { features = ["full"], version = "2" } [package.metadata.docs.rs] all-features = true diff --git a/macros/Cargo.toml b/macros/Cargo.toml index 8adaa70e..b6092ea6 100644 --- a/macros/Cargo.toml +++ b/macros/Cargo.toml @@ -11,14 +11,19 @@ readme.workspace = true repository.workspace = true version.workspace = true +[features] +default = [] + [lib] +bench = false crate-type = ["cdylib", "rlib"] -test = false +doctest = false +test = true [dependencies] proc-macro2 = "1" quote = "1" -syn = { features = ["full"], version = "1" } +syn = { features = ["full"], version = "2" } [package.metadata.docs.rs] all-features = true From 4ce9eda42dfd9556b2952896771c366729a0e39e Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 13 Oct 2023 11:05:06 -0500 Subject: [PATCH 011/118] update Signed-off-by: FL03 --- .github/workflows/clippy.yml | 2 +- .github/workflows/publish.yml | 37 ++++++++ .github/workflows/rust.yml | 27 +----- Cargo.toml | 5 +- README.md | 2 + concision/Cargo.toml | 18 +++- concision/examples/simple.rs | 2 +- concision/src/lib.rs | 50 ++++++----- concision/src/nn/mod.rs | 1 + concision/src/num/complex.rs | 3 +- concision/src/primitives.rs | 1 - concision/src/specs.rs | 4 + concision/tests/default.rs | 5 +- core/Cargo.toml | 1 - core/src/lib.rs | 19 +++-- core/src/linspace.rs | 154 ++++++++++++++++++++++++++++++++++ core/src/linstep.rs | 5 -- core/src/specs.rs | 25 ++++++ core/tests/default.rs | 5 +- derive/Cargo.toml | 2 +- derive/src/lib.rs | 11 +-- macros/Cargo.toml | 6 +- macros/src/lib.rs | 5 +- 23 files changed, 299 insertions(+), 91 deletions(-) create mode 100644 .github/workflows/publish.yml create mode 100644 concision/src/nn/mod.rs create mode 100644 concision/src/specs.rs create mode 100644 core/src/linspace.rs delete mode 100644 core/src/linstep.rs create mode 100644 core/src/specs.rs diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml index 3824c6c8..297efdc3 100644 --- a/.github/workflows/clippy.yml +++ b/.github/workflows/clippy.yml @@ -21,7 +21,7 @@ jobs: security-events: write runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup rust toolchain uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af #@v1 with: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..1cbbeda6 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,37 @@ +name: Publish + +env: + CARGO_TERM_COLOR: always + +on: + release: + types: [ created ] + workflow_dispatch: + inputs: + publish: + default: true + description: 'Publish' + required: true + type: boolean + +jobs: + features: + name: Publish (features) + runs-on: ubuntu-latest + strategy: + matrix: + feature: [ core, derive, macros ] + env: + PACKAGE_NAME: ${{ github.event.repository.name }}-${{ matrix.feature }} + steps: + - uses: actions/checkout@v4 + - name: Publish (${{env.PACKAGE_NAME}}) + run: cargo publish --all-features -v -p ${{ env.PACKAGE_NAME }} --token ${{ secrets.CARGO_REGISTRY_TOKEN }} + publish: + name: Publish (sdk) + needs: features + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Publish (sdk) + run: cargo publish --all-features -v -p ${{ github.event.repository.name }} --token ${{ secrets.CARGO_REGISTRY_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5b7c10aa..87c73405 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -31,7 +31,7 @@ jobs: toolchain: [ stable, nightly ] runs-on: ${{ matrix.platform }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: setup (langspace) run: | rustup update @@ -39,7 +39,7 @@ jobs: - name: Build run: cargo build -F full --release -v --workspace - name: Cache build - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | ~/.cargo/registry @@ -51,26 +51,3 @@ jobs: - name: Bench if: matrix.toolchain == 'nightly' run: cargo bench --all -v - features: - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.event_name == 'release' && github.event.action == 'created' || github.event.inputs.publish == 'true' - name: Publish (features) - needs: build - runs-on: ubuntu-latest - strategy: - matrix: - feature: [core, derive, macros ] - env: - PACKAGE_NAME: ${{ github.event.repository.name }}-${{ matrix.feature }} - steps: - - uses: actions/checkout@v3 - - name: Publish (${{env.PACKAGE_NAME}}) - run: cargo publish --all-features -v -p ${{ env.PACKAGE_NAME }} --token ${{ secrets.CARGO_REGISTRY_TOKEN }} - publish: - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.event_name == 'release' && github.event.action == 'created' || github.event.inputs.publish == 'true' - name: Publish (sdk) - needs: features - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Publish (sdk) - run: cargo publish --all-features -v -p ${{ github.event.repository.name }} --token ${{ secrets.CARGO_REGISTRY_TOKEN }} \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index f226a5d1..5a27a2b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,10 +11,11 @@ repository = "https://github.com/FL03/concision" version = "0.1.12" # TODO - Update the cargo package version [workspace.dependencies] +anyhow = "1" serde = { features = ["derive"], version = "1" } serde_json = "1" smart-default = "0.7" -strum = { features = ["derive"], version = "0.24" } +strum = { features = ["derive"], version = "0.25" } [workspace] default-members = [ @@ -28,6 +29,8 @@ members = [ "macros" ] +resolver = "2" + [profile.dev] opt-level = 0 debug = true diff --git a/README.md b/README.md index af80b615..fe4ef358 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ [![crates.io](https://img.shields.io/crates/v/concision.svg)](https://crates.io/crates/concision) [![docs.rs](https://docs.rs/concision/badge.svg)](https://docs.rs/concision) + [![Clippy](https://github.com/FL03/concision/actions/workflows/clippy.yml/badge.svg)](https://github.com/FL03/concision/actions/workflows/clippy.yml) +[![publish](https://github.com/FL03/concision/actions/workflows/publish.yml/badge.svg)](https://github.com/FL03/concision/actions/workflows/publish.yml) [![Rust](https://github.com/FL03/concision/actions/workflows/rust.yml/badge.svg)](https://github.com/FL03/concision/actions/workflows/rust.yml) *** diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 81d27b05..37ddb405 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -13,11 +13,21 @@ version.workspace = true [features] default = ["core"] -full = ["core", "derive", "macros"] -core = ["concision-core/default"] -derive = ["concision-derive"] -macros = ["concision-macros"] +full = ["core", "derive"] + +core = [ + "concision-core/default" +] + +derive = [ + "concision-derive", + "macros", +] + +macros = [ + "concision-macros" +] [lib] bench = true diff --git a/concision/examples/simple.rs b/concision/examples/simple.rs index fd4dc389..0351b11e 100644 --- a/concision/examples/simple.rs +++ b/concision/examples/simple.rs @@ -1,7 +1,7 @@ extern crate concision; use concision::num::complex::C; -use concision::Numerical; +use concision::prelude::Numerical; // Define a holomorphic function that squares its input. fn square(z: C) -> C { diff --git a/concision/src/lib.rs b/concision/src/lib.rs index 89b213fb..94479f6f 100644 --- a/concision/src/lib.rs +++ b/concision/src/lib.rs @@ -1,11 +1,13 @@ /* Appellation: concision Contrib: FL03 - Description: - Concision is a robust framework for creating powerful data-centric applications in Rust. */ +//! # Concision +//! +//! Concision aims to be a complete machine learning library written in pure Rust. +//! #[doc(inline)] -pub use crate::{primitives::*, utils::*}; +pub use crate::{primitives::*, specs::*, utils::*}; #[cfg(feature = "core")] pub use concision_core as core; #[cfg(feature = "derive")] @@ -14,29 +16,25 @@ pub use concision_derive::*; pub use concision_macros::*; pub mod math; +pub mod nn; pub mod num; -mod primitives; -mod utils; - -use std::ops::{Add, Div, Mul, Sub}; - -/// [Numerical] is a basic trait describing numerical objects -pub trait Numerical: - Add - + Div - + Mul - + Sub - + Clone - + Copy - + Sized -{ +pub(crate) mod primitives; +pub(crate) mod specs; +pub(crate) mod utils; + +pub mod prelude { + pub use crate::math::*; + pub use crate::nn::*; + pub use crate::num::*; + pub use crate::primitives::*; + pub use crate::specs::*; + pub use crate::utils::*; + + #[cfg(feature = "core")] + pub use concision_core::prelude::*; + #[cfg(feature = "derive")] + pub use concision_derive::*; + #[cfg(feature = "macros")] + pub use concision_macros::*; } - -impl Numerical for f32 {} - -impl Numerical for f64 {} - -impl Numerical for i64 {} - -impl Numerical for usize {} diff --git a/concision/src/nn/mod.rs b/concision/src/nn/mod.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/concision/src/nn/mod.rs @@ -0,0 +1 @@ + diff --git a/concision/src/num/complex.rs b/concision/src/num/complex.rs index d5d9b2f9..caef186c 100644 --- a/concision/src/num/complex.rs +++ b/concision/src/num/complex.rs @@ -1,9 +1,8 @@ /* Appellation: complex Contrib: FL03 - Description: ... Summary ... */ -use crate::Numerical; +use crate::core::Numerical; use serde::{Deserialize, Serialize}; use std::{ convert::From, diff --git a/concision/src/primitives.rs b/concision/src/primitives.rs index 43288c57..14b38b52 100644 --- a/concision/src/primitives.rs +++ b/concision/src/primitives.rs @@ -1,7 +1,6 @@ /* Appellation: primitives Contrib: FL03 - Description: ... Summary ... */ pub use self::{constants::*, statics::*, types::*}; diff --git a/concision/src/specs.rs b/concision/src/specs.rs new file mode 100644 index 00000000..1d8faa71 --- /dev/null +++ b/concision/src/specs.rs @@ -0,0 +1,4 @@ +/* + Appellation: specs + Contrib: FL03 +*/ diff --git a/concision/tests/default.rs b/concision/tests/default.rs index 834d08a6..0cac1eb5 100644 --- a/concision/tests/default.rs +++ b/concision/tests/default.rs @@ -1,7 +1,8 @@ #[cfg(test)] #[test] -fn lib_compiles() { +fn compiles() { let f = |x: usize, y: usize| x + y; - assert_eq!(f(10, 10), 20) + assert_eq!(f(10, 10), 20); + assert_ne!(f(1, 1), 3); } diff --git a/core/Cargo.toml b/core/Cargo.toml index e9c05502..9f8ac379 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -14,7 +14,6 @@ version.workspace = true [features] default = [] -wasm = [] [lib] bench = true diff --git a/core/src/lib.rs b/core/src/lib.rs index 61d16121..82848fcf 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -1,11 +1,20 @@ /* Appellation: core Contrib: FL03 - Description: Implements the core functionality of the concision. Concision is an advanced data-science and machine-learning crate written in pure Rust and optimized for WASM environments. */ -pub use self::{primitives::*, utils::*}; +//! # Concision Core +pub use self::{primitives::*, specs::*, utils::*}; -mod primitives; -mod utils; +pub(crate) mod primitives; +pub(crate) mod specs; +pub(crate) mod utils; -pub mod linstep; +pub mod linspace; + +pub mod prelude { + pub use crate::linspace::*; + + pub use crate::primitives::*; + pub use crate::specs::*; + pub use crate::utils::*; +} diff --git a/core/src/linspace.rs b/core/src/linspace.rs new file mode 100644 index 00000000..2e9ed2aa --- /dev/null +++ b/core/src/linspace.rs @@ -0,0 +1,154 @@ +/* + Appellation: linspace + Contrib: FL03 +*/ +use std::ops::{self, Range}; + +pub fn round(num: f64, decimals: usize) -> f64 { + let factor = 10.0_f64.powi(decimals as i32); + (num * factor).round() / factor +} + +fn calculate_step + ops::Sub>( + bounds: Range, + capacity: T, +) -> T { + (bounds.end - bounds.start) / capacity +} + +fn stepsize + ops::Sub>(from: T, to: T, capacity: T) -> T { + (to - from) / capacity +} + +fn _linspace(a: f64, b: f64, capacity: usize) -> Vec { + let stepsize = stepsize(a, b, capacity as f64); + let mut data = Vec::with_capacity(capacity); + let mut current = a; + for _ in 0..capacity { + data.push(current); + current += stepsize; + } + data +} + +pub struct Linspace { + bounds: Range, + capacity: usize, + data: Vec, + stepsize: f64, +} + +impl Linspace { + pub fn new(bounds: Range, capacity: usize) -> Self { + let stepsize = calculate_step(bounds.clone(), capacity as f64); + let mut data = Vec::with_capacity(capacity); + let mut current = bounds.start; + for _ in 0..capacity { + data.push(current); + current += stepsize; + } + Self { + bounds, + capacity, + data, + stepsize, + } + } + + pub fn bounds(&self) -> &Range { + &self.bounds + } + + pub fn capacity(&self) -> usize { + self.capacity + } + + pub fn data(&self) -> &Vec { + &self.data + } + + pub fn stepsize(&self) -> f64 { + self.stepsize + } + + pub fn compute(&mut self) { + let mut current = self.bounds().start; + for i in 0..self.capacity() { + self.data[i] = current; + current += self.stepsize; + } + } + + pub fn update(&mut self) { + self.stepsize = stepsize(self.bounds().start, self.bounds().end, self.capacity as f64); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_rounding() { + let val = 1.23456789; + assert_eq!(round(val, 3), 1.235); + assert_eq!(round(val, 4), 1.2346); + assert_eq!(round(val, 5), 1.23457); + assert_eq!(round(val, 6), 1.234568); + assert_eq!(round(val, 7), 1.2345679); + assert_eq!(round(val, 8), 1.23456789); + } + + #[test] + fn test_linspace_simple() { + let bounds = 0.0..10.0; + let capacity = 10; + + let res = Linspace::new(bounds.clone(), capacity); + assert_eq!(res.bounds(), &bounds); + assert_eq!(res.capacity(), capacity); + assert_eq!( + res.data(), + &vec![0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] + ); + } + + #[test] + fn test_linspace() { + let bounds = 0.0..5.0; + let capacity = 10; + + let res = Linspace::new(bounds.clone(), capacity); + assert_eq!(res.bounds(), &bounds); + assert_eq!(res.capacity(), capacity); + assert_eq!( + res.data(), + &vec![0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5] + ); + } + + #[test] + fn test_linspace_large() { + let bounds = 0.0..10.0; + let capacity = 100; + + let res = Linspace::new(bounds.clone(), capacity); + assert_eq!(res.bounds(), &bounds); + assert_eq!(res.capacity(), capacity); + assert_eq!(res.data()[1], 0.1); + + let bounds = 1.2..10.0; + let capacity = 100; + + let res = Linspace::new(bounds.clone(), capacity); + assert_eq!(res.bounds(), &bounds); + assert_eq!(res.capacity(), capacity); + assert_eq!(res.data()[0], 1.2); + assert_eq!(res.data()[1], 1.288); + let last = { + let tmp = format!("{:.3}", res.data().last().unwrap()); + tmp.parse::().unwrap() + }; + assert_eq!(last, 10.0 - res.stepsize()); + } +} diff --git a/core/src/linstep.rs b/core/src/linstep.rs deleted file mode 100644 index ca2e26af..00000000 --- a/core/src/linstep.rs +++ /dev/null @@ -1,5 +0,0 @@ -/* - Appellation: linstep - Contrib: FL03 - Description: ... Summary ... -*/ diff --git a/core/src/specs.rs b/core/src/specs.rs new file mode 100644 index 00000000..b6db4c0c --- /dev/null +++ b/core/src/specs.rs @@ -0,0 +1,25 @@ +/* + Appellation: specs + Contrib: FL03 +*/ +use std::ops::{Add, Div, Mul, Sub}; + +/// [Numerical] is a basic trait describing numerical objects +pub trait Numerical: + Add + + Div + + Mul + + Sub + + Clone + + Copy + + Sized +{ +} + +impl Numerical for f32 {} + +impl Numerical for f64 {} + +impl Numerical for i64 {} + +impl Numerical for usize {} diff --git a/core/tests/default.rs b/core/tests/default.rs index 834d08a6..0cac1eb5 100644 --- a/core/tests/default.rs +++ b/core/tests/default.rs @@ -1,7 +1,8 @@ #[cfg(test)] #[test] -fn lib_compiles() { +fn compiles() { let f = |x: usize, y: usize| x + y; - assert_eq!(f(10, 10), 20) + assert_eq!(f(10, 10), 20); + assert_ne!(f(1, 1), 3); } diff --git a/derive/Cargo.toml b/derive/Cargo.toml index f885219f..8320775f 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -21,7 +21,7 @@ proc-macro = true test = false [dependencies] -proc-macro2 = "1" +proc-macro2 = "1.0.69" quote = "1" syn = { features = ["full"], version = "2" } diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 55d35644..3a2cb732 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -1,10 +1,5 @@ /* - Appellation: concision-derive - Creator: FL03 - Description: - Derive macros for concision, a robust data-science toolkit for creating powerful Rust apps. + Appellation: concision-Derive + Contrib: FL03 */ - -extern crate proc_macro2; -extern crate quote; -extern crate syn; +//! # Concision Derive diff --git a/macros/Cargo.toml b/macros/Cargo.toml index b6092ea6..e063745a 100644 --- a/macros/Cargo.toml +++ b/macros/Cargo.toml @@ -16,12 +16,12 @@ default = [] [lib] bench = false -crate-type = ["cdylib", "rlib"] doctest = false -test = true +proc-macro = true +test = false [dependencies] -proc-macro2 = "1" +proc-macro2 = "1.0.69" quote = "1" syn = { features = ["full"], version = "2" } diff --git a/macros/src/lib.rs b/macros/src/lib.rs index a64e07bb..7111079e 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1,6 +1,5 @@ /* Appellation: concision-macros - Creator: FL03 - Description: - Useful macros for data-centric workloads + Contrib: FL03 */ +//! # Concision Macros From a30da81cb6ecbd899e4759e214ef8fb137bc1d35 Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 13 Oct 2023 13:13:37 -0500 Subject: [PATCH 012/118] update Signed-off-by: FL03 --- .github/dependabot.yml | 8 + .github/workflows/publish.yml | 23 +++ Cargo.toml | 8 +- concision/Cargo.toml | 29 ++- concision/examples/basic.rs | 17 ++ concision/src/lib.rs | 26 +-- concision/src/math/linalg/mod.rs | 8 - concision/src/math/mod.rs | 12 -- concision/src/nn/mod.rs | 1 - concision/src/num/complex.rs | 97 ---------- concision/src/num/mod.rs | 7 - concision/src/utils.rs | 5 - core/Cargo.toml | 2 - core/src/primitives.rs | 14 +- core/src/specs.rs | 21 --- math/Cargo.toml | 43 +++++ {concision => math}/examples/simple.rs | 5 +- .../src}/calculus/derivatives/mod.rs | 0 .../src}/calculus/derivatives/nderive.rs | 0 .../src/math => math/src}/calculus/mod.rs | 0 .../src/math => math/src}/factorials.rs | 23 ++- math/src/lib.rs | 30 ++++ math/src/linalg/mod.rs | 6 + {concision/src/math => math/src}/linalg/vs.rs | 3 +- math/src/num/arithmetic.rs | 23 +++ math/src/num/complex.rs | 167 ++++++++++++++++++ math/src/num/mod.rs | 36 ++++ {concision => math}/src/primitives.rs | 2 +- {concision => math}/src/specs.rs | 2 +- .../math => math/src}/statistics/deviation.rs | 0 .../src/math => math/src}/statistics/mod.rs | 0 .../src}/statistics/regression/linear.rs | 5 +- .../src}/statistics/regression/mod.rs | 0 math/src/utils.rs | 4 + math/tests/default.rs | 8 + ml/nn/Cargo.toml | 44 +++++ ml/nn/src/lib.rs | 21 +++ ml/nn/src/neurons/activate/mod.rs | 12 ++ ml/nn/src/neurons/mod.rs | 12 ++ ml/nn/src/neurons/neuron.rs | 96 ++++++++++ ml/nn/src/primitives.rs | 14 ++ ml/nn/src/specs.rs | 4 + ml/nn/src/utils.rs | 4 + ml/nn/tests/default.rs | 8 + 44 files changed, 643 insertions(+), 207 deletions(-) create mode 100644 concision/examples/basic.rs delete mode 100644 concision/src/math/linalg/mod.rs delete mode 100644 concision/src/math/mod.rs delete mode 100644 concision/src/nn/mod.rs delete mode 100644 concision/src/num/complex.rs delete mode 100644 concision/src/num/mod.rs delete mode 100644 concision/src/utils.rs create mode 100644 math/Cargo.toml rename {concision => math}/examples/simple.rs (75%) rename {concision/src/math => math/src}/calculus/derivatives/mod.rs (100%) rename {concision/src/math => math/src}/calculus/derivatives/nderive.rs (100%) rename {concision/src/math => math/src}/calculus/mod.rs (100%) rename {concision/src/math => math/src}/factorials.rs (68%) create mode 100644 math/src/lib.rs create mode 100644 math/src/linalg/mod.rs rename {concision/src/math => math/src}/linalg/vs.rs (83%) create mode 100644 math/src/num/arithmetic.rs create mode 100644 math/src/num/complex.rs create mode 100644 math/src/num/mod.rs rename {concision => math}/src/primitives.rs (89%) rename {concision => math}/src/specs.rs (59%) rename {concision/src/math => math/src}/statistics/deviation.rs (100%) rename {concision/src/math => math/src}/statistics/mod.rs (100%) rename {concision/src/math => math/src}/statistics/regression/linear.rs (87%) rename {concision/src/math => math/src}/statistics/regression/mod.rs (100%) create mode 100644 math/src/utils.rs create mode 100644 math/tests/default.rs create mode 100644 ml/nn/Cargo.toml create mode 100644 ml/nn/src/lib.rs create mode 100644 ml/nn/src/neurons/activate/mod.rs create mode 100644 ml/nn/src/neurons/mod.rs create mode 100644 ml/nn/src/neurons/neuron.rs create mode 100644 ml/nn/src/primitives.rs create mode 100644 ml/nn/src/specs.rs create mode 100644 ml/nn/src/utils.rs create mode 100644 ml/nn/tests/default.rs diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fff40f86..97bd8e39 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -22,5 +22,13 @@ updates: interval: daily - package-ecosystem: cargo directory: /macros + schedule: + interval: daily + - package-ecosystem: cargo + directory: /math + schedule: + interval: daily + - package-ecosystem: cargo + directory: /ml/nn schedule: interval: daily \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1cbbeda6..669e205c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,7 +1,12 @@ name: Publish +concurrency: + cancel-in-progress: true + group: ${{ github.event.repository.name }} + env: CARGO_TERM_COLOR: always + CRATE_BASENAME: ${{ github.event.repository.name }} on: release: @@ -14,6 +19,12 @@ on: required: true type: boolean +permissions: + contents: read + packages: write + pull-requests: write + statuses: write + jobs: features: name: Publish (features) @@ -27,6 +38,18 @@ jobs: - uses: actions/checkout@v4 - name: Publish (${{env.PACKAGE_NAME}}) run: cargo publish --all-features -v -p ${{ env.PACKAGE_NAME }} --token ${{ secrets.CARGO_REGISTRY_TOKEN }} + dependents: + name: Publish (dependents) + runs-on: ubuntu-latest + strategy: + matrix: + feature: [ nn ] + env: + PACKAGE_NAME: ${{ github.event.repository.name }}-${{ matrix.feature }} + steps: + - uses: actions/checkout@v4 + - name: Publish (${{env.PACKAGE_NAME}}) + run: cargo publish --all-features -v -p ${{ env.PACKAGE_NAME }} --token ${{ secrets.CARGO_REGISTRY_TOKEN }} publish: name: Publish (sdk) needs: features diff --git a/Cargo.toml b/Cargo.toml index 5a27a2b7..8accc8df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,11 @@ repository = "https://github.com/FL03/concision" version = "0.1.12" # TODO - Update the cargo package version [workspace.dependencies] +concision-core = { path = "core", version = "0.1.12" } + anyhow = "1" +ndarray = { features = ["serde-1"], version = "0.15" } +num = { features = ["serde"], version = "0.4" } serde = { features = ["derive"], version = "1" } serde_json = "1" smart-default = "0.7" @@ -26,7 +30,9 @@ members = [ "concision", "core", "derive", - "macros" + "macros", + "math", + "ml/nn", ] resolver = "2" diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 37ddb405..57236f44 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -12,9 +12,17 @@ repository.workspace = true version.workspace = true [features] -default = ["core"] +default = [ + "core", + "math", +] -full = ["core", "derive"] +full = [ + "core", + "derive", + "math", + "nn" +] core = [ "concision-core/default" @@ -29,6 +37,14 @@ macros = [ "concision-macros" ] +math = [ + "concision-math" +] + +nn = [ + "concision-nn" +] + [lib] bench = true crate-type = ["cdylib", "rlib"] @@ -38,16 +54,11 @@ test = true [build-dependencies] [dependencies] -ndarray = { features = ["blas", "serde-1"], version = "0.15" } -num = { features = [], version = "0.4" } -serde.workspace = true -serde_json.workspace = true -smart-default.workspace = true -strum.workspace = true - concision-core = { features = [], optional = true, path = "../core", version = "0.1.12" } concision-derive = { features = [], optional = true, path = "../derive", version = "0.1.12" } concision-macros = { features = [], optional = true, path = "../macros", version = "0.1.12" } +concision-math = { features = [], optional = true, path = "../math", version = "0.1.12" } +concision-nn = { features = [], optional = true, path = "../ml/nn", version = "0.1.12" } [dev-dependencies] diff --git a/concision/examples/basic.rs b/concision/examples/basic.rs new file mode 100644 index 00000000..2da79037 --- /dev/null +++ b/concision/examples/basic.rs @@ -0,0 +1,17 @@ +extern crate concision; + +use concision::math::num::complex::C; +use concision::prelude::Numerical; + +// Define a holomorphic function that squares its input. +fn square(z: C) -> C { + z.clone() * z +} + +fn main() { + let c = C::from((1.0, 1.0)); + let res = square(c); + assert_eq!(res.clone(), C::from((0.0, 2.0))); + + println!("{:?}", res); +} diff --git a/concision/src/lib.rs b/concision/src/lib.rs index 94479f6f..1cc27484 100644 --- a/concision/src/lib.rs +++ b/concision/src/lib.rs @@ -6,35 +6,27 @@ //! //! Concision aims to be a complete machine learning library written in pure Rust. //! -#[doc(inline)] -pub use crate::{primitives::*, specs::*, utils::*}; + #[cfg(feature = "core")] pub use concision_core as core; #[cfg(feature = "derive")] pub use concision_derive::*; #[cfg(feature = "macros")] pub use concision_macros::*; - -pub mod math; -pub mod nn; -pub mod num; - -pub(crate) mod primitives; -pub(crate) mod specs; -pub(crate) mod utils; +#[cfg(feature = "math")] +pub use concision_math as math; +#[cfg(feature = "nn")] +pub use concision_nn as nn; pub mod prelude { - pub use crate::math::*; - pub use crate::nn::*; - pub use crate::num::*; - pub use crate::primitives::*; - pub use crate::specs::*; - pub use crate::utils::*; - #[cfg(feature = "core")] pub use concision_core::prelude::*; #[cfg(feature = "derive")] pub use concision_derive::*; #[cfg(feature = "macros")] pub use concision_macros::*; + #[cfg(feature = "math")] + pub use concision_math::prelude::*; + #[cfg(feature = "nn")] + pub use concision_nn::prelude::*; } diff --git a/concision/src/math/linalg/mod.rs b/concision/src/math/linalg/mod.rs deleted file mode 100644 index d392460b..00000000 --- a/concision/src/math/linalg/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -/* - Appellation: linalg - Contributors: FL03 (https://gitlab.com/FL03) - Description: - ... Summary ... -*/ - -pub mod vs; diff --git a/concision/src/math/mod.rs b/concision/src/math/mod.rs deleted file mode 100644 index 5bcaf74a..00000000 --- a/concision/src/math/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -/* - Appellation: math - Contrib: FL03 - Description: ... Summary ... -*/ -pub use self::factorials::*; - -pub mod calculus; -pub mod linalg; -pub mod statistics; - -mod factorials; diff --git a/concision/src/nn/mod.rs b/concision/src/nn/mod.rs deleted file mode 100644 index 8b137891..00000000 --- a/concision/src/nn/mod.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/concision/src/num/complex.rs b/concision/src/num/complex.rs deleted file mode 100644 index caef186c..00000000 --- a/concision/src/num/complex.rs +++ /dev/null @@ -1,97 +0,0 @@ -/* - Appellation: complex - Contrib: FL03 -*/ -use crate::core::Numerical; -use serde::{Deserialize, Serialize}; -use std::{ - convert::From, - ops::{Add, Mul}, -}; - -// Define a trait for complex numbers. -pub trait Complex { - fn new(re: T, im: T) -> Self; - fn re(&self) -> T; - fn im(&self) -> T; -} - -// Implement the Complex trait for the tuple (f64, f64). -impl Complex for (f64, f64) { - fn new(re: f64, im: f64) -> Self { - (re, im) - } - - fn re(&self) -> f64 { - self.0 - } - - fn im(&self) -> f64 { - self.1 - } -} - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, PartialOrd, Serialize)] -pub struct C(T, T); - -impl Complex for C { - fn new(re: T, im: T) -> Self { - Self(re, im) - } - fn re(&self) -> T { - self.0 - } - fn im(&self) -> T { - self.1 - } -} - -impl From<(T, T)> for C { - fn from(data: (T, T)) -> Self { - Self::new(data.0, data.1) - } -} - -impl From> for (T, T) { - fn from(data: C) -> (T, T) { - (data.re(), data.im()) - } -} - -// Implement the Add and Mul traits for complex numbers. -impl Add for C { - type Output = Self; - - fn add(self, other: Self) -> Self { - Self::from((self.re() + other.re(), self.im() + other.im())) - } -} - -impl Mul for C { - type Output = Self; - - fn mul(self, other: Self) -> Self { - Self::from(( - self.re() * other.re() - self.im() * other.im(), - self.re() * other.im() + self.im() * other.re(), - )) - } -} - -#[cfg(test)] -mod tests { - use super::*; - // Define a holomorphic function that squares its input. - fn square(z: C) -> C { - z.clone() * z - } - - #[test] - fn test_square() { - // Create a complex number (1 + i) and square it. - let mut a = C::from((1.0, 1.0)); - a = square(a.clone()); - let b = C::new(0.0, 2.0); - assert_eq!(&a, &b); - } -} diff --git a/concision/src/num/mod.rs b/concision/src/num/mod.rs deleted file mode 100644 index cddd73df..00000000 --- a/concision/src/num/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -/* - Appellation: num - Contrib: FL03 - Description: ... Summary ... -*/ - -pub mod complex; diff --git a/concision/src/utils.rs b/concision/src/utils.rs deleted file mode 100644 index 5a7b9ed6..00000000 --- a/concision/src/utils.rs +++ /dev/null @@ -1,5 +0,0 @@ -/* - Appellation: utils - Contrib: FL03 - Description: ... Summary ... -*/ diff --git a/core/Cargo.toml b/core/Cargo.toml index 9f8ac379..06db8b77 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -24,8 +24,6 @@ test = true [build-dependencies] [dependencies] -ndarray = { features = ["blas", "serde-1"], version = "0.15" } -num = { features = [], version = "0.4" } serde.workspace = true serde_json.workspace = true smart-default.workspace = true diff --git a/core/src/primitives.rs b/core/src/primitives.rs index 43288c57..3e1b7491 100644 --- a/core/src/primitives.rs +++ b/core/src/primitives.rs @@ -1,15 +1,19 @@ /* - Appellation: primitives + Appellation: primitives Contrib: FL03 - Description: ... Summary ... */ pub use self::{constants::*, statics::*, types::*}; /// Collection of constants used throughout the system -mod constants {} +pub(crate) mod constants {} /// Collection of static references used throughout -mod statics {} +pub(crate) mod statics {} /// Collection of types used throughout the system -mod types {} +pub(crate) mod types { + /// + pub type BoxError = Box; + /// + pub type BoxResult = std::result::Result; +} diff --git a/core/src/specs.rs b/core/src/specs.rs index b6db4c0c..1d8faa71 100644 --- a/core/src/specs.rs +++ b/core/src/specs.rs @@ -2,24 +2,3 @@ Appellation: specs Contrib: FL03 */ -use std::ops::{Add, Div, Mul, Sub}; - -/// [Numerical] is a basic trait describing numerical objects -pub trait Numerical: - Add - + Div - + Mul - + Sub - + Clone - + Copy - + Sized -{ -} - -impl Numerical for f32 {} - -impl Numerical for f64 {} - -impl Numerical for i64 {} - -impl Numerical for usize {} diff --git a/math/Cargo.toml b/math/Cargo.toml new file mode 100644 index 00000000..63b67c59 --- /dev/null +++ b/math/Cargo.toml @@ -0,0 +1,43 @@ +[package] +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "concision-math" +readme.workspace = true +repository.workspace = true +version.workspace = true + +[features] +default = [] + +[lib] +bench = true +crate-type = ["cdylib", "rlib"] +doctest = false +test = true + +[build-dependencies] + +[dependencies] +concision-core.workspace = true + +ndarray.workspace = true +num.workspace = true +serde.workspace = true +serde_json.workspace = true +smart-default.workspace = true +strum.workspace = true + +[dev-dependencies] + +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] + +[target.wasm32-unknown-unknown] + +[target.wasm32-wasi] \ No newline at end of file diff --git a/concision/examples/simple.rs b/math/examples/simple.rs similarity index 75% rename from concision/examples/simple.rs rename to math/examples/simple.rs index 0351b11e..38a262ea 100644 --- a/concision/examples/simple.rs +++ b/math/examples/simple.rs @@ -1,7 +1,6 @@ -extern crate concision; +extern crate concision_math as math; -use concision::num::complex::C; -use concision::prelude::Numerical; +use math::prelude::{complex::C, Numerical}; // Define a holomorphic function that squares its input. fn square(z: C) -> C { diff --git a/concision/src/math/calculus/derivatives/mod.rs b/math/src/calculus/derivatives/mod.rs similarity index 100% rename from concision/src/math/calculus/derivatives/mod.rs rename to math/src/calculus/derivatives/mod.rs diff --git a/concision/src/math/calculus/derivatives/nderive.rs b/math/src/calculus/derivatives/nderive.rs similarity index 100% rename from concision/src/math/calculus/derivatives/nderive.rs rename to math/src/calculus/derivatives/nderive.rs diff --git a/concision/src/math/calculus/mod.rs b/math/src/calculus/mod.rs similarity index 100% rename from concision/src/math/calculus/mod.rs rename to math/src/calculus/mod.rs diff --git a/concision/src/math/factorials.rs b/math/src/factorials.rs similarity index 68% rename from concision/src/math/factorials.rs rename to math/src/factorials.rs index 0a133213..ef051db7 100644 --- a/concision/src/math/factorials.rs +++ b/math/src/factorials.rs @@ -1,13 +1,9 @@ /* - Appellation: factorials + Appellation: factorials Contrib: FL03 - Description: ... Summary ... */ -use std::{ - ops::{Mul, Sub}, - str::FromStr, - string::ToString, -}; +use std::ops::{Mul, Sub}; +use std::str::FromStr; #[derive(Clone, Debug, Default, Eq, Ord, PartialEq, PartialOrd)] pub struct Factorial + Sub>(T); @@ -37,11 +33,11 @@ where } } -pub fn factorial(data: usize) -> usize { - match data { - 0 | 1 => 1, - _ => factorial(data - 1) * data, +pub fn factorial(data: T) -> T { + if data.is_zero() || data.is_one() { + return T::one(); } + factorial(data - T::one()) * data } #[cfg(test)] @@ -50,6 +46,9 @@ mod tests { #[test] fn test_factorial() { - assert_eq!(Factorial::new(0).data().clone(), 1) + assert_eq!(Factorial::new(0).data().clone(), 1); + assert_eq!(factorial(3), 6); + assert_eq!(factorial(4.0), 24.0); + assert_eq!(factorial(5.0), 120.0); } } diff --git a/math/src/lib.rs b/math/src/lib.rs new file mode 100644 index 00000000..8d1dd740 --- /dev/null +++ b/math/src/lib.rs @@ -0,0 +1,30 @@ +/* + Appellation: concision-math + Contrib: FL03 +*/ +//! # concision-math + +pub use self::{factorials::*, primitives::*, specs::*, utils::*}; + +pub mod calculus; +pub mod linalg; +pub mod num; +pub mod statistics; + +pub(crate) mod factorials; + +// pub(crate) use concision_core as core; + +pub(crate) mod primitives; +pub(crate) mod specs; +pub(crate) mod utils; + +pub mod prelude { + pub use crate::calculus::*; + pub use crate::linalg::*; + pub use crate::num::*; + pub use crate::primitives::*; + pub use crate::specs::*; + pub use crate::statistics::*; + pub use crate::utils::*; +} diff --git a/math/src/linalg/mod.rs b/math/src/linalg/mod.rs new file mode 100644 index 00000000..ca5e460a --- /dev/null +++ b/math/src/linalg/mod.rs @@ -0,0 +1,6 @@ +/* + Appellation: linalg + Contrib: FL03 (https://github.com/FL03) +*/ +//! # linalg +pub mod vs; diff --git a/concision/src/math/linalg/vs.rs b/math/src/linalg/vs.rs similarity index 83% rename from concision/src/math/linalg/vs.rs rename to math/src/linalg/vs.rs index 9790b147..42ab10eb 100644 --- a/concision/src/math/linalg/vs.rs +++ b/math/src/linalg/vs.rs @@ -1,7 +1,6 @@ /* - Appellation: vectors + Appellation: vs Contrib: FL03 - Description: ... Summary ... */ #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] diff --git a/math/src/num/arithmetic.rs b/math/src/num/arithmetic.rs new file mode 100644 index 00000000..8c8fedbe --- /dev/null +++ b/math/src/num/arithmetic.rs @@ -0,0 +1,23 @@ +/* + Appellation: arithmetic + Contrib: FL03 +*/ +use std::ops::{self, Add, Div, Mul, Neg, Sub}; + +pub trait Addition: Add + ops::AddAssign + Sized {} + +impl Addition for T where T: Add + ops::AddAssign + Sized {} + +pub trait Division: Div + ops::DivAssign + Sized {} + +impl Division for T where T: Div + ops::DivAssign + Sized {} + +pub trait Multiplication: Mul + ops::MulAssign + Sized {} + +impl Multiplication for T where T: Mul + ops::MulAssign + Sized {} + +pub trait Subtraction: Sub + ops::SubAssign + Sized {} + +impl Subtraction for T where T: Sub + ops::SubAssign + Sized {} + +pub trait Negative: Neg + Sized {} diff --git a/math/src/num/complex.rs b/math/src/num/complex.rs new file mode 100644 index 00000000..8c0ee0ad --- /dev/null +++ b/math/src/num/complex.rs @@ -0,0 +1,167 @@ +/* + Appellation: complex + Contrib: FL03 +*/ +use super::Numerical; +use serde::{Deserialize, Serialize}; +use std::ops; + +// Define a trait for complex numbers. +pub trait Complex { + fn new(re: T, im: T) -> Self; + fn re(&self) -> T; + fn im(&self) -> T; +} + +// Implement the Complex trait for the tuple (f64, f64). +impl Complex for (T, T) { + fn new(re: T, im: T) -> Self { + (re, im) + } + + fn re(&self) -> T { + self.0 + } + + fn im(&self) -> T { + self.1 + } +} + +impl Complex for [T; 2] { + fn new(re: T, im: T) -> Self { + [re, im] + } + + fn re(&self) -> T { + self[0] + } + + fn im(&self) -> T { + self[1] + } +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, PartialOrd, Serialize)] +pub struct C(T, T); + +impl Complex for C { + fn new(re: T, im: T) -> Self { + Self(re, im) + } + fn re(&self) -> T { + self.0 + } + fn im(&self) -> T { + self.1 + } +} + +impl From<(T, T)> for C { + fn from(data: (T, T)) -> Self { + Self::new(data.0, data.1) + } +} + +impl From> for (T, T) { + fn from(data: C) -> (T, T) { + (data.re(), data.im()) + } +} + +// Implement the Add and Mul traits for complex numbers. +impl ops::Add for C { + type Output = Self; + + fn add(self, other: Self) -> Self { + Self::from((self.re() + other.re(), self.im() + other.im())) + } +} + +impl ops::AddAssign for C +where + T: Numerical + ops::AddAssign, +{ + fn add_assign(&mut self, other: Self) { + self.0 += other.0; + self.1 += other.1; + } +} + +impl ops::Div for C { + type Output = Self; + + fn div(self, other: Self) -> Self { + let denom = other.re() * other.re() + other.im() * other.im(); + Self::from(( + (self.re() * other.re() + self.im() * other.im()) / denom, + (self.im() * other.re() - self.re() * other.im()) / denom, + )) + } +} + +impl ops::DivAssign for C { + fn div_assign(&mut self, other: Self) { + let denom = other.re() * other.re() + other.im() * other.im(); + let re = (self.re() * other.re() + self.im() * other.im()) / denom; + let im = (self.im() * other.re() - self.re() * other.im()) / denom; + self.0 = re; + self.1 = im; + } +} + +impl ops::Mul for C { + type Output = Self; + + fn mul(self, other: Self) -> Self { + Self::from(( + self.re() * other.re() - self.im() * other.im(), + self.re() * other.im() + self.im() * other.re(), + )) + } +} + +impl ops::MulAssign for C { + fn mul_assign(&mut self, other: Self) { + let re = self.re() * other.re() - self.im() * other.im(); + let im = self.re() * other.im() + self.im() * other.re(); + self.0 = re; + self.1 = im; + } +} + +impl ops::Sub for C { + type Output = Self; + + fn sub(self, other: Self) -> Self { + Self::from((self.re() - other.re(), self.im() - other.im())) + } +} + +impl ops::SubAssign for C +where + T: Numerical + ops::SubAssign, +{ + fn sub_assign(&mut self, other: Self) { + self.0 -= other.0; + self.1 -= other.1; + } +} + +#[cfg(test)] +mod tests { + use super::*; + // Define a holomorphic function that squares its input. + fn square(z: C) -> C { + z.clone() * z + } + + #[test] + fn test_square() { + // Create a complex number (1 + i) and square it. + let mut a = C::from((1.0, 1.0)); + a = square(a.clone()); + let b = C::new(0.0, 2.0); + assert_eq!(&a, &b); + } +} diff --git a/math/src/num/mod.rs b/math/src/num/mod.rs new file mode 100644 index 00000000..e220f63d --- /dev/null +++ b/math/src/num/mod.rs @@ -0,0 +1,36 @@ +/* + Appellation: num + Contrib: FL03 +*/ +pub use self::{arithmetic::*, utils::*}; + +pub(crate) mod arithmetic; + +pub mod complex; + +use std::ops::{self, Div, Mul, Sub}; + +pub trait Binary: num::One + num::Zero {} + +impl Binary for T where T: num::One + num::Zero {} + +pub trait Arithmetic: + ops::Add + Div + Mul + Sub + Sized +{ +} + +impl Arithmetic for T where + T: ops::Add + + Div + + Mul + + Sub + + Sized +{ +} + +/// [Numerical] is a basic trait describing numerical objects +pub trait Numerical: Arithmetic + Copy + PartialEq {} + +impl Numerical for T where T: Arithmetic + Copy + PartialEq {} + +pub(crate) mod utils {} diff --git a/concision/src/primitives.rs b/math/src/primitives.rs similarity index 89% rename from concision/src/primitives.rs rename to math/src/primitives.rs index 14b38b52..f16ac25f 100644 --- a/concision/src/primitives.rs +++ b/math/src/primitives.rs @@ -1,5 +1,5 @@ /* - Appellation: primitives + Appellation: primitives Contrib: FL03 */ pub use self::{constants::*, statics::*, types::*}; diff --git a/concision/src/specs.rs b/math/src/specs.rs similarity index 59% rename from concision/src/specs.rs rename to math/src/specs.rs index 1d8faa71..8abf79cb 100644 --- a/concision/src/specs.rs +++ b/math/src/specs.rs @@ -1,4 +1,4 @@ /* - Appellation: specs + Appellation: specs Contrib: FL03 */ diff --git a/concision/src/math/statistics/deviation.rs b/math/src/statistics/deviation.rs similarity index 100% rename from concision/src/math/statistics/deviation.rs rename to math/src/statistics/deviation.rs diff --git a/concision/src/math/statistics/mod.rs b/math/src/statistics/mod.rs similarity index 100% rename from concision/src/math/statistics/mod.rs rename to math/src/statistics/mod.rs diff --git a/concision/src/math/statistics/regression/linear.rs b/math/src/statistics/regression/linear.rs similarity index 87% rename from concision/src/math/statistics/regression/linear.rs rename to math/src/statistics/regression/linear.rs index 2e53fdd9..3ae53777 100644 --- a/concision/src/math/statistics/regression/linear.rs +++ b/math/src/statistics/regression/linear.rs @@ -1,9 +1,8 @@ /* - Appellation: linear + Appellation: linear Contrib: FL03 - Description: ... Summary ... */ -use crate::math::statistics::{covariance, deviation, mean, variance}; +use crate::statistics::{covariance, deviation, mean, variance}; pub struct LinearRegression { pub slope: f64, diff --git a/concision/src/math/statistics/regression/mod.rs b/math/src/statistics/regression/mod.rs similarity index 100% rename from concision/src/math/statistics/regression/mod.rs rename to math/src/statistics/regression/mod.rs diff --git a/math/src/utils.rs b/math/src/utils.rs new file mode 100644 index 00000000..f8840c69 --- /dev/null +++ b/math/src/utils.rs @@ -0,0 +1,4 @@ +/* + Appellation: utils + Contrib: FL03 +*/ diff --git a/math/tests/default.rs b/math/tests/default.rs new file mode 100644 index 00000000..0cac1eb5 --- /dev/null +++ b/math/tests/default.rs @@ -0,0 +1,8 @@ +#[cfg(test)] +#[test] +fn compiles() { + let f = |x: usize, y: usize| x + y; + + assert_eq!(f(10, 10), 20); + assert_ne!(f(1, 1), 3); +} diff --git a/ml/nn/Cargo.toml b/ml/nn/Cargo.toml new file mode 100644 index 00000000..0f061f47 --- /dev/null +++ b/ml/nn/Cargo.toml @@ -0,0 +1,44 @@ +[package] +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "concision-nn" +readme.workspace = true +repository.workspace = true +version.workspace = true + + +[features] +default = [] + +[lib] +bench = true +crate-type = ["cdylib", "rlib"] +doctest = false +test = true + +[build-dependencies] + +[dependencies] +concision-core.workspace = true + +ndarray.workspace = true +num.workspace = true +serde.workspace = true +serde_json.workspace = true +smart-default.workspace = true +strum.workspace = true + +[dev-dependencies] + +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] + +[target.wasm32-unknown-unknown] + +[target.wasm32-wasi] diff --git a/ml/nn/src/lib.rs b/ml/nn/src/lib.rs new file mode 100644 index 00000000..a02adbb0 --- /dev/null +++ b/ml/nn/src/lib.rs @@ -0,0 +1,21 @@ +/* + Appellation: nn + Contrib: FL03 +*/ +//! # concision-nn +pub use self::{primitives::*, specs::*, utils::*}; + +pub(crate) mod primitives; +pub(crate) mod specs; +pub(crate) mod utils; + +pub mod neurons; + +// pub(crate) use concision_core as core; + +pub mod prelude { + pub use crate::neurons::*; + pub use crate::primitives::*; + pub use crate::specs::*; + pub use crate::utils::*; +} diff --git a/ml/nn/src/neurons/activate/mod.rs b/ml/nn/src/neurons/activate/mod.rs new file mode 100644 index 00000000..b092265c --- /dev/null +++ b/ml/nn/src/neurons/activate/mod.rs @@ -0,0 +1,12 @@ +/* + Appellation: activate + Contrib: FL03 +*/ +//! # activate +//! +//! This module contains the activation functions for the neurons. +pub use self::utils::*; + +pub type ActivationFn = fn(T) -> T; + +pub(crate) mod utils {} \ No newline at end of file diff --git a/ml/nn/src/neurons/mod.rs b/ml/nn/src/neurons/mod.rs new file mode 100644 index 00000000..d9f167d5 --- /dev/null +++ b/ml/nn/src/neurons/mod.rs @@ -0,0 +1,12 @@ +/* + Appellation: neurons + Contrib: FL03 +*/ +//! # neurons +pub use self::{neuron::*, utils::*}; + +pub(crate) mod neuron; + +pub mod activate; + +pub(crate) mod utils {} \ No newline at end of file diff --git a/ml/nn/src/neurons/neuron.rs b/ml/nn/src/neurons/neuron.rs new file mode 100644 index 00000000..80bc2a39 --- /dev/null +++ b/ml/nn/src/neurons/neuron.rs @@ -0,0 +1,96 @@ +/* + Appellation: neuron + Contrib: FL03 +*/ +use super::activate::ActivationFn; +use ndarray::prelude::Array1; + +fn _heavyside(x: f64) -> f64 { + if x > 0.0 { + 1.0 + } else { + 0.0 + } +} +pub struct Neuron { + activation: ActivationFn, + bias: f64, + data: Array1, + weights: Array1, +} + +impl Neuron { + pub fn new(activation: ActivationFn, bias: f64, weights: Array1) -> Self { + Self { + activation, + bias, + data: Array1::default(weights.len()), + weights, + } + } + + pub fn bias(&self) -> f64 { + self.bias + } + + pub fn dot(&self) -> f64 { + self.data.dot(&self.weights) + } + + pub fn compute(&self) -> f64 { + self.rho()(self.dot() - self.bias()) + } + + pub fn rho(&self) -> ActivationFn { + self.activation + } + + pub fn set_bias(&mut self, bias: f64) { + self.bias = bias; + } + + pub fn set_data(&mut self, data: Array1) { + self.data = data; + } + + pub fn set_weights(&mut self, weights: Array1) { + self.weights = weights; + } + + pub fn with_data(mut self, data: Array1) -> Self { + self.data = data; + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ndarray::array; + + fn _artificial(args: &Array1, bias: Option, rho: ActivationFn, weights: &Array1) -> f64 { + rho(args.dot(weights) - bias.unwrap_or_default()) + } + + #[test] + fn test_neuron() { + let bias = 0.0; + + let a_data = array![10.0, 10.0, 6.0, 1.0, 8.0]; + let a_weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; + let a = Neuron::new(_heavyside, bias, a_weights.clone()).with_data(a_data.clone()); + + let exp = _artificial(&a_data, Some(bias), _heavyside, &a_weights); + assert_eq!(a.compute(), exp); + + let b_data = array![0.0, 9.0, 3.0, 5.0, 3.0]; + let b_weights = array![2.0, 8.0, 8.0, 0.0, 3.0]; + + let b = Neuron::new(_heavyside, bias, b_weights.clone()).with_data(b_data.clone()); + + let exp = _artificial(&b_data, Some(bias), _heavyside, &b_weights); + assert_eq!(b.compute(), exp); + + assert_eq!(a.dot() + b.dot(), 252.0); + } +} \ No newline at end of file diff --git a/ml/nn/src/primitives.rs b/ml/nn/src/primitives.rs new file mode 100644 index 00000000..b06763f7 --- /dev/null +++ b/ml/nn/src/primitives.rs @@ -0,0 +1,14 @@ +/* + Appellation: primitives + Contrib: FL03 +*/ +pub use self::{constants::*, statics::*, types::*}; + +pub(crate) mod constants {} + +mod statics {} + +pub(crate) mod types { + + +} diff --git a/ml/nn/src/specs.rs b/ml/nn/src/specs.rs new file mode 100644 index 00000000..8abf79cb --- /dev/null +++ b/ml/nn/src/specs.rs @@ -0,0 +1,4 @@ +/* + Appellation: specs + Contrib: FL03 +*/ diff --git a/ml/nn/src/utils.rs b/ml/nn/src/utils.rs new file mode 100644 index 00000000..f8840c69 --- /dev/null +++ b/ml/nn/src/utils.rs @@ -0,0 +1,4 @@ +/* + Appellation: utils + Contrib: FL03 +*/ diff --git a/ml/nn/tests/default.rs b/ml/nn/tests/default.rs new file mode 100644 index 00000000..0cac1eb5 --- /dev/null +++ b/ml/nn/tests/default.rs @@ -0,0 +1,8 @@ +#[cfg(test)] +#[test] +fn compiles() { + let f = |x: usize, y: usize| x + y; + + assert_eq!(f(10, 10), 20); + assert_ne!(f(1, 1), 3); +} From 32e00a67a2fb62db6fcb5b6e8b59f8e4cb02db87 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 14 Oct 2023 11:27:40 -0500 Subject: [PATCH 013/118] update Signed-off-by: FL03 --- Cargo.toml | 1 + math/src/linalg/mod.rs | 9 +++++- math/src/linalg/vs/mod.rs | 39 ++++++++++++++++++++++++++ math/src/linalg/{vs.rs => vs/space.rs} | 17 +++++++++-- math/src/num/arithmetic.rs | 20 ++++++------- ml/nn/src/primitives.rs | 7 +++-- 6 files changed, 76 insertions(+), 17 deletions(-) create mode 100644 math/src/linalg/vs/mod.rs rename math/src/linalg/{vs.rs => vs/space.rs} (58%) diff --git a/Cargo.toml b/Cargo.toml index 8accc8df..e72e89ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ version = "0.1.12" # TODO - Update the cargo package version concision-core = { path = "core", version = "0.1.12" } anyhow = "1" +lazy_static = "1" ndarray = { features = ["serde-1"], version = "0.15" } num = { features = ["serde"], version = "0.4" } serde = { features = ["derive"], version = "1" } diff --git a/math/src/linalg/mod.rs b/math/src/linalg/mod.rs index ca5e460a..788bff98 100644 --- a/math/src/linalg/mod.rs +++ b/math/src/linalg/mod.rs @@ -1,6 +1,13 @@ /* Appellation: linalg - Contrib: FL03 (https://github.com/FL03) + Contrib: FL03 */ //! # linalg +//! +//! This module implements the linear algebra functions and logic required to build +//! efficient neural networks. +pub use self::utils::*; + pub mod vs; + +pub(crate) mod utils {} \ No newline at end of file diff --git a/math/src/linalg/vs/mod.rs b/math/src/linalg/vs/mod.rs new file mode 100644 index 00000000..a8064ec4 --- /dev/null +++ b/math/src/linalg/vs/mod.rs @@ -0,0 +1,39 @@ +/* + Appellation: vs + Contrib: FL03 +*/ +//! # Vector Space +//! +//! A vector space is defined to be a set V, whose elements are called vectors, on which are defined two operations, +//! called addition and multiplication by scalars (real numbers), subject to the ten axioms listed below. +//! +//! ## Axioms +//! +//! 1. Closure under addition +//! 2. Closure under scalar multiplication +//! 3. Commutativity of addition +//! 4. Associativity of addition +//! 5. Additive identity +//! 6. Additive inverse +//! 7. Distributivity of scalar multiplication with respect to vector addition +//! 8. Distributivity of scalar multiplication with respect to field addition +//! 9. Compatibility of scalar multiplication with field multiplication +//! 10. Identity element of scalar multiplication +pub use self::{space::*, utils::*}; + +pub(crate) mod space; + + + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests { + #[test] + fn test() { + let f = |x: usize, y: usize| x + y; + let actual = f(4, 4); + let expected: usize = 8; + assert_eq!(actual, expected) + } +} diff --git a/math/src/linalg/vs.rs b/math/src/linalg/vs/space.rs similarity index 58% rename from math/src/linalg/vs.rs rename to math/src/linalg/vs/space.rs index 42ab10eb..90263d02 100644 --- a/math/src/linalg/vs.rs +++ b/math/src/linalg/vs/space.rs @@ -1,10 +1,21 @@ /* - Appellation: vs + Appellation: space Contrib: FL03 */ -#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] -pub struct VectorSpace(pub Vec); +pub trait Subspace: VectorSpace { + +} + +pub struct Space { + pub data: Vec, + pub shape: Vec, +} + +pub trait VectorSpace { + type Dim; + +} #[cfg(test)] mod tests { diff --git a/math/src/num/arithmetic.rs b/math/src/num/arithmetic.rs index 8c8fedbe..55853287 100644 --- a/math/src/num/arithmetic.rs +++ b/math/src/num/arithmetic.rs @@ -2,22 +2,20 @@ Appellation: arithmetic Contrib: FL03 */ -use std::ops::{self, Add, Div, Mul, Neg, Sub}; +use std::ops; -pub trait Addition: Add + ops::AddAssign + Sized {} +pub trait Addition: ops::Add + ops::AddAssign + Sized {} -impl Addition for T where T: Add + ops::AddAssign + Sized {} +impl Addition for T where T: ops::Add + ops::AddAssign + Sized {} -pub trait Division: Div + ops::DivAssign + Sized {} +pub trait Division: ops::Div + ops::DivAssign + Sized {} -impl Division for T where T: Div + ops::DivAssign + Sized {} +impl Division for T where T: ops::Div + ops::DivAssign + Sized {} -pub trait Multiplication: Mul + ops::MulAssign + Sized {} +pub trait Multiplication: ops::Mul + ops::MulAssign + Sized {} -impl Multiplication for T where T: Mul + ops::MulAssign + Sized {} +impl Multiplication for T where T: ops::Mul + ops::MulAssign + Sized {} -pub trait Subtraction: Sub + ops::SubAssign + Sized {} +pub trait Subtraction: ops::Sub + ops::SubAssign + Sized {} -impl Subtraction for T where T: Sub + ops::SubAssign + Sized {} - -pub trait Negative: Neg + Sized {} +impl Subtraction for T where T: ops::Sub + ops::SubAssign + Sized {} diff --git a/ml/nn/src/primitives.rs b/ml/nn/src/primitives.rs index b06763f7..3d192eec 100644 --- a/ml/nn/src/primitives.rs +++ b/ml/nn/src/primitives.rs @@ -4,11 +4,14 @@ */ pub use self::{constants::*, statics::*, types::*}; -pub(crate) mod constants {} +pub(crate) mod constants { + pub const DEFAULT_BUFFER: usize = 1024; +} -mod statics {} +pub(crate) mod statics {} pub(crate) mod types { + pub type BoxedFunction = Box T>; } From 445ce973a52032cd4e6a0966517b864d6b217d18 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 14 Oct 2023 15:01:15 -0500 Subject: [PATCH 014/118] update Signed-off-by: FL03 --- ml/nn/src/neurons/activate/mod.rs | 5 +++++ ml/nn/src/neurons/mod.rs | 2 ++ 2 files changed, 7 insertions(+) diff --git a/ml/nn/src/neurons/activate/mod.rs b/ml/nn/src/neurons/activate/mod.rs index b092265c..0bd6bb80 100644 --- a/ml/nn/src/neurons/activate/mod.rs +++ b/ml/nn/src/neurons/activate/mod.rs @@ -9,4 +9,9 @@ pub use self::utils::*; pub type ActivationFn = fn(T) -> T; +pub trait Activate { + + fn activate(&mut self, args: &[T]) -> T; +} + pub(crate) mod utils {} \ No newline at end of file diff --git a/ml/nn/src/neurons/mod.rs b/ml/nn/src/neurons/mod.rs index d9f167d5..df86acee 100644 --- a/ml/nn/src/neurons/mod.rs +++ b/ml/nn/src/neurons/mod.rs @@ -9,4 +9,6 @@ pub(crate) mod neuron; pub mod activate; +pub trait Weight {} + pub(crate) mod utils {} \ No newline at end of file From 186a7deb228aa3bad696258655287979d05f28dd Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 15 Oct 2023 10:50:50 -0500 Subject: [PATCH 015/118] update Signed-off-by: FL03 --- Cargo.toml | 2 +- concision/Cargo.toml | 8 +- core/src/lib.rs | 4 +- core/src/{ => step}/linspace.rs | 2 + core/src/step/mod.rs | 11 +++ core/src/step/stepper.rs | 9 ++ math/src/specs.rs | 24 +++++ ml/{nn => neural}/Cargo.toml | 2 +- ml/neural/src/layers/layer.rs | 20 ++++ ml/neural/src/layers/mod.rs | 18 ++++ ml/{nn => neural}/src/lib.rs | 6 +- ml/{nn => neural}/src/neurons/activate/mod.rs | 11 ++- ml/{nn => neural}/src/neurons/mod.rs | 3 +- ml/neural/src/neurons/neuron.rs | 91 ++++++++++++++++++ ml/neural/src/neurons/node.rs | 82 ++++++++++++++++ ml/neural/src/nn/loss/mod.rs | 10 ++ ml/neural/src/nn/mod.rs | 10 ++ ml/{nn => neural}/src/primitives.rs | 0 ml/neural/src/specs.rs | 12 +++ ml/{nn => neural}/src/utils.rs | 0 ml/{nn => neural}/tests/default.rs | 0 ml/nn/src/neurons/neuron.rs | 96 ------------------- ml/nn/src/specs.rs | 4 - 23 files changed, 313 insertions(+), 112 deletions(-) rename core/src/{ => step}/linspace.rs (99%) create mode 100644 core/src/step/mod.rs create mode 100644 core/src/step/stepper.rs rename ml/{nn => neural}/Cargo.toml (96%) create mode 100644 ml/neural/src/layers/layer.rs create mode 100644 ml/neural/src/layers/mod.rs rename ml/{nn => neural}/src/lib.rs (82%) rename ml/{nn => neural}/src/neurons/activate/mod.rs (66%) rename ml/{nn => neural}/src/neurons/mod.rs (72%) create mode 100644 ml/neural/src/neurons/neuron.rs create mode 100644 ml/neural/src/neurons/node.rs create mode 100644 ml/neural/src/nn/loss/mod.rs create mode 100644 ml/neural/src/nn/mod.rs rename ml/{nn => neural}/src/primitives.rs (100%) create mode 100644 ml/neural/src/specs.rs rename ml/{nn => neural}/src/utils.rs (100%) rename ml/{nn => neural}/tests/default.rs (100%) delete mode 100644 ml/nn/src/neurons/neuron.rs delete mode 100644 ml/nn/src/specs.rs diff --git a/Cargo.toml b/Cargo.toml index e72e89ce..5463445f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ members = [ "derive", "macros", "math", - "ml/nn", + "ml/*", ] resolver = "2" diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 57236f44..245fdb45 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -21,7 +21,7 @@ full = [ "core", "derive", "math", - "nn" + "neural" ] core = [ @@ -41,8 +41,8 @@ math = [ "concision-math" ] -nn = [ - "concision-nn" +neural = [ + "concision-neural" ] [lib] @@ -58,7 +58,7 @@ concision-core = { features = [], optional = true, path = "../core", version = " concision-derive = { features = [], optional = true, path = "../derive", version = "0.1.12" } concision-macros = { features = [], optional = true, path = "../macros", version = "0.1.12" } concision-math = { features = [], optional = true, path = "../math", version = "0.1.12" } -concision-nn = { features = [], optional = true, path = "../ml/nn", version = "0.1.12" } +concision-neural = { features = [], optional = true, path = "../ml/neural", version = "0.1.12" } [dev-dependencies] diff --git a/core/src/lib.rs b/core/src/lib.rs index 82848fcf..ac527888 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -9,10 +9,10 @@ pub(crate) mod primitives; pub(crate) mod specs; pub(crate) mod utils; -pub mod linspace; +pub mod step; pub mod prelude { - pub use crate::linspace::*; + pub use crate::step::*; pub use crate::primitives::*; pub use crate::specs::*; diff --git a/core/src/linspace.rs b/core/src/step/linspace.rs similarity index 99% rename from core/src/linspace.rs rename to core/src/step/linspace.rs index 2e9ed2aa..7b0a63c1 100644 --- a/core/src/linspace.rs +++ b/core/src/step/linspace.rs @@ -9,6 +9,8 @@ pub fn round(num: f64, decimals: usize) -> f64 { (num * factor).round() / factor } + + fn calculate_step + ops::Sub>( bounds: Range, capacity: T, diff --git a/core/src/step/mod.rs b/core/src/step/mod.rs new file mode 100644 index 00000000..5d83f386 --- /dev/null +++ b/core/src/step/mod.rs @@ -0,0 +1,11 @@ +/* + Appellation: step + Contrib: FL03 +*/ +pub use self::{stepper::*, utils::*}; + +pub(crate) mod stepper; + +pub mod linspace; + +pub(crate) mod utils {} \ No newline at end of file diff --git a/core/src/step/stepper.rs b/core/src/step/stepper.rs new file mode 100644 index 00000000..e3b767f1 --- /dev/null +++ b/core/src/step/stepper.rs @@ -0,0 +1,9 @@ +/* + Appellation: stepper + Contrib: FL03 +*/ + + +pub struct Stepper { + +} \ No newline at end of file diff --git a/math/src/specs.rs b/math/src/specs.rs index 8abf79cb..46416ffc 100644 --- a/math/src/specs.rs +++ b/math/src/specs.rs @@ -2,3 +2,27 @@ Appellation: specs Contrib: FL03 */ + +pub trait RoundTo { + fn round_to(&self, decimals: usize) -> Self; +} + +impl RoundTo for T where T: num::Float { + fn round_to(&self, decimals: usize) -> Self { + let val = T::from(self.clone()).expect("Failed to convert to type T"); + let factor = T::from(10).expect("").powi(decimals as i32); + (val * factor).round() / factor + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_round_to() { + let num = 1.23456789_f64; + assert_eq!(num.round_to(2), 1.23_f64); + assert_eq!(num.round_to(3), 1.235_f64); + } +} \ No newline at end of file diff --git a/ml/nn/Cargo.toml b/ml/neural/Cargo.toml similarity index 96% rename from ml/nn/Cargo.toml rename to ml/neural/Cargo.toml index 0f061f47..9c6d49fc 100644 --- a/ml/nn/Cargo.toml +++ b/ml/neural/Cargo.toml @@ -6,7 +6,7 @@ edition.workspace = true homepage.workspace = true keywords.workspace = true license.workspace = true -name = "concision-nn" +name = "concision-neural" readme.workspace = true repository.workspace = true version.workspace = true diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs new file mode 100644 index 00000000..0ed36855 --- /dev/null +++ b/ml/neural/src/layers/layer.rs @@ -0,0 +1,20 @@ +/* + Appellation: layer + Contrib: FL03 +*/ +use crate::neurons::Node; + +#[derive(Clone, Debug, PartialEq)] +pub struct Layer { + nodes: Vec, +} + +impl Layer { + pub fn new(nodes: Vec) -> Self { + Self { nodes } + } + + pub fn nodes(&self) -> &[Node] { + &self.nodes + } +} \ No newline at end of file diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs new file mode 100644 index 00000000..7eba6dd4 --- /dev/null +++ b/ml/neural/src/layers/mod.rs @@ -0,0 +1,18 @@ +/* + Appellation: layers + Contrib: FL03 +*/ +//! # Layers +pub use self::{layer::*, utils::*}; + +pub(crate) mod layer; + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests { + // use super::*; + + #[test] + fn test_layer() {} +} \ No newline at end of file diff --git a/ml/nn/src/lib.rs b/ml/neural/src/lib.rs similarity index 82% rename from ml/nn/src/lib.rs rename to ml/neural/src/lib.rs index a02adbb0..34cd613c 100644 --- a/ml/nn/src/lib.rs +++ b/ml/neural/src/lib.rs @@ -1,15 +1,17 @@ /* - Appellation: nn + Appellation: neural Contrib: FL03 */ -//! # concision-nn +//! # concision-neural pub use self::{primitives::*, specs::*, utils::*}; pub(crate) mod primitives; pub(crate) mod specs; pub(crate) mod utils; +pub mod layers; pub mod neurons; +pub mod nn; // pub(crate) use concision_core as core; diff --git a/ml/nn/src/neurons/activate/mod.rs b/ml/neural/src/neurons/activate/mod.rs similarity index 66% rename from ml/nn/src/neurons/activate/mod.rs rename to ml/neural/src/neurons/activate/mod.rs index 0bd6bb80..de738789 100644 --- a/ml/nn/src/neurons/activate/mod.rs +++ b/ml/neural/src/neurons/activate/mod.rs @@ -14,4 +14,13 @@ pub trait Activate { fn activate(&mut self, args: &[T]) -> T; } -pub(crate) mod utils {} \ No newline at end of file +pub(crate) mod utils { + + pub fn heavyside(x: f64) -> f64 { + if x > 0.0 { + 1.0 + } else { + 0.0 + } + } +} \ No newline at end of file diff --git a/ml/nn/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs similarity index 72% rename from ml/nn/src/neurons/mod.rs rename to ml/neural/src/neurons/mod.rs index df86acee..e3ea6651 100644 --- a/ml/nn/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -3,9 +3,10 @@ Contrib: FL03 */ //! # neurons -pub use self::{neuron::*, utils::*}; +pub use self::{neuron::*, node::*, utils::*}; pub(crate) mod neuron; +pub(crate) mod node; pub mod activate; diff --git a/ml/neural/src/neurons/neuron.rs b/ml/neural/src/neurons/neuron.rs new file mode 100644 index 00000000..4346a1bd --- /dev/null +++ b/ml/neural/src/neurons/neuron.rs @@ -0,0 +1,91 @@ +/* + Appellation: neuron + Contrib: FL03 +*/ +use super::activate::ActivationFn; +use ndarray::prelude::Array1; + +/// Artificial Neuron +#[derive(Clone, Debug, PartialEq)] +pub struct Neuron { + activation: ActivationFn, + bias: f64, + weights: Array1, +} + +impl Neuron { + pub fn new(activation: ActivationFn, bias: f64, weights: Array1) -> Self { + Self { + activation, + bias, + weights, + } + } + + pub fn bias(&self) -> f64 { + self.bias + } + + pub fn compute(&self, args: &Array1) -> f64 { + let dot = args.dot(&self.weights); + self.rho()(dot - self.bias()) + } + + pub fn process(&self, args: impl AsRef<[f64]>) -> f64 { + let data = Array1::from(args.as_ref().to_vec()); + let dot = data.dot(&self.weights); + self.rho()(dot - self.bias()) + } + + pub fn rho(&self) -> ActivationFn { + self.activation + } + + pub fn weights(&self) -> &Array1 { + &self.weights + } + + pub fn set_bias(&mut self, bias: f64) { + self.bias = bias; + } + + + pub fn set_weights(&mut self, weights: Array1) { + self.weights = weights; + } + +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::neurons::activate::heavyside; + use ndarray::array; + + fn _artificial(args: &[f64], bias: Option, rho: ActivationFn, weights: &Array1) -> f64 { + let data = Array1::from(args.to_vec()); + rho(data.dot(weights) - bias.unwrap_or_default()) + } + + #[test] + fn test_neuron() { + let bias = 0.0; + + let a_data = [10.0, 10.0, 6.0, 1.0, 8.0]; + let a_weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; + let a = Neuron::new(heavyside, bias, a_weights.clone()); + + let exp = _artificial(&a_data, Some(bias), heavyside, &a_weights); + assert_eq!(a.process(&a_data), exp); + + let b_data = [0.0, 9.0, 3.0, 5.0, 3.0]; + let b_weights = array![2.0, 8.0, 8.0, 0.0, 3.0]; + + let b = Neuron::new(heavyside, bias, b_weights.clone()); + + let exp = _artificial(&b_data, Some(bias), heavyside, &b_weights); + assert_eq!(b.process(&b_data), exp); + + // assert_eq!(a.dot() + b.dot(), 252.0); + } +} \ No newline at end of file diff --git a/ml/neural/src/neurons/node.rs b/ml/neural/src/neurons/node.rs new file mode 100644 index 00000000..6cca0376 --- /dev/null +++ b/ml/neural/src/neurons/node.rs @@ -0,0 +1,82 @@ +/* + Appellation: node + Contrib: FL03 +*/ +use super::Neuron; + +use ndarray::prelude::Array1; + +#[derive(Clone, Debug, PartialEq)] +pub struct Node { + data: Array1, + neuron: Neuron, +} + +impl Node { + pub fn new(neuron: Neuron) -> Self { + let shape = neuron.weights().shape(); + Self { data: Array1::default([shape[0]]), neuron } + } + + pub fn data(&self) -> &Array1 { + &self.data + } + + pub fn dot(&self) -> f64 { + self.data.dot(self.neuron.weights()) + } + + pub fn neuron(&self) -> &Neuron { + &self.neuron + } + + pub fn process(&self) -> f64 { + self.neuron.compute(&self.data) + } + + pub fn set_data(&mut self, data: Array1) { + self.data = data; + } + + pub fn with_data(mut self, data: Array1) -> Self { + self.data = data; + self + } +} + + + +#[cfg(test)] +mod tests { + use super::*; + use crate::neurons::activate::{heavyside, ActivationFn}; + use ndarray::array; + + fn _artificial(args: &[f64], bias: Option, rho: ActivationFn, weights: &Array1) -> f64 { + let data = Array1::from(args.to_vec()); + rho(data.dot(weights) - bias.unwrap_or_default()) + } + + #[test] + fn test_node() { + let bias = 0.0; + + let a_data = [10.0, 10.0, 6.0, 1.0, 8.0]; + let a_weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; + let a = Neuron::new(heavyside, bias, a_weights.clone()); + let node_a = Node::new(a.clone()).with_data(Array1::from(a_data.to_vec())); + + let exp = _artificial(&a_data, Some(bias), heavyside, &a_weights); + assert_eq!(node_a.process(), exp); + + let b_data = [0.0, 9.0, 3.0, 5.0, 3.0]; + let b_weights = array![2.0, 8.0, 8.0, 0.0, 3.0]; + + let b = Neuron::new(heavyside, bias, b_weights.clone()); + let node_b = Node::new(b.clone()).with_data(Array1::from(b_data.to_vec())); + let exp = _artificial(&b_data, Some(bias), heavyside, &b_weights); + assert_eq!(node_b.process(), exp); + + assert_eq!(node_a.dot() + node_b.dot(), 252.0); + } +} \ No newline at end of file diff --git a/ml/neural/src/nn/loss/mod.rs b/ml/neural/src/nn/loss/mod.rs new file mode 100644 index 00000000..7049e0fc --- /dev/null +++ b/ml/neural/src/nn/loss/mod.rs @@ -0,0 +1,10 @@ +/* + Appellation: loss + Contrib: FL03 +*/ +//! # Loss Functions +pub use self::utils::*; + + + +pub(crate) mod utils {} \ No newline at end of file diff --git a/ml/neural/src/nn/mod.rs b/ml/neural/src/nn/mod.rs new file mode 100644 index 00000000..6f0e8aaf --- /dev/null +++ b/ml/neural/src/nn/mod.rs @@ -0,0 +1,10 @@ +/* + Appellation: nn + Contrib: FL03 +*/ +//! # Neural Network +pub use self::utils::*; + +pub mod loss; + +pub(crate) mod utils {} \ No newline at end of file diff --git a/ml/nn/src/primitives.rs b/ml/neural/src/primitives.rs similarity index 100% rename from ml/nn/src/primitives.rs rename to ml/neural/src/primitives.rs diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs new file mode 100644 index 00000000..093e01dc --- /dev/null +++ b/ml/neural/src/specs.rs @@ -0,0 +1,12 @@ +/* + Appellation: specs + Contrib: FL03 +*/ + +pub trait NeuralNetwork: Trainable { + +} + +pub trait Trainable { + fn train(&mut self, args: &[f64]) -> f64; +} \ No newline at end of file diff --git a/ml/nn/src/utils.rs b/ml/neural/src/utils.rs similarity index 100% rename from ml/nn/src/utils.rs rename to ml/neural/src/utils.rs diff --git a/ml/nn/tests/default.rs b/ml/neural/tests/default.rs similarity index 100% rename from ml/nn/tests/default.rs rename to ml/neural/tests/default.rs diff --git a/ml/nn/src/neurons/neuron.rs b/ml/nn/src/neurons/neuron.rs deleted file mode 100644 index 80bc2a39..00000000 --- a/ml/nn/src/neurons/neuron.rs +++ /dev/null @@ -1,96 +0,0 @@ -/* - Appellation: neuron - Contrib: FL03 -*/ -use super::activate::ActivationFn; -use ndarray::prelude::Array1; - -fn _heavyside(x: f64) -> f64 { - if x > 0.0 { - 1.0 - } else { - 0.0 - } -} -pub struct Neuron { - activation: ActivationFn, - bias: f64, - data: Array1, - weights: Array1, -} - -impl Neuron { - pub fn new(activation: ActivationFn, bias: f64, weights: Array1) -> Self { - Self { - activation, - bias, - data: Array1::default(weights.len()), - weights, - } - } - - pub fn bias(&self) -> f64 { - self.bias - } - - pub fn dot(&self) -> f64 { - self.data.dot(&self.weights) - } - - pub fn compute(&self) -> f64 { - self.rho()(self.dot() - self.bias()) - } - - pub fn rho(&self) -> ActivationFn { - self.activation - } - - pub fn set_bias(&mut self, bias: f64) { - self.bias = bias; - } - - pub fn set_data(&mut self, data: Array1) { - self.data = data; - } - - pub fn set_weights(&mut self, weights: Array1) { - self.weights = weights; - } - - pub fn with_data(mut self, data: Array1) -> Self { - self.data = data; - self - } -} - -#[cfg(test)] -mod tests { - use super::*; - use ndarray::array; - - fn _artificial(args: &Array1, bias: Option, rho: ActivationFn, weights: &Array1) -> f64 { - rho(args.dot(weights) - bias.unwrap_or_default()) - } - - #[test] - fn test_neuron() { - let bias = 0.0; - - let a_data = array![10.0, 10.0, 6.0, 1.0, 8.0]; - let a_weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; - let a = Neuron::new(_heavyside, bias, a_weights.clone()).with_data(a_data.clone()); - - let exp = _artificial(&a_data, Some(bias), _heavyside, &a_weights); - assert_eq!(a.compute(), exp); - - let b_data = array![0.0, 9.0, 3.0, 5.0, 3.0]; - let b_weights = array![2.0, 8.0, 8.0, 0.0, 3.0]; - - let b = Neuron::new(_heavyside, bias, b_weights.clone()).with_data(b_data.clone()); - - let exp = _artificial(&b_data, Some(bias), _heavyside, &b_weights); - assert_eq!(b.compute(), exp); - - assert_eq!(a.dot() + b.dot(), 252.0); - } -} \ No newline at end of file diff --git a/ml/nn/src/specs.rs b/ml/nn/src/specs.rs deleted file mode 100644 index 8abf79cb..00000000 --- a/ml/nn/src/specs.rs +++ /dev/null @@ -1,4 +0,0 @@ -/* - Appellation: specs - Contrib: FL03 -*/ From d72c8543e2730e7ff96eaa73b4156f70575c9cbb Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 15 Oct 2023 12:41:53 -0500 Subject: [PATCH 016/118] update Signed-off-by: FL03 --- core/Cargo.toml | 2 + core/src/errors/error.rs | 102 ++++++++++++++++++++++++++ core/src/errors/mod.rs | 9 +++ core/src/lib.rs | 2 + core/src/step/linspace.rs | 2 - core/src/step/mod.rs | 2 +- core/src/step/stepper.rs | 5 +- core/src/utils.rs | 7 ++ math/src/linalg/mod.rs | 4 +- math/src/linalg/vs/mod.rs | 10 +-- math/src/linalg/vs/space.rs | 5 +- math/src/specs.rs | 7 +- ml/neural/src/layers/layer.rs | 34 ++++++++- ml/neural/src/layers/mod.rs | 2 +- ml/neural/src/neurons/activate/mod.rs | 7 +- ml/neural/src/neurons/mod.rs | 2 +- ml/neural/src/neurons/neuron.rs | 11 ++- ml/neural/src/neurons/node.rs | 16 ++-- ml/neural/src/nn/loss/mod.rs | 23 +++++- ml/neural/src/nn/mod.rs | 2 +- ml/neural/src/primitives.rs | 1 - ml/neural/src/specs.rs | 6 +- 22 files changed, 216 insertions(+), 45 deletions(-) create mode 100644 core/src/errors/error.rs create mode 100644 core/src/errors/mod.rs diff --git a/core/Cargo.toml b/core/Cargo.toml index 06db8b77..c1ddc070 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -24,6 +24,8 @@ test = true [build-dependencies] [dependencies] +anyhow.workspace = true +chrono = "0.4" serde.workspace = true serde_json.workspace = true smart-default.workspace = true diff --git a/core/src/errors/error.rs b/core/src/errors/error.rs new file mode 100644 index 00000000..b87c4e3a --- /dev/null +++ b/core/src/errors/error.rs @@ -0,0 +1,102 @@ +/* + Appellation: error + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use smart_default::SmartDefault; +use strum::{Display, EnumIs, EnumIter, EnumVariantNames}; + +#[derive( + Clone, + Debug, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, + SmartDefault, +)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum Errors { + Async, + Connection, + #[default] + Error(String), + Execution, + IO, + Process, + Runtime, + Syntax, +} + +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "lowercase")] +pub struct Error { + pub kind: Errors, + pub message: String, + pub ts: u128, +} + +impl Error { + pub fn new(kind: Errors, message: String) -> Self { + let ts = crate::now(); + Self { kind, message, ts } + } + + pub fn kind(&self) -> &Errors { + &self.kind + } + + pub fn message(&self) -> &str { + &self.message + } + + pub fn ts(&self) -> u128 { + self.ts + } + + pub fn set_kind(&mut self, kind: Errors) { + self.kind = kind; + self.on_update(); + } + + pub fn set_message(&mut self, message: String) { + self.message = message; + self.on_update(); + } + + pub fn with_kind(mut self, kind: Errors) -> Self { + self.kind = kind; + self + } + + pub fn with_message(mut self, message: String) -> Self { + self.message = message; + self + } + + fn on_update(&mut self) { + self.ts = crate::now(); + } +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "Error: {}", self.message()) + } +} + +impl std::error::Error for Error {} + +impl From for Error { + fn from(err: std::io::Error) -> Self { + Self::new(Errors::IO, err.to_string()) + } +} diff --git a/core/src/errors/mod.rs b/core/src/errors/mod.rs new file mode 100644 index 00000000..267dace8 --- /dev/null +++ b/core/src/errors/mod.rs @@ -0,0 +1,9 @@ +/* + Appellation: errors + Contrib: FL03 +*/ +pub use self::{error::*, utils::*}; + +pub(crate) mod error; + +pub(crate) mod utils {} diff --git a/core/src/lib.rs b/core/src/lib.rs index ac527888..ec57f2c4 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -9,9 +9,11 @@ pub(crate) mod primitives; pub(crate) mod specs; pub(crate) mod utils; +pub mod errors; pub mod step; pub mod prelude { + pub use crate::errors::*; pub use crate::step::*; pub use crate::primitives::*; diff --git a/core/src/step/linspace.rs b/core/src/step/linspace.rs index 7b0a63c1..2e9ed2aa 100644 --- a/core/src/step/linspace.rs +++ b/core/src/step/linspace.rs @@ -9,8 +9,6 @@ pub fn round(num: f64, decimals: usize) -> f64 { (num * factor).round() / factor } - - fn calculate_step + ops::Sub>( bounds: Range, capacity: T, diff --git a/core/src/step/mod.rs b/core/src/step/mod.rs index 5d83f386..6cb2c8e0 100644 --- a/core/src/step/mod.rs +++ b/core/src/step/mod.rs @@ -8,4 +8,4 @@ pub(crate) mod stepper; pub mod linspace; -pub(crate) mod utils {} \ No newline at end of file +pub(crate) mod utils {} diff --git a/core/src/step/stepper.rs b/core/src/step/stepper.rs index e3b767f1..07909956 100644 --- a/core/src/step/stepper.rs +++ b/core/src/step/stepper.rs @@ -3,7 +3,4 @@ Contrib: FL03 */ - -pub struct Stepper { - -} \ No newline at end of file +pub struct Stepper {} diff --git a/core/src/utils.rs b/core/src/utils.rs index 5a7b9ed6..c57e8121 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -3,3 +3,10 @@ Contrib: FL03 Description: ... Summary ... */ + +pub fn now() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() +} diff --git a/math/src/linalg/mod.rs b/math/src/linalg/mod.rs index 788bff98..1b4dc1a0 100644 --- a/math/src/linalg/mod.rs +++ b/math/src/linalg/mod.rs @@ -3,11 +3,11 @@ Contrib: FL03 */ //! # linalg -//! +//! //! This module implements the linear algebra functions and logic required to build //! efficient neural networks. pub use self::utils::*; pub mod vs; -pub(crate) mod utils {} \ No newline at end of file +pub(crate) mod utils {} diff --git a/math/src/linalg/vs/mod.rs b/math/src/linalg/vs/mod.rs index a8064ec4..b5f7ba9f 100644 --- a/math/src/linalg/vs/mod.rs +++ b/math/src/linalg/vs/mod.rs @@ -3,12 +3,12 @@ Contrib: FL03 */ //! # Vector Space -//! -//! A vector space is defined to be a set V, whose elements are called vectors, on which are defined two operations, +//! +//! A vector space is defined to be a set V, whose elements are called vectors, on which are defined two operations, //! called addition and multiplication by scalars (real numbers), subject to the ten axioms listed below. -//! +//! //! ## Axioms -//! +//! //! 1. Closure under addition //! 2. Closure under scalar multiplication //! 3. Commutativity of addition @@ -23,8 +23,6 @@ pub use self::{space::*, utils::*}; pub(crate) mod space; - - pub(crate) mod utils {} #[cfg(test)] diff --git a/math/src/linalg/vs/space.rs b/math/src/linalg/vs/space.rs index 90263d02..dfaa0730 100644 --- a/math/src/linalg/vs/space.rs +++ b/math/src/linalg/vs/space.rs @@ -3,9 +3,7 @@ Contrib: FL03 */ -pub trait Subspace: VectorSpace { - -} +pub trait Subspace: VectorSpace {} pub struct Space { pub data: Vec, @@ -14,7 +12,6 @@ pub struct Space { pub trait VectorSpace { type Dim; - } #[cfg(test)] diff --git a/math/src/specs.rs b/math/src/specs.rs index 46416ffc..31f77629 100644 --- a/math/src/specs.rs +++ b/math/src/specs.rs @@ -7,7 +7,10 @@ pub trait RoundTo { fn round_to(&self, decimals: usize) -> Self; } -impl RoundTo for T where T: num::Float { +impl RoundTo for T +where + T: num::Float, +{ fn round_to(&self, decimals: usize) -> Self { let val = T::from(self.clone()).expect("Failed to convert to type T"); let factor = T::from(10).expect("").powi(decimals as i32); @@ -25,4 +28,4 @@ mod tests { assert_eq!(num.round_to(2), 1.23_f64); assert_eq!(num.round_to(3), 1.235_f64); } -} \ No newline at end of file +} diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 0ed36855..c3f6341a 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -3,8 +3,9 @@ Contrib: FL03 */ use crate::neurons::Node; +use std::ops; -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, Default, PartialEq)] pub struct Layer { nodes: Vec, } @@ -17,4 +18,33 @@ impl Layer { pub fn nodes(&self) -> &[Node] { &self.nodes } -} \ No newline at end of file +} + +impl ops::Index for Layer { + type Output = Node; + + fn index(&self, index: usize) -> &Self::Output { + &self.nodes[index] + } +} + +impl ops::IndexMut for Layer { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.nodes[index] + } +} + +impl ops::Index> for Layer { + type Output = [Node]; + + fn index(&self, index: ops::Range) -> &Self::Output { + &self.nodes[index] + } +} + +impl ops::IndexMut> for Layer { + fn index_mut(&mut self, index: ops::Range) -> &mut Self::Output { + &mut self.nodes[index] + } +} + diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index 7eba6dd4..18c80230 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -15,4 +15,4 @@ mod tests { #[test] fn test_layer() {} -} \ No newline at end of file +} diff --git a/ml/neural/src/neurons/activate/mod.rs b/ml/neural/src/neurons/activate/mod.rs index de738789..aac4bd2e 100644 --- a/ml/neural/src/neurons/activate/mod.rs +++ b/ml/neural/src/neurons/activate/mod.rs @@ -3,19 +3,18 @@ Contrib: FL03 */ //! # activate -//! +//! //! This module contains the activation functions for the neurons. pub use self::utils::*; pub type ActivationFn = fn(T) -> T; pub trait Activate { - fn activate(&mut self, args: &[T]) -> T; } pub(crate) mod utils { - + pub fn heavyside(x: f64) -> f64 { if x > 0.0 { 1.0 @@ -23,4 +22,4 @@ pub(crate) mod utils { 0.0 } } -} \ No newline at end of file +} diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index e3ea6651..2818ca7e 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -12,4 +12,4 @@ pub mod activate; pub trait Weight {} -pub(crate) mod utils {} \ No newline at end of file +pub(crate) mod utils {} diff --git a/ml/neural/src/neurons/neuron.rs b/ml/neural/src/neurons/neuron.rs index 4346a1bd..d42ff4a6 100644 --- a/ml/neural/src/neurons/neuron.rs +++ b/ml/neural/src/neurons/neuron.rs @@ -49,11 +49,9 @@ impl Neuron { self.bias = bias; } - pub fn set_weights(&mut self, weights: Array1) { self.weights = weights; } - } #[cfg(test)] @@ -62,7 +60,12 @@ mod tests { use crate::neurons::activate::heavyside; use ndarray::array; - fn _artificial(args: &[f64], bias: Option, rho: ActivationFn, weights: &Array1) -> f64 { + fn _artificial( + args: &[f64], + bias: Option, + rho: ActivationFn, + weights: &Array1, + ) -> f64 { let data = Array1::from(args.to_vec()); rho(data.dot(weights) - bias.unwrap_or_default()) } @@ -88,4 +91,4 @@ mod tests { // assert_eq!(a.dot() + b.dot(), 252.0); } -} \ No newline at end of file +} diff --git a/ml/neural/src/neurons/node.rs b/ml/neural/src/neurons/node.rs index 6cca0376..e0127a65 100644 --- a/ml/neural/src/neurons/node.rs +++ b/ml/neural/src/neurons/node.rs @@ -15,7 +15,10 @@ pub struct Node { impl Node { pub fn new(neuron: Neuron) -> Self { let shape = neuron.weights().shape(); - Self { data: Array1::default([shape[0]]), neuron } + Self { + data: Array1::default([shape[0]]), + neuron, + } } pub fn data(&self) -> &Array1 { @@ -44,15 +47,18 @@ impl Node { } } - - #[cfg(test)] mod tests { use super::*; use crate::neurons::activate::{heavyside, ActivationFn}; use ndarray::array; - fn _artificial(args: &[f64], bias: Option, rho: ActivationFn, weights: &Array1) -> f64 { + fn _artificial( + args: &[f64], + bias: Option, + rho: ActivationFn, + weights: &Array1, + ) -> f64 { let data = Array1::from(args.to_vec()); rho(data.dot(weights) - bias.unwrap_or_default()) } @@ -79,4 +85,4 @@ mod tests { assert_eq!(node_a.dot() + node_b.dot(), 252.0); } -} \ No newline at end of file +} diff --git a/ml/neural/src/nn/loss/mod.rs b/ml/neural/src/nn/loss/mod.rs index 7049e0fc..e448b36b 100644 --- a/ml/neural/src/nn/loss/mod.rs +++ b/ml/neural/src/nn/loss/mod.rs @@ -3,8 +3,29 @@ Contrib: FL03 */ //! # Loss Functions +//! +//! Loss functions consider the differences between predicted and target outputs. +//! Overall, neural network models aim to minimize the average loss by adjusting certain hyperparameters, +//! the weights and biases. + pub use self::utils::*; +pub trait Loss { + fn loss(&self, pred: f64, target: f64) -> f64; +} +pub(crate) mod utils { + use ndarray::Array1; -pub(crate) mod utils {} \ No newline at end of file + pub fn mse(pred: Array1, target: Array1) -> f64 { + if pred.shape() != target.shape() { + panic!( + "Mismatched shapes: {:?} and {:?}", + pred.shape(), + target.shape() + ); + } + let n = pred.len() as f64; + (target - pred).mapv(|x| x.powi(2)).sum() / n + } +} diff --git a/ml/neural/src/nn/mod.rs b/ml/neural/src/nn/mod.rs index 6f0e8aaf..fceb374f 100644 --- a/ml/neural/src/nn/mod.rs +++ b/ml/neural/src/nn/mod.rs @@ -7,4 +7,4 @@ pub use self::utils::*; pub mod loss; -pub(crate) mod utils {} \ No newline at end of file +pub(crate) mod utils {} diff --git a/ml/neural/src/primitives.rs b/ml/neural/src/primitives.rs index 3d192eec..92a83797 100644 --- a/ml/neural/src/primitives.rs +++ b/ml/neural/src/primitives.rs @@ -13,5 +13,4 @@ pub(crate) mod statics {} pub(crate) mod types { pub type BoxedFunction = Box T>; - } diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index 093e01dc..7b91c0ef 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -3,10 +3,8 @@ Contrib: FL03 */ -pub trait NeuralNetwork: Trainable { - -} +pub trait NeuralNetwork: Trainable {} pub trait Trainable { fn train(&mut self, args: &[f64]) -> f64; -} \ No newline at end of file +} From 24e8f54a9d27c97d10df6a02227f9816d49ed914 Mon Sep 17 00:00:00 2001 From: FL03 Date: Mon, 16 Oct 2023 09:55:39 -0500 Subject: [PATCH 017/118] update Signed-off-by: FL03 --- ml/neural/src/neurons/activate/mod.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ml/neural/src/neurons/activate/mod.rs b/ml/neural/src/neurons/activate/mod.rs index aac4bd2e..f36510d2 100644 --- a/ml/neural/src/neurons/activate/mod.rs +++ b/ml/neural/src/neurons/activate/mod.rs @@ -10,7 +10,12 @@ pub use self::utils::*; pub type ActivationFn = fn(T) -> T; pub trait Activate { - fn activate(&mut self, args: &[T]) -> T; + + fn activate(&mut self) -> T; +} + +pub trait ActivateWith { + fn activate_with(&mut self, args: &[T]) -> T; } pub(crate) mod utils { From e37e935d8f29bce2327b4e9abba459d8c6349827 Mon Sep 17 00:00:00 2001 From: FL03 Date: Mon, 16 Oct 2023 11:10:15 -0500 Subject: [PATCH 018/118] update Signed-off-by: FL03 --- math/src/statistics/deviation.rs | 34 +++++++++++-- math/src/statistics/mod.rs | 61 ++++++++++++++---------- math/src/statistics/regression/linear.rs | 4 +- ml/neural/src/layers/kinds.rs | 34 +++++++++++++ ml/neural/src/layers/layer.rs | 36 ++++++++++++-- ml/neural/src/layers/mod.rs | 3 +- ml/neural/src/neurons/activate/mod.rs | 1 - ml/neural/src/nn/loss/mod.rs | 14 ++++++ ml/neural/src/nn/loss/regress.rs | 10 ++++ 9 files changed, 161 insertions(+), 36 deletions(-) create mode 100644 ml/neural/src/layers/kinds.rs create mode 100644 ml/neural/src/nn/loss/regress.rs diff --git a/math/src/statistics/deviation.rs b/math/src/statistics/deviation.rs index d6fd8945..c14ab1c1 100644 --- a/math/src/statistics/deviation.rs +++ b/math/src/statistics/deviation.rs @@ -1,24 +1,48 @@ /* - Appellation: deviation + Appellation: deviation Contrib: FL03 - Description: ... Summary ... */ +use super::utils::*; +use serde::{Deserialize, Serialize}; +#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, PartialOrd, Serialize)] pub struct StandardDeviation { + pub deviation: f64, pub mean: f64, pub variance: f64, - pub deviation: f64, } impl StandardDeviation { pub fn new(x: &[f64]) -> StandardDeviation { - let mean = x.iter().sum::() / x.len() as f64; + let mean = mean(x); let variance = x.iter().map(|&x| x * x).sum::() / x.len() as f64 - mean * mean; let deviation = variance.sqrt(); StandardDeviation { + deviation, mean, variance, - deviation, } } + + pub fn compute(&mut self, x: &[f64]) -> f64 { + let mean = x.iter().sum::() / x.len() as f64; + let variance = x.iter().map(|&x| x * x).sum::() / x.len() as f64 - mean * mean; + let deviation = variance.sqrt(); + self.deviation = deviation; + self.mean = mean; + self.variance = variance; + deviation + } + + pub fn deviation(&self) -> f64 { + self.deviation + } + + pub fn mean(&self) -> f64 { + self.mean + } + + pub fn variance(&self) -> f64 { + self.variance + } } diff --git a/math/src/statistics/mod.rs b/math/src/statistics/mod.rs index c42b0cae..1efd8a1b 100644 --- a/math/src/statistics/mod.rs +++ b/math/src/statistics/mod.rs @@ -1,33 +1,47 @@ /* Appellation: statistics Contrib: FL03 - Description: ... Summary ... */ -pub use self::deviation::*; +//! # statistics +pub use self::{deviation::*, utils::*}; pub mod regression; -mod deviation; +pub(crate) mod deviation; -/// Covariance is the average of the products of the deviations from the mean. -pub fn covariance(x: Vec, y: Vec) -> f64 { - let dx = deviation(&x, mean(&x)); - let dy = deviation(&y, mean(&y)); - dx.iter().zip(dy.iter()).map(|(&x, &y)| x * y).sum::() / dx.len() as f64 -} -/// Deviation is the distance from the mean. -pub fn deviation(x: &[f64], mean: f64) -> Vec { - x.iter().map(|&x| x - mean).collect() -} -/// Mean is the average of the data. -pub fn mean(x: &[f64]) -> f64 { - x.iter().sum::() / x.len() as f64 +pub trait Statistics {} + +pub trait Mean: + Clone + IntoIterator + ExactSizeIterator +{ + fn mean(&self) -> T { + self.clone().into_iter().sum::() / T::from(self.len()).unwrap() + } } -/// Variance is the average of the squared deviations from the mean. -pub fn variance(x: Vec) -> f64 { - let mean = mean(&x); - let dev = deviation(&x, mean); - dev.iter().map(|&x| x * x).sum::() / dev.len() as f64 + +pub(crate) mod utils { + use std::iter::Sum; + + /// Covariance is the average of the products of the deviations from the mean. + pub fn covariance(x: Vec, y: Vec) -> T { + let dx = deviation(&x); + let dy = deviation(&y); + dx.iter().zip(dy.iter()).map(|(&x, &y)| x * y).sum::() / T::from(dx.len()).unwrap() + } + /// Deviation is the distance from the mean. + pub fn deviation(x: &[T]) -> Vec { + let mean = mean(x); + x.iter().map(|&x| x - mean).collect() + } + /// Mean is the average of the data. + pub fn mean(x: &[T]) -> T { + x.iter().cloned().sum::() / T::from(x.len()).unwrap() + } + /// Variance is the average of the squared deviations from the mean. + pub fn variance(x: &[T]) -> T { + let dev = deviation(&x); + dev.iter().map(|&x| x * x).sum::() / T::from(dev.len()).unwrap() + } } #[cfg(test)] @@ -44,8 +58,7 @@ mod tests { #[test] fn test_deviation() { let x = vec![1.0, 2.0, 3.0, 4.0, 5.0]; - let mean = mean(&x); - assert_eq!(deviation(&x, mean), vec![-2.0, -1.0, 0.0, 1.0, 2.0]); + assert_eq!(deviation(&x), vec![-2.0, -1.0, 0.0, 1.0, 2.0]); } #[test] @@ -57,6 +70,6 @@ mod tests { #[test] fn test_variance() { let x = vec![1.0, 2.0, 3.0, 4.0, 5.0]; - assert_eq!(variance(x), 2.0); + assert_eq!(variance(&x), 2.0); } } diff --git a/math/src/statistics/regression/linear.rs b/math/src/statistics/regression/linear.rs index 3ae53777..21ff61fa 100644 --- a/math/src/statistics/regression/linear.rs +++ b/math/src/statistics/regression/linear.rs @@ -12,8 +12,8 @@ pub struct LinearRegression { impl LinearRegression { pub fn new(x: &[f64], y: &[f64]) -> LinearRegression { let (x_mean, y_mean) = (mean(x), mean(y)); - let (x_dev, y_dev) = (deviation(x, x_mean), deviation(y, y_mean)); - let slope = covariance(x_dev.clone(), y_dev) / variance(x_dev); + let (x_dev, y_dev) = (deviation(x), deviation(y)); + let slope = covariance(x_dev.clone(), y_dev) / variance(&x_dev); let intercept = y_mean - slope * x_mean; LinearRegression { slope, intercept } } diff --git a/ml/neural/src/layers/kinds.rs b/ml/neural/src/layers/kinds.rs new file mode 100644 index 00000000..6d6451a1 --- /dev/null +++ b/ml/neural/src/layers/kinds.rs @@ -0,0 +1,34 @@ +/* + Appellation: kinds + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum LayerType { + #[default] + Input = 0, + Hidden(usize), + Output, +} diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index c3f6341a..6ddc5b8f 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -2,22 +2,53 @@ Appellation: layer Contrib: FL03 */ +use super::LayerType; use crate::neurons::Node; use std::ops; #[derive(Clone, Debug, Default, PartialEq)] pub struct Layer { + layer: LayerType, nodes: Vec, } impl Layer { - pub fn new(nodes: Vec) -> Self { - Self { nodes } + pub fn new(layer: LayerType, nodes: Vec) -> Self { + Self { layer, nodes } + } + + pub fn layer(&self) -> &LayerType { + &self.layer } pub fn nodes(&self) -> &[Node] { &self.nodes } + + pub fn set_layer(&mut self, layer: LayerType) { + self.layer = layer; + } +} + +impl AsRef<[Node]> for Layer { + fn as_ref(&self) -> &[Node] { + &self.nodes + } +} + +impl AsMut<[Node]> for Layer { + fn as_mut(&mut self) -> &mut [Node] { + &mut self.nodes + } +} + +impl IntoIterator for Layer { + type Item = Node; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.nodes.into_iter() + } } impl ops::Index for Layer { @@ -47,4 +78,3 @@ impl ops::IndexMut> for Layer { &mut self.nodes[index] } } - diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index 18c80230..8b6405cd 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -3,8 +3,9 @@ Contrib: FL03 */ //! # Layers -pub use self::{layer::*, utils::*}; +pub use self::{kinds::*, layer::*, utils::*}; +pub(crate) mod kinds; pub(crate) mod layer; pub(crate) mod utils {} diff --git a/ml/neural/src/neurons/activate/mod.rs b/ml/neural/src/neurons/activate/mod.rs index f36510d2..0e5ec803 100644 --- a/ml/neural/src/neurons/activate/mod.rs +++ b/ml/neural/src/neurons/activate/mod.rs @@ -10,7 +10,6 @@ pub use self::utils::*; pub type ActivationFn = fn(T) -> T; pub trait Activate { - fn activate(&mut self) -> T; } diff --git a/ml/neural/src/nn/loss/mod.rs b/ml/neural/src/nn/loss/mod.rs index e448b36b..a713dc55 100644 --- a/ml/neural/src/nn/loss/mod.rs +++ b/ml/neural/src/nn/loss/mod.rs @@ -10,6 +10,8 @@ pub use self::utils::*; +pub mod regress; + pub trait Loss { fn loss(&self, pred: f64, target: f64) -> f64; } @@ -17,6 +19,18 @@ pub trait Loss { pub(crate) mod utils { use ndarray::Array1; + pub fn mae(pred: Array1, target: Array1) -> f64 { + if pred.shape() != target.shape() { + panic!( + "Mismatched shapes: {:?} and {:?}", + pred.shape(), + target.shape() + ); + } + let n = pred.len() as f64; + (target - pred).mapv(|x| x.abs()).sum() / n + } + pub fn mse(pred: Array1, target: Array1) -> f64 { if pred.shape() != target.shape() { panic!( diff --git a/ml/neural/src/nn/loss/regress.rs b/ml/neural/src/nn/loss/regress.rs new file mode 100644 index 00000000..df40c4e9 --- /dev/null +++ b/ml/neural/src/nn/loss/regress.rs @@ -0,0 +1,10 @@ +/* + Appellation: regress + Contrib: FL03 +*/ + +pub enum RegressiveLoss { + MeanAbsoluteError, + MeanSquaredError, + RootMeanSquaredError, +} From 1bff67dff724aa341005e2f8213f8c96d8f83abb Mon Sep 17 00:00:00 2001 From: FL03 Date: Tue, 17 Oct 2023 04:36:01 -0500 Subject: [PATCH 019/118] update Signed-off-by: FL03 --- .github/dependabot.yml | 4 ++ .github/workflows/clippy.yml | 14 +++-- .github/workflows/rust.yml | 29 ++++----- Cargo.toml | 1 + concision/Cargo.toml | 5 ++ concision/src/lib.rs | 4 ++ core/Cargo.toml | 4 +- core/src/errors/error.rs | 38 ++++++++++++ core/src/step/linspace.rs | 16 ----- data/Cargo.toml | 43 +++++++++++++ data/benches/default.rs | 52 ++++++++++++++++ data/src/df/dataframe.rs | 23 +++++++ data/src/df/mod.rs | 10 +++ data/src/lib.rs | 20 ++++++ data/src/primitives.rs | 19 ++++++ data/src/specs.rs | 4 ++ data/src/utils.rs | 12 ++++ data/tests/default.rs | 8 +++ math/Cargo.toml | 1 + math/src/statistics/deviation.rs | 15 +++++ math/src/statistics/mod.rs | 79 +++++++++++++++++++++--- math/src/statistics/regression/linear.rs | 33 ++++++++-- math/src/statistics/regression/mod.rs | 15 ++++- math/src/utils.rs | 5 ++ ml/neural/Cargo.toml | 1 + ml/neural/src/nn/loss/kinds.rs | 43 +++++++++++++ ml/neural/src/nn/loss/mod.rs | 30 ++++++--- ml/neural/src/nn/loss/regress.rs | 39 +++++++++++- 28 files changed, 506 insertions(+), 61 deletions(-) create mode 100644 data/Cargo.toml create mode 100644 data/benches/default.rs create mode 100644 data/src/df/dataframe.rs create mode 100644 data/src/df/mod.rs create mode 100644 data/src/lib.rs create mode 100644 data/src/primitives.rs create mode 100644 data/src/specs.rs create mode 100644 data/src/utils.rs create mode 100644 data/tests/default.rs create mode 100644 ml/neural/src/nn/loss/kinds.rs diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 97bd8e39..44be9222 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -16,6 +16,10 @@ updates: directory: /core schedule: interval: daily + - package-ecosystem: cargo + directory: /data + schedule: + interval: daily - package-ecosystem: cargo directory: /derive schedule: diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml index 297efdc3..95a63a2f 100644 --- a/.github/workflows/clippy.yml +++ b/.github/workflows/clippy.yml @@ -7,18 +7,20 @@ on: branches-ignore: [ "beta*", "dev*", "next*" ] tags: [ "nightly*", "v*.*.*" ] release: - types: [created] + repository_dispatch: + types: [ analysis ] schedule: - - cron: "30 9 * * *" # 9:30am UTC + - cron: "30 21 * * *" # 9:30pm UTC workflow_dispatch: +permissions: + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + contents: read + security-events: write + jobs: clippy: name: Clippy - permissions: - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - contents: read - security-events: write runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 87c73405..caac03ae 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,33 +1,34 @@ name: Rust +concurrency: + cancel-in-progress: false + group: ${{ github.event.repository.name }}-rust + env: CARGO_TERM_COLOR: always on: pull_request: - branches-ignore: [ "beta*", "dev*", "next*" ] + branches: [ "main", "v*.*.*" ] push: - branches-ignore: [ "beta*", "dev*", "next*" ] + branches: [ "main" ] tags: [ "nightly*", "v*.*.*" ] release: repository_dispatch: - types: [ rust-publish ] + types: [ rust ] schedule: - - cron: "30 9 * * *" # 9:30am UTC + - cron: "30 21 * * *" # 9:30pm UTC workflow_dispatch: - inputs: - publish: - default: true - description: 'Publish' - required: true - type: boolean + +permissions: write-all jobs: build: name: Build and Test strategy: matrix: - platform: [ macos-latest, ubuntu-latest, windows-latest ] + platform: [ ubuntu-latest ] + target: [ wasm32-unknown-unknown, wasm32-wasi ] toolchain: [ stable, nightly ] runs-on: ${{ matrix.platform }} steps: @@ -37,17 +38,17 @@ jobs: rustup update rustup default ${{ matrix.toolchain }} - name: Build - run: cargo build -F full --release -v --workspace + run: cargo build -r -v --workspace - name: Cache build uses: actions/cache@v3 with: + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} path: | ~/.cargo/registry ~/.cargo/git target/release - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Test - run: cargo test --all -F full --release -v + run: cargo test --all-features -r -v --workspace - name: Bench if: matrix.toolchain == 'nightly' run: cargo bench --all -v diff --git a/Cargo.toml b/Cargo.toml index 5463445f..932d4942 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ default-members = [ members = [ "concision", "core", + "data", "derive", "macros", "math", diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 245fdb45..ced26616 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -28,6 +28,10 @@ core = [ "concision-core/default" ] +data = [ + "concision-data/default" +] + derive = [ "concision-derive", "macros", @@ -55,6 +59,7 @@ test = true [dependencies] concision-core = { features = [], optional = true, path = "../core", version = "0.1.12" } +concision-data = { features = [], optional = true, path = "../data", version = "0.1.12" } concision-derive = { features = [], optional = true, path = "../derive", version = "0.1.12" } concision-macros = { features = [], optional = true, path = "../macros", version = "0.1.12" } concision-math = { features = [], optional = true, path = "../math", version = "0.1.12" } diff --git a/concision/src/lib.rs b/concision/src/lib.rs index 1cc27484..63f1ef7f 100644 --- a/concision/src/lib.rs +++ b/concision/src/lib.rs @@ -9,6 +9,8 @@ #[cfg(feature = "core")] pub use concision_core as core; +#[cfg(feature = "data")] +pub use concision_data as data; #[cfg(feature = "derive")] pub use concision_derive::*; #[cfg(feature = "macros")] @@ -21,6 +23,8 @@ pub use concision_nn as nn; pub mod prelude { #[cfg(feature = "core")] pub use concision_core::prelude::*; + #[cfg(feature = "data")] + pub use concision_data::prelude::*; #[cfg(feature = "derive")] pub use concision_derive::*; #[cfg(feature = "macros")] diff --git a/core/Cargo.toml b/core/Cargo.toml index c1ddc070..79dea624 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -14,9 +14,8 @@ version.workspace = true [features] default = [] - [lib] -bench = true +bench = false crate-type = ["cdylib", "rlib"] doctest = false test = true @@ -26,6 +25,7 @@ test = true [dependencies] anyhow.workspace = true chrono = "0.4" +num.workspace = true serde.workspace = true serde_json.workspace = true smart-default.workspace = true diff --git a/core/src/errors/error.rs b/core/src/errors/error.rs index b87c4e3a..126e2ebc 100644 --- a/core/src/errors/error.rs +++ b/core/src/errors/error.rs @@ -27,6 +27,7 @@ use strum::{Display, EnumIs, EnumIter, EnumVariantNames}; pub enum Errors { Async, Connection, + #[default] Error(String), Execution, @@ -34,6 +35,7 @@ pub enum Errors { Process, Runtime, Syntax, + Unknown, } #[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] @@ -95,8 +97,44 @@ impl std::fmt::Display for Error { impl std::error::Error for Error {} +impl From for Error { + fn from(err: Errors) -> Self { + Self::new(err, String::new()) + } +} + +impl From<&str> for Error { + fn from(err: &str) -> Self { + Self::new(Errors::Unknown, err.to_string()) + } +} + +impl From for Error { + fn from(err: String) -> Self { + Self::new(Errors::Unknown, err) + } +} + impl From for Error { fn from(err: std::io::Error) -> Self { Self::new(Errors::IO, err.to_string()) } } + +impl From for Error { + fn from(err: std::num::ParseFloatError) -> Self { + Self::new(Errors::Syntax, err.to_string()) + } +} + +impl From for Error { + fn from(err: std::num::ParseIntError) -> Self { + Self::new(Errors::Syntax, err.to_string()) + } +} + +impl From for Error { + fn from(err: anyhow::Error) -> Self { + Self::new(Errors::Unknown, err.to_string()) + } +} \ No newline at end of file diff --git a/core/src/step/linspace.rs b/core/src/step/linspace.rs index 2e9ed2aa..48580a2a 100644 --- a/core/src/step/linspace.rs +++ b/core/src/step/linspace.rs @@ -4,11 +4,6 @@ */ use std::ops::{self, Range}; -pub fn round(num: f64, decimals: usize) -> f64 { - let factor = 10.0_f64.powi(decimals as i32); - (num * factor).round() / factor -} - fn calculate_step + ops::Sub>( bounds: Range, capacity: T, @@ -88,17 +83,6 @@ impl Linspace { mod tests { use super::*; - #[test] - fn test_rounding() { - let val = 1.23456789; - assert_eq!(round(val, 3), 1.235); - assert_eq!(round(val, 4), 1.2346); - assert_eq!(round(val, 5), 1.23457); - assert_eq!(round(val, 6), 1.234568); - assert_eq!(round(val, 7), 1.2345679); - assert_eq!(round(val, 8), 1.23456789); - } - #[test] fn test_linspace_simple() { let bounds = 0.0..10.0; diff --git a/data/Cargo.toml b/data/Cargo.toml new file mode 100644 index 00000000..df4ae40c --- /dev/null +++ b/data/Cargo.toml @@ -0,0 +1,43 @@ +[package] +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "concision-data" +readme.workspace = true +repository.workspace = true +version.workspace = true + +[features] +default = [] + + +[lib] +bench = true +crate-type = ["cdylib", "rlib"] +doctest = false +test = true + +[build-dependencies] + +[dependencies] +anyhow.workspace = true +ndarray.workspace = true +num.workspace = true +serde.workspace = true +serde_json.workspace = true +smart-default.workspace = true +strum.workspace = true + +[dev-dependencies] + +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] + +[target.wasm32-unknown-unknown] + +[target.wasm32-wasi] diff --git a/data/benches/default.rs b/data/benches/default.rs new file mode 100644 index 00000000..937f2387 --- /dev/null +++ b/data/benches/default.rs @@ -0,0 +1,52 @@ +// bench.rs +#![feature(test)] + +extern crate test; + +use std::mem::replace; +use test::Bencher; + +// bench: find the `BENCH_SIZE` first terms of the fibonacci sequence +static BENCH_SIZE: usize = 20; + +// recursive fibonacci +fn fibonacci(n: usize) -> u32 { + if n < 2 { + 1 + } else { + fibonacci(n - 1) + fibonacci(n - 2) + } +} + +// iterative fibonacci +struct Fibonacci { + curr: u32, + next: u32, +} + +impl Iterator for Fibonacci { + type Item = u32; + fn next(&mut self) -> Option { + let new_next = self.curr + self.next; + let new_curr = replace(&mut self.next, new_next); + + Some(replace(&mut self.curr, new_curr)) + } +} + +fn fibonacci_sequence() -> Fibonacci { + Fibonacci { curr: 1, next: 1 } +} + +// function to benchmark must be annotated with `#[bench]` +#[bench] +fn recursive_fibonacci(b: &mut Bencher) { + // exact code to benchmark must be passed as a closure to the iter + // method of Bencher + b.iter(|| (0..BENCH_SIZE).map(fibonacci).collect::>()) +} + +#[bench] +fn iterative_fibonacci(b: &mut Bencher) { + b.iter(|| fibonacci_sequence().take(BENCH_SIZE).collect::>()) +} diff --git a/data/src/df/dataframe.rs b/data/src/df/dataframe.rs new file mode 100644 index 00000000..13a257e3 --- /dev/null +++ b/data/src/df/dataframe.rs @@ -0,0 +1,23 @@ +/* + Appellation: dataframe + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "lowercase")] +pub struct DataFrame { + data: Vec, +} + +impl DataFrame { + pub fn new() -> Self { + Self { data: Vec::new() } + } +} + +impl std::fmt::Display for DataFrame { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", serde_json::to_string(self).unwrap()) + } +} diff --git a/data/src/df/mod.rs b/data/src/df/mod.rs new file mode 100644 index 00000000..d5d9cfa7 --- /dev/null +++ b/data/src/df/mod.rs @@ -0,0 +1,10 @@ +/* + Appellation: df + Contrib: FL03 +*/ +//! # DataFrame +pub use self::{dataframe::*, utils::*}; + +pub(crate) mod dataframe; + +pub(crate) mod utils {} diff --git a/data/src/lib.rs b/data/src/lib.rs new file mode 100644 index 00000000..9a87d65c --- /dev/null +++ b/data/src/lib.rs @@ -0,0 +1,20 @@ +/* + Appellation: data + Contrib: FL03 +*/ +//! # Concision Data +pub use self::{primitives::*, specs::*, utils::*}; + +pub(crate) mod primitives; +pub(crate) mod specs; +pub(crate) mod utils; + +pub mod df; + +pub mod prelude { + pub use crate::df::*; + + pub use crate::primitives::*; + pub use crate::specs::*; + pub use crate::utils::*; +} diff --git a/data/src/primitives.rs b/data/src/primitives.rs new file mode 100644 index 00000000..3e1b7491 --- /dev/null +++ b/data/src/primitives.rs @@ -0,0 +1,19 @@ +/* + Appellation: primitives + Contrib: FL03 +*/ +pub use self::{constants::*, statics::*, types::*}; + +/// Collection of constants used throughout the system +pub(crate) mod constants {} + +/// Collection of static references used throughout +pub(crate) mod statics {} + +/// Collection of types used throughout the system +pub(crate) mod types { + /// + pub type BoxError = Box; + /// + pub type BoxResult = std::result::Result; +} diff --git a/data/src/specs.rs b/data/src/specs.rs new file mode 100644 index 00000000..1d8faa71 --- /dev/null +++ b/data/src/specs.rs @@ -0,0 +1,4 @@ +/* + Appellation: specs + Contrib: FL03 +*/ diff --git a/data/src/utils.rs b/data/src/utils.rs new file mode 100644 index 00000000..c57e8121 --- /dev/null +++ b/data/src/utils.rs @@ -0,0 +1,12 @@ +/* + Appellation: utils + Contrib: FL03 + Description: ... Summary ... +*/ + +pub fn now() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() +} diff --git a/data/tests/default.rs b/data/tests/default.rs new file mode 100644 index 00000000..0cac1eb5 --- /dev/null +++ b/data/tests/default.rs @@ -0,0 +1,8 @@ +#[cfg(test)] +#[test] +fn compiles() { + let f = |x: usize, y: usize| x + y; + + assert_eq!(f(10, 10), 20); + assert_ne!(f(1, 1), 3); +} diff --git a/math/Cargo.toml b/math/Cargo.toml index 63b67c59..3990bb49 100644 --- a/math/Cargo.toml +++ b/math/Cargo.toml @@ -33,6 +33,7 @@ smart-default.workspace = true strum.workspace = true [dev-dependencies] +rand = "0.8" [package.metadata.docs.rs] all-features = true diff --git a/math/src/statistics/deviation.rs b/math/src/statistics/deviation.rs index c14ab1c1..a38fb9f0 100644 --- a/math/src/statistics/deviation.rs +++ b/math/src/statistics/deviation.rs @@ -46,3 +46,18 @@ impl StandardDeviation { self.variance } } + +#[cfg(test)] +mod tests { + use super::*; + + use crate::prelude::{RoundTo, Statistics}; + + #[test] + fn test_std() { + let x = vec![1.0, 2.0, 4.0, 9.0, 3.0, 4.0, 5.0]; + let sd = StandardDeviation::new(&x); + + assert_eq!(x.std().round_to(5), sd.deviation().round_to(5)); + } +} diff --git a/math/src/statistics/mod.rs b/math/src/statistics/mod.rs index 1efd8a1b..76818145 100644 --- a/math/src/statistics/mod.rs +++ b/math/src/statistics/mod.rs @@ -9,21 +9,65 @@ pub mod regression; pub(crate) mod deviation; -pub trait Statistics {} - -pub trait Mean: - Clone + IntoIterator + ExactSizeIterator +pub trait Statistics +where + T: num::Float + std::iter::Sum, + Self: Clone + IntoIterator, { + fn covariance(&self, other: &Self) -> T { + let dx = self.deviation(); + let dy = other.deviation(); + dx.iter().zip(dy.iter()).map(|(&x, &y)| x * y).sum::() / T::from(dx.len()).unwrap() + } + + fn deviation(&self) -> Vec { + let mean = self.mean(); + self.clone().into_iter().map(|x| x - mean).collect() + } + fn len(&self) -> usize { + Vec::from_iter(self.clone().into_iter()).len() + } + /// [Statistics::mean] calculates the mean or average of the data fn mean(&self) -> T { self.clone().into_iter().sum::() / T::from(self.len()).unwrap() } + /// [Statistics::std] calculates the standard deviation of the data + fn std(&self) -> T { + let mean = self.mean(); + let mut res = self + .clone() + .into_iter() + .map(|x| (x - mean).powi(2)) + .sum::(); + res = res / T::from(self.len()).unwrap(); + res.sqrt() + } + + fn variance(&self) -> T { + let dev = self.deviation(); + dev.iter().map(|&x| x * x).sum::() / T::from(dev.len()).unwrap() + } +} + +impl Statistics for Vec +where + T: num::Float + std::iter::Sum, + Self: Clone + IntoIterator, +{ +} + +impl Statistics for ndarray::Array1 +where + T: num::Float + std::iter::Sum, + Self: Clone + IntoIterator, +{ } pub(crate) mod utils { use std::iter::Sum; /// Covariance is the average of the products of the deviations from the mean. - pub fn covariance(x: Vec, y: Vec) -> T { + pub fn covariance(x: &[T], y: &[T]) -> T { let dx = deviation(&x); let dy = deviation(&y); dx.iter().zip(dy.iter()).map(|(&x, &y)| x * y).sum::() / T::from(dx.len()).unwrap() @@ -47,12 +91,33 @@ pub(crate) mod utils { #[cfg(test)] mod tests { use super::*; + use rand::Rng; + + fn random_vec() -> Vec { + let mut rng = rand::thread_rng(); + let mut v = Vec::new(); + for _ in 0..100 { + v.push(rng.gen_range(0.0..25.0)); + } + v + } + + #[test] + fn test_statistics() { + let x = random_vec(); + let y = random_vec(); + assert_eq!(covariance(&x, &y), x.covariance(&y)); + assert_eq!(deviation(&x), x.deviation()); + assert_eq!(mean(&x), x.mean()); + assert_eq!(variance(&x), x.variance()); + } #[test] fn test_covariance() { - let x = vec![1.0, 2.0, 3.0, 4.0, 5.0]; + let x: Vec = (1..=5).map(|i| i as f64).collect(); let y = vec![1.0, 2.0, 3.0, 4.0, 5.0]; - assert_eq!(covariance(x, y), 2.0); + assert_eq!(covariance(&x, &y), 2.0); + assert_eq!(x.covariance(&y), 2.0); } #[test] diff --git a/math/src/statistics/regression/linear.rs b/math/src/statistics/regression/linear.rs index 21ff61fa..fc78bd78 100644 --- a/math/src/statistics/regression/linear.rs +++ b/math/src/statistics/regression/linear.rs @@ -2,25 +2,50 @@ Appellation: linear Contrib: FL03 */ +use super::Regression; use crate::statistics::{covariance, deviation, mean, variance}; +#[derive(Clone, Copy, Debug, Default, PartialEq, PartialOrd)] pub struct LinearRegression { - pub slope: f64, - pub intercept: f64, + intercept: f64, + slope: f64, } impl LinearRegression { pub fn new(x: &[f64], y: &[f64]) -> LinearRegression { let (x_mean, y_mean) = (mean(x), mean(y)); let (x_dev, y_dev) = (deviation(x), deviation(y)); - let slope = covariance(x_dev.clone(), y_dev) / variance(&x_dev); + let slope = covariance(&x_dev, &y_dev) / variance(&x_dev); let intercept = y_mean - slope * x_mean; - LinearRegression { slope, intercept } + LinearRegression { intercept, slope } + } + + pub fn intercept(&self) -> f64 { + self.intercept } pub fn predict(&self, x: &[f64]) -> Vec { x.iter().map(|&x| self.slope * x + self.intercept).collect() } + + pub fn slope(&self) -> f64 { + self.slope + } +} + +impl Regression for LinearRegression { + type Item = f64; + + fn fit(&mut self, args: &[Self::Item], target: &[Self::Item]) { + let (x_mean, y_mean) = (mean(args), mean(target)); + let (x_dev, y_dev) = (deviation(args), deviation(target)); + self.slope = covariance(&x_dev, &y_dev) / variance(&x_dev); + self.intercept = y_mean - self.slope * x_mean; + } + + fn predict(&self, args: &[Self::Item]) -> Vec { + args.iter().map(|&x| self.slope * x + self.intercept).collect() + } } #[cfg(test)] diff --git a/math/src/statistics/regression/mod.rs b/math/src/statistics/regression/mod.rs index 259b9c19..af241f52 100644 --- a/math/src/statistics/regression/mod.rs +++ b/math/src/statistics/regression/mod.rs @@ -1,7 +1,20 @@ /* Appellation: regression Contrib: FL03 - Description: ... Summary ... */ +//! # regression +//! +pub use self::utils::*; pub mod linear; + +pub trait Regression { + type Item: num::Float; + + fn fit(&mut self, args: &[Self::Item], target: &[Self::Item]); + + fn predict(&self, args: &[Self::Item]) -> Vec; +} + + +pub(crate) mod utils {} diff --git a/math/src/utils.rs b/math/src/utils.rs index f8840c69..e880e121 100644 --- a/math/src/utils.rs +++ b/math/src/utils.rs @@ -2,3 +2,8 @@ Appellation: utils Contrib: FL03 */ + +pub fn round(num: T, decimals: usize) -> T { + let factor = T::from(10.0).expect("").powi(decimals as i32); + (num * factor).round() / factor +} diff --git a/ml/neural/Cargo.toml b/ml/neural/Cargo.toml index 9c6d49fc..ae0b6aef 100644 --- a/ml/neural/Cargo.toml +++ b/ml/neural/Cargo.toml @@ -26,6 +26,7 @@ test = true [dependencies] concision-core.workspace = true +anyhow.workspace = true ndarray.workspace = true num.workspace = true serde.workspace = true diff --git a/ml/neural/src/nn/loss/kinds.rs b/ml/neural/src/nn/loss/kinds.rs new file mode 100644 index 00000000..676dd32b --- /dev/null +++ b/ml/neural/src/nn/loss/kinds.rs @@ -0,0 +1,43 @@ +/* + Appellation: kinds + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum LossKind { + Classification, + #[default] + Regression, +} + +impl LossKind { + pub fn classification() -> Self { + Self::Classification + } + + pub fn regression() -> Self { + Self::Regression + } +} diff --git a/ml/neural/src/nn/loss/mod.rs b/ml/neural/src/nn/loss/mod.rs index a713dc55..a6a7066c 100644 --- a/ml/neural/src/nn/loss/mod.rs +++ b/ml/neural/src/nn/loss/mod.rs @@ -8,38 +8,48 @@ //! Overall, neural network models aim to minimize the average loss by adjusting certain hyperparameters, //! the weights and biases. -pub use self::utils::*; +pub use self::{kinds::*, utils::*}; + +pub(crate) mod kinds; pub mod regress; pub trait Loss { - fn loss(&self, pred: f64, target: f64) -> f64; + fn loss(&self, pred: &[f64], target: &[f64]) -> f64; } + + pub(crate) mod utils { use ndarray::Array1; - pub fn mae(pred: Array1, target: Array1) -> f64 { + pub fn mae(pred: Array1, target: Array1) -> anyhow::Result { if pred.shape() != target.shape() { - panic!( + return Err(anyhow::anyhow!( "Mismatched shapes: {:?} and {:?}", pred.shape(), target.shape() - ); + )); } + // the number of elements in the array let n = pred.len() as f64; - (target - pred).mapv(|x| x.abs()).sum() / n + let mut res = (target - pred).mapv(|x| x.abs()).sum(); + res /= n; + Ok(res) } - pub fn mse(pred: Array1, target: Array1) -> f64 { + pub fn mse(pred: Array1, target: Array1) -> anyhow::Result { if pred.shape() != target.shape() { - panic!( + return Err(anyhow::anyhow!( "Mismatched shapes: {:?} and {:?}", pred.shape(), target.shape() - ); + )); } + let n = pred.len() as f64; - (target - pred).mapv(|x| x.powi(2)).sum() / n + let mut res = (target - pred).mapv(|x| x.powi(2)).sum(); + res /= n; + Ok(res) } } diff --git a/ml/neural/src/nn/loss/regress.rs b/ml/neural/src/nn/loss/regress.rs index df40c4e9..86defa8f 100644 --- a/ml/neural/src/nn/loss/regress.rs +++ b/ml/neural/src/nn/loss/regress.rs @@ -2,9 +2,46 @@ Appellation: regress Contrib: FL03 */ +use super::Loss; pub enum RegressiveLoss { + Huber(HuberLoss), MeanAbsoluteError, MeanSquaredError, - RootMeanSquaredError, + Other(String), +} + +pub struct HuberLoss { + delta: f64, +} + +impl HuberLoss { + pub fn new(delta: f64) -> Self { + Self { delta } + } + + pub fn delta(&self) -> f64 { + self.delta + } + + pub fn set_delta(&mut self, delta: f64) { + self.delta = delta; + } +} + +impl Loss for HuberLoss { + fn loss(&self, pred: &[f64], target: &[f64]) -> f64 { + let mut loss = 0.0; + for (x, y) in pred.iter().zip(target.iter()) { + let diff = x - y; + if diff.abs() <= self.delta { + // If the difference is sufficiently small, use the squared error. + loss += 0.5 * diff.powi(2); + } else { + // Otherwise, use a variant of the absolute error. + loss += self.delta * (diff.abs() - 0.5 * self.delta); + } + } + loss / pred.len() as f64 + } } From 330174d9cff9825dd52b3bbccd50ff687554f86c Mon Sep 17 00:00:00 2001 From: FL03 Date: Tue, 17 Oct 2023 11:13:42 -0500 Subject: [PATCH 020/118] update Signed-off-by: FL03 --- concision/Cargo.toml | 12 +++++- concision/src/lib.rs | 4 ++ core/src/errors/error.rs | 2 +- math/src/statistics/mod.rs | 14 +------ math/src/statistics/regression/linear.rs | 4 +- math/src/statistics/regression/mod.rs | 1 - ml/neural/src/nn/loss/mod.rs | 2 - ml/transformers/Cargo.toml | 43 ++++++++++++++++++++ ml/transformers/benches/default.rs | 52 ++++++++++++++++++++++++ ml/transformers/src/attention/head.rs | 23 +++++++++++ ml/transformers/src/attention/mod.rs | 18 ++++++++ ml/transformers/src/codec/decode.rs | 6 +++ ml/transformers/src/codec/encode.rs | 6 +++ ml/transformers/src/codec/mod.rs | 19 +++++++++ ml/transformers/src/lib.rs | 22 ++++++++++ ml/transformers/src/primitives.rs | 19 +++++++++ ml/transformers/src/specs.rs | 4 ++ ml/transformers/src/utils.rs | 12 ++++++ ml/transformers/tests/default.rs | 8 ++++ 19 files changed, 253 insertions(+), 18 deletions(-) create mode 100644 ml/transformers/Cargo.toml create mode 100644 ml/transformers/benches/default.rs create mode 100644 ml/transformers/src/attention/head.rs create mode 100644 ml/transformers/src/attention/mod.rs create mode 100644 ml/transformers/src/codec/decode.rs create mode 100644 ml/transformers/src/codec/encode.rs create mode 100644 ml/transformers/src/codec/mod.rs create mode 100644 ml/transformers/src/lib.rs create mode 100644 ml/transformers/src/primitives.rs create mode 100644 ml/transformers/src/specs.rs create mode 100644 ml/transformers/src/utils.rs create mode 100644 ml/transformers/tests/default.rs diff --git a/concision/Cargo.toml b/concision/Cargo.toml index ced26616..72130273 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -21,7 +21,7 @@ full = [ "core", "derive", "math", - "neural" + "ml", ] core = [ @@ -45,10 +45,19 @@ math = [ "concision-math" ] +ml = [ + "neural", + "transformers" +] + neural = [ "concision-neural" ] +transformers = [ + "concision-transformers" +] + [lib] bench = true crate-type = ["cdylib", "rlib"] @@ -64,6 +73,7 @@ concision-derive = { features = [], optional = true, path = "../derive", version concision-macros = { features = [], optional = true, path = "../macros", version = "0.1.12" } concision-math = { features = [], optional = true, path = "../math", version = "0.1.12" } concision-neural = { features = [], optional = true, path = "../ml/neural", version = "0.1.12" } +concision-transformers = { features = [], optional = true, path = "../ml/transformers", version = "0.1.12" } [dev-dependencies] diff --git a/concision/src/lib.rs b/concision/src/lib.rs index 63f1ef7f..11d3b21b 100644 --- a/concision/src/lib.rs +++ b/concision/src/lib.rs @@ -19,6 +19,8 @@ pub use concision_macros::*; pub use concision_math as math; #[cfg(feature = "nn")] pub use concision_nn as nn; +#[cfg(feature = "transformers")] +pub use concision_transformers as transformers; pub mod prelude { #[cfg(feature = "core")] @@ -33,4 +35,6 @@ pub mod prelude { pub use concision_math::prelude::*; #[cfg(feature = "nn")] pub use concision_nn::prelude::*; + #[cfg(feature = "transformers")] + pub use concision_transformers::prelude::*; } diff --git a/core/src/errors/error.rs b/core/src/errors/error.rs index 126e2ebc..a849d66a 100644 --- a/core/src/errors/error.rs +++ b/core/src/errors/error.rs @@ -137,4 +137,4 @@ impl From for Error { fn from(err: anyhow::Error) -> Self { Self::new(Errors::Unknown, err.to_string()) } -} \ No newline at end of file +} diff --git a/math/src/statistics/mod.rs b/math/src/statistics/mod.rs index 76818145..df4be1be 100644 --- a/math/src/statistics/mod.rs +++ b/math/src/statistics/mod.rs @@ -49,19 +49,9 @@ where } } -impl Statistics for Vec -where - T: num::Float + std::iter::Sum, - Self: Clone + IntoIterator, -{ -} +impl Statistics for Vec where T: num::Float + std::iter::Sum {} -impl Statistics for ndarray::Array1 -where - T: num::Float + std::iter::Sum, - Self: Clone + IntoIterator, -{ -} +impl Statistics for ndarray::Array1 where T: num::Float + std::iter::Sum {} pub(crate) mod utils { use std::iter::Sum; diff --git a/math/src/statistics/regression/linear.rs b/math/src/statistics/regression/linear.rs index fc78bd78..ec5f0818 100644 --- a/math/src/statistics/regression/linear.rs +++ b/math/src/statistics/regression/linear.rs @@ -44,7 +44,9 @@ impl Regression for LinearRegression { } fn predict(&self, args: &[Self::Item]) -> Vec { - args.iter().map(|&x| self.slope * x + self.intercept).collect() + args.iter() + .map(|&x| self.slope * x + self.intercept) + .collect() } } diff --git a/math/src/statistics/regression/mod.rs b/math/src/statistics/regression/mod.rs index af241f52..9566f194 100644 --- a/math/src/statistics/regression/mod.rs +++ b/math/src/statistics/regression/mod.rs @@ -16,5 +16,4 @@ pub trait Regression { fn predict(&self, args: &[Self::Item]) -> Vec; } - pub(crate) mod utils {} diff --git a/ml/neural/src/nn/loss/mod.rs b/ml/neural/src/nn/loss/mod.rs index a6a7066c..6262565b 100644 --- a/ml/neural/src/nn/loss/mod.rs +++ b/ml/neural/src/nn/loss/mod.rs @@ -18,8 +18,6 @@ pub trait Loss { fn loss(&self, pred: &[f64], target: &[f64]) -> f64; } - - pub(crate) mod utils { use ndarray::Array1; diff --git a/ml/transformers/Cargo.toml b/ml/transformers/Cargo.toml new file mode 100644 index 00000000..138e8529 --- /dev/null +++ b/ml/transformers/Cargo.toml @@ -0,0 +1,43 @@ +[package] +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "concision-transformers" +readme.workspace = true +repository.workspace = true +version.workspace = true + +[features] +default = [] + + +[lib] +bench = true +crate-type = ["cdylib", "rlib"] +doctest = false +test = true + +[build-dependencies] + +[dependencies] +anyhow.workspace = true +ndarray.workspace = true +num.workspace = true +serde.workspace = true +serde_json.workspace = true +smart-default.workspace = true +strum.workspace = true + +[dev-dependencies] + +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] + +[target.wasm32-unknown-unknown] + +[target.wasm32-wasi] diff --git a/ml/transformers/benches/default.rs b/ml/transformers/benches/default.rs new file mode 100644 index 00000000..937f2387 --- /dev/null +++ b/ml/transformers/benches/default.rs @@ -0,0 +1,52 @@ +// bench.rs +#![feature(test)] + +extern crate test; + +use std::mem::replace; +use test::Bencher; + +// bench: find the `BENCH_SIZE` first terms of the fibonacci sequence +static BENCH_SIZE: usize = 20; + +// recursive fibonacci +fn fibonacci(n: usize) -> u32 { + if n < 2 { + 1 + } else { + fibonacci(n - 1) + fibonacci(n - 2) + } +} + +// iterative fibonacci +struct Fibonacci { + curr: u32, + next: u32, +} + +impl Iterator for Fibonacci { + type Item = u32; + fn next(&mut self) -> Option { + let new_next = self.curr + self.next; + let new_curr = replace(&mut self.next, new_next); + + Some(replace(&mut self.curr, new_curr)) + } +} + +fn fibonacci_sequence() -> Fibonacci { + Fibonacci { curr: 1, next: 1 } +} + +// function to benchmark must be annotated with `#[bench]` +#[bench] +fn recursive_fibonacci(b: &mut Bencher) { + // exact code to benchmark must be passed as a closure to the iter + // method of Bencher + b.iter(|| (0..BENCH_SIZE).map(fibonacci).collect::>()) +} + +#[bench] +fn iterative_fibonacci(b: &mut Bencher) { + b.iter(|| fibonacci_sequence().take(BENCH_SIZE).collect::>()) +} diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs new file mode 100644 index 00000000..147cc341 --- /dev/null +++ b/ml/transformers/src/attention/head.rs @@ -0,0 +1,23 @@ +/* + Appellation: head + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "lowercase")] +pub struct AttentionHead { + pos: usize, +} + +impl AttentionHead { + pub fn new() -> Self { + Self { pos: 0 } + } +} + +impl std::fmt::Display for AttentionHead { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", serde_json::to_string(self).unwrap()) + } +} diff --git a/ml/transformers/src/attention/mod.rs b/ml/transformers/src/attention/mod.rs new file mode 100644 index 00000000..36efcfb3 --- /dev/null +++ b/ml/transformers/src/attention/mod.rs @@ -0,0 +1,18 @@ +/* + Appellation: attention + Contrib: FL03 +*/ +//! # Attention +pub use self::{head::*, utils::*}; + +pub(crate) mod head; + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests { + // use super::*; + + #[test] + fn test_attention() {} +} diff --git a/ml/transformers/src/codec/decode.rs b/ml/transformers/src/codec/decode.rs new file mode 100644 index 00000000..3224d978 --- /dev/null +++ b/ml/transformers/src/codec/decode.rs @@ -0,0 +1,6 @@ +/* + Appellation: encode + Contrib: FL03 +*/ + +pub struct Decode {} diff --git a/ml/transformers/src/codec/encode.rs b/ml/transformers/src/codec/encode.rs new file mode 100644 index 00000000..3d58906d --- /dev/null +++ b/ml/transformers/src/codec/encode.rs @@ -0,0 +1,6 @@ +/* + Appellation: encode + Contrib: FL03 +*/ + +pub struct Encode {} diff --git a/ml/transformers/src/codec/mod.rs b/ml/transformers/src/codec/mod.rs new file mode 100644 index 00000000..03c5b5d6 --- /dev/null +++ b/ml/transformers/src/codec/mod.rs @@ -0,0 +1,19 @@ +/* + Appellation: attention + Contrib: FL03 +*/ +//! # Attention +pub use self::{decode::*, encode::*, utils::*}; + +pub(crate) mod decode; +pub(crate) mod encode; + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests { + // use super::*; + + #[test] + fn test_codec() {} +} diff --git a/ml/transformers/src/lib.rs b/ml/transformers/src/lib.rs new file mode 100644 index 00000000..cac299cd --- /dev/null +++ b/ml/transformers/src/lib.rs @@ -0,0 +1,22 @@ +/* + Appellation: transformers + Contrib: FL03 +*/ +//! # Concision Transformers +pub use self::{primitives::*, specs::*, utils::*}; + +pub(crate) mod primitives; +pub(crate) mod specs; +pub(crate) mod utils; + +pub mod attention; +pub mod codec; + +pub mod prelude { + pub use crate::attention::*; + pub use crate::codec::*; + + pub use crate::primitives::*; + pub use crate::specs::*; + pub use crate::utils::*; +} diff --git a/ml/transformers/src/primitives.rs b/ml/transformers/src/primitives.rs new file mode 100644 index 00000000..3e1b7491 --- /dev/null +++ b/ml/transformers/src/primitives.rs @@ -0,0 +1,19 @@ +/* + Appellation: primitives + Contrib: FL03 +*/ +pub use self::{constants::*, statics::*, types::*}; + +/// Collection of constants used throughout the system +pub(crate) mod constants {} + +/// Collection of static references used throughout +pub(crate) mod statics {} + +/// Collection of types used throughout the system +pub(crate) mod types { + /// + pub type BoxError = Box; + /// + pub type BoxResult = std::result::Result; +} diff --git a/ml/transformers/src/specs.rs b/ml/transformers/src/specs.rs new file mode 100644 index 00000000..1d8faa71 --- /dev/null +++ b/ml/transformers/src/specs.rs @@ -0,0 +1,4 @@ +/* + Appellation: specs + Contrib: FL03 +*/ diff --git a/ml/transformers/src/utils.rs b/ml/transformers/src/utils.rs new file mode 100644 index 00000000..c57e8121 --- /dev/null +++ b/ml/transformers/src/utils.rs @@ -0,0 +1,12 @@ +/* + Appellation: utils + Contrib: FL03 + Description: ... Summary ... +*/ + +pub fn now() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() +} diff --git a/ml/transformers/tests/default.rs b/ml/transformers/tests/default.rs new file mode 100644 index 00000000..0cac1eb5 --- /dev/null +++ b/ml/transformers/tests/default.rs @@ -0,0 +1,8 @@ +#[cfg(test)] +#[test] +fn compiles() { + let f = |x: usize, y: usize| x + y; + + assert_eq!(f(10, 10), 20); + assert_ne!(f(1, 1), 3); +} From bb12805f3457143cd46a0e114d20b20dfaa777f8 Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 20 Oct 2023 12:27:04 -0500 Subject: [PATCH 021/118] update Signed-off-by: FL03 --- Cargo.toml | 2 +- concision/Cargo.toml | 2 +- concision/examples/basic.rs | 2 +- data/src/flows/flow.rs | 23 +++++++++ data/src/flows/mod.rs | 10 ++++ data/src/lib.rs | 2 + math/linalg/Cargo.toml | 44 +++++++++++++++++ math/linalg/src/lib.rs | 22 +++++++++ math/{ => linalg}/src/primitives.rs | 0 math/linalg/src/specs.rs | 4 ++ math/linalg/src/utils.rs | 4 ++ math/{src/linalg => linalg/src}/vs/mod.rs | 0 math/{src/linalg => linalg/src}/vs/space.rs | 0 math/{ => linalg}/tests/default.rs | 0 math/math/Cargo.toml | 48 +++++++++++++++++++ math/{ => math}/examples/simple.rs | 0 .../src/calculus/derivatives/mod.rs | 0 .../src/calculus/derivatives/nderive.rs | 0 math/{ => math}/src/calculus/mod.rs | 0 math/math/src/lib.rs | 33 +++++++++++++ math/math/src/primitives.rs | 14 ++++++ math/math/src/specs.rs | 4 ++ math/math/src/utils.rs | 4 ++ math/math/tests/default.rs | 8 ++++ math/{ => num}/Cargo.toml | 4 +- math/{ => num}/src/factorials.rs | 0 math/{ => num}/src/lib.rs | 7 +-- math/{ => num}/src/num/arithmetic.rs | 0 math/{ => num}/src/num/complex.rs | 0 math/{ => num}/src/num/mod.rs | 0 math/num/src/primitives.rs | 14 ++++++ math/{ => num}/src/specs.rs | 0 math/{ => num}/src/utils.rs | 0 math/num/tests/default.rs | 8 ++++ math/src/linalg/mod.rs | 13 ----- math/statistics/Cargo.toml | 45 +++++++++++++++++ .../src/calc}/deviation.rs | 3 +- .../statistics => statistics/src/calc}/mod.rs | 47 +----------------- math/statistics/src/lib.rs | 26 ++++++++++ math/statistics/src/primitives.rs | 14 ++++++ .../src}/regression/linear.rs | 2 +- .../src}/regression/mod.rs | 0 math/statistics/src/specs.rs | 48 +++++++++++++++++++ math/statistics/src/utils.rs | 4 ++ math/statistics/tests/default.rs | 8 ++++ ml/neural/src/lib.rs | 5 ++ ml/neural/src/prop/mod.rs | 15 ++++++ ml/transformers/src/attention/mod.rs | 2 + ml/transformers/src/codec/decode.rs | 6 --- ml/transformers/src/codec/decode/decoder.rs | 6 +++ ml/transformers/src/codec/decode/mod.rs | 22 +++++++++ ml/transformers/src/codec/encode.rs | 6 --- ml/transformers/src/codec/encode/encoder.rs | 6 +++ ml/transformers/src/codec/encode/mod.rs | 20 ++++++++ ml/transformers/src/codec/mod.rs | 12 +++-- ml/transformers/src/lib.rs | 2 + ml/transformers/src/primitives.rs | 11 ++--- ml/transformers/src/transform/mod.rs | 20 ++++++++ ml/transformers/src/transform/transformer.rs | 11 +++++ ml/transformers/src/utils.rs | 8 ---- 60 files changed, 518 insertions(+), 103 deletions(-) create mode 100644 data/src/flows/flow.rs create mode 100644 data/src/flows/mod.rs create mode 100644 math/linalg/Cargo.toml create mode 100644 math/linalg/src/lib.rs rename math/{ => linalg}/src/primitives.rs (100%) create mode 100644 math/linalg/src/specs.rs create mode 100644 math/linalg/src/utils.rs rename math/{src/linalg => linalg/src}/vs/mod.rs (100%) rename math/{src/linalg => linalg/src}/vs/space.rs (100%) rename math/{ => linalg}/tests/default.rs (100%) create mode 100644 math/math/Cargo.toml rename math/{ => math}/examples/simple.rs (100%) rename math/{ => math}/src/calculus/derivatives/mod.rs (100%) rename math/{ => math}/src/calculus/derivatives/nderive.rs (100%) rename math/{ => math}/src/calculus/mod.rs (100%) create mode 100644 math/math/src/lib.rs create mode 100644 math/math/src/primitives.rs create mode 100644 math/math/src/specs.rs create mode 100644 math/math/src/utils.rs create mode 100644 math/math/tests/default.rs rename math/{ => num}/Cargo.toml (96%) rename math/{ => num}/src/factorials.rs (100%) rename math/{ => num}/src/lib.rs (75%) rename math/{ => num}/src/num/arithmetic.rs (100%) rename math/{ => num}/src/num/complex.rs (100%) rename math/{ => num}/src/num/mod.rs (100%) create mode 100644 math/num/src/primitives.rs rename math/{ => num}/src/specs.rs (100%) rename math/{ => num}/src/utils.rs (100%) create mode 100644 math/num/tests/default.rs delete mode 100644 math/src/linalg/mod.rs create mode 100644 math/statistics/Cargo.toml rename math/{src/statistics => statistics/src/calc}/deviation.rs (95%) rename math/{src/statistics => statistics/src/calc}/mod.rs (62%) create mode 100644 math/statistics/src/lib.rs create mode 100644 math/statistics/src/primitives.rs rename math/{src/statistics => statistics/src}/regression/linear.rs (96%) rename math/{src/statistics => statistics/src}/regression/mod.rs (100%) create mode 100644 math/statistics/src/specs.rs create mode 100644 math/statistics/src/utils.rs create mode 100644 math/statistics/tests/default.rs create mode 100644 ml/neural/src/prop/mod.rs delete mode 100644 ml/transformers/src/codec/decode.rs create mode 100644 ml/transformers/src/codec/decode/decoder.rs create mode 100644 ml/transformers/src/codec/decode/mod.rs delete mode 100644 ml/transformers/src/codec/encode.rs create mode 100644 ml/transformers/src/codec/encode/encoder.rs create mode 100644 ml/transformers/src/codec/encode/mod.rs create mode 100644 ml/transformers/src/transform/mod.rs create mode 100644 ml/transformers/src/transform/transformer.rs diff --git a/Cargo.toml b/Cargo.toml index 932d4942..7d241638 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ members = [ "data", "derive", "macros", - "math", + "math/*", "ml/*", ] diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 72130273..6cabb535 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -71,7 +71,7 @@ concision-core = { features = [], optional = true, path = "../core", version = " concision-data = { features = [], optional = true, path = "../data", version = "0.1.12" } concision-derive = { features = [], optional = true, path = "../derive", version = "0.1.12" } concision-macros = { features = [], optional = true, path = "../macros", version = "0.1.12" } -concision-math = { features = [], optional = true, path = "../math", version = "0.1.12" } +concision-math = { features = [], optional = true, path = "../math/math", version = "0.1.12" } concision-neural = { features = [], optional = true, path = "../ml/neural", version = "0.1.12" } concision-transformers = { features = [], optional = true, path = "../ml/transformers", version = "0.1.12" } diff --git a/concision/examples/basic.rs b/concision/examples/basic.rs index 2da79037..af3ae1d4 100644 --- a/concision/examples/basic.rs +++ b/concision/examples/basic.rs @@ -1,6 +1,6 @@ extern crate concision; -use concision::math::num::complex::C; +use concision::math::prelude::complex::C; use concision::prelude::Numerical; // Define a holomorphic function that squares its input. diff --git a/data/src/flows/flow.rs b/data/src/flows/flow.rs new file mode 100644 index 00000000..07f5229b --- /dev/null +++ b/data/src/flows/flow.rs @@ -0,0 +1,23 @@ +/* + Appellation: flow + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "lowercase")] +pub struct Flow { + data: Vec, +} + +impl Flow { + pub fn new() -> Self { + Self { data: Vec::new() } + } +} + +impl std::fmt::Display for Flow { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", serde_json::to_string(self).unwrap()) + } +} diff --git a/data/src/flows/mod.rs b/data/src/flows/mod.rs new file mode 100644 index 00000000..9d06cd2c --- /dev/null +++ b/data/src/flows/mod.rs @@ -0,0 +1,10 @@ +/* + Appellation: flows + Contrib: FL03 +*/ +//! # Flows +pub use self::{flow::*, utils::*}; + +pub(crate) mod flow; + +pub(crate) mod utils {} diff --git a/data/src/lib.rs b/data/src/lib.rs index 9a87d65c..d8ca8940 100644 --- a/data/src/lib.rs +++ b/data/src/lib.rs @@ -10,9 +10,11 @@ pub(crate) mod specs; pub(crate) mod utils; pub mod df; +pub mod flows; pub mod prelude { pub use crate::df::*; + pub use crate::flows::*; pub use crate::primitives::*; pub use crate::specs::*; diff --git a/math/linalg/Cargo.toml b/math/linalg/Cargo.toml new file mode 100644 index 00000000..d960eec6 --- /dev/null +++ b/math/linalg/Cargo.toml @@ -0,0 +1,44 @@ +[package] +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "concision-linalg" +readme.workspace = true +repository.workspace = true +version.workspace = true + +[features] +default = [] + +[lib] +bench = true +crate-type = ["cdylib", "rlib"] +doctest = false +test = true + +[build-dependencies] + +[dependencies] +concision-core.workspace = true + +ndarray.workspace = true +num.workspace = true +serde.workspace = true +serde_json.workspace = true +smart-default.workspace = true +strum.workspace = true + +[dev-dependencies] +rand = "0.8" + +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] + +[target.wasm32-unknown-unknown] + +[target.wasm32-wasi] \ No newline at end of file diff --git a/math/linalg/src/lib.rs b/math/linalg/src/lib.rs new file mode 100644 index 00000000..2f483ed9 --- /dev/null +++ b/math/linalg/src/lib.rs @@ -0,0 +1,22 @@ +/* + Appellation: concision-linalg + Contrib: FL03 +*/ +//! # concision-linalg + +pub use self::{primitives::*, specs::*, utils::*}; + +pub mod vs; + +// pub(crate) use concision_core as core; + +pub(crate) mod primitives; +pub(crate) mod specs; +pub(crate) mod utils; + +pub mod prelude { + pub use crate::primitives::*; + pub use crate::specs::*; + pub use crate::utils::*; + pub use crate::vs::*; +} diff --git a/math/src/primitives.rs b/math/linalg/src/primitives.rs similarity index 100% rename from math/src/primitives.rs rename to math/linalg/src/primitives.rs diff --git a/math/linalg/src/specs.rs b/math/linalg/src/specs.rs new file mode 100644 index 00000000..8abf79cb --- /dev/null +++ b/math/linalg/src/specs.rs @@ -0,0 +1,4 @@ +/* + Appellation: specs + Contrib: FL03 +*/ diff --git a/math/linalg/src/utils.rs b/math/linalg/src/utils.rs new file mode 100644 index 00000000..f8840c69 --- /dev/null +++ b/math/linalg/src/utils.rs @@ -0,0 +1,4 @@ +/* + Appellation: utils + Contrib: FL03 +*/ diff --git a/math/src/linalg/vs/mod.rs b/math/linalg/src/vs/mod.rs similarity index 100% rename from math/src/linalg/vs/mod.rs rename to math/linalg/src/vs/mod.rs diff --git a/math/src/linalg/vs/space.rs b/math/linalg/src/vs/space.rs similarity index 100% rename from math/src/linalg/vs/space.rs rename to math/linalg/src/vs/space.rs diff --git a/math/tests/default.rs b/math/linalg/tests/default.rs similarity index 100% rename from math/tests/default.rs rename to math/linalg/tests/default.rs diff --git a/math/math/Cargo.toml b/math/math/Cargo.toml new file mode 100644 index 00000000..94e5dd11 --- /dev/null +++ b/math/math/Cargo.toml @@ -0,0 +1,48 @@ +[package] +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "concision-math" +readme.workspace = true +repository.workspace = true +version.workspace = true + +[features] +default = [] + +[lib] +bench = true +crate-type = ["cdylib", "rlib"] +doctest = false +test = true + +[build-dependencies] + +[dependencies] +concision-linalg = { features = [], path = "../linalg", version = "0.1.12" } +concision-num = { features = [], path = "../num", version = "0.1.12" } +concision-statistics = { features = [], path = "../statistics", version = "0.1.12" } + +concision-core.workspace = true + +ndarray.workspace = true +num.workspace = true +serde.workspace = true +serde_json.workspace = true +smart-default.workspace = true +strum.workspace = true + +[dev-dependencies] +rand = "0.8" + +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] + +[target.wasm32-unknown-unknown] + +[target.wasm32-wasi] \ No newline at end of file diff --git a/math/examples/simple.rs b/math/math/examples/simple.rs similarity index 100% rename from math/examples/simple.rs rename to math/math/examples/simple.rs diff --git a/math/src/calculus/derivatives/mod.rs b/math/math/src/calculus/derivatives/mod.rs similarity index 100% rename from math/src/calculus/derivatives/mod.rs rename to math/math/src/calculus/derivatives/mod.rs diff --git a/math/src/calculus/derivatives/nderive.rs b/math/math/src/calculus/derivatives/nderive.rs similarity index 100% rename from math/src/calculus/derivatives/nderive.rs rename to math/math/src/calculus/derivatives/nderive.rs diff --git a/math/src/calculus/mod.rs b/math/math/src/calculus/mod.rs similarity index 100% rename from math/src/calculus/mod.rs rename to math/math/src/calculus/mod.rs diff --git a/math/math/src/lib.rs b/math/math/src/lib.rs new file mode 100644 index 00000000..3f7db8ba --- /dev/null +++ b/math/math/src/lib.rs @@ -0,0 +1,33 @@ +/* + Appellation: concision-math + Contrib: FL03 +*/ +//! # concision-math + +pub use self::{primitives::*, specs::*, utils::*}; + +pub mod calculus; + +pub use concision_linalg as linalg; + +pub use concision_num as num; + +pub use concision_statistics as stats; + +// pub(crate) use concision_core as core; + +pub(crate) mod primitives; +pub(crate) mod specs; +pub(crate) mod utils; + +pub mod prelude { + + pub use concision_linalg::prelude::*; + pub use concision_num::prelude::*; + pub use concision_statistics::prelude::*; + + pub use crate::calculus::*; + pub use crate::primitives::*; + pub use crate::specs::*; + pub use crate::utils::*; +} diff --git a/math/math/src/primitives.rs b/math/math/src/primitives.rs new file mode 100644 index 00000000..f16ac25f --- /dev/null +++ b/math/math/src/primitives.rs @@ -0,0 +1,14 @@ +/* + Appellation: primitives + Contrib: FL03 +*/ +pub use self::{constants::*, statics::*, types::*}; + +/// Collection of constants used throughout the system +mod constants {} + +/// Collection of static references used throughout +mod statics {} + +/// Collection of types used throughout the system +mod types {} diff --git a/math/math/src/specs.rs b/math/math/src/specs.rs new file mode 100644 index 00000000..8abf79cb --- /dev/null +++ b/math/math/src/specs.rs @@ -0,0 +1,4 @@ +/* + Appellation: specs + Contrib: FL03 +*/ diff --git a/math/math/src/utils.rs b/math/math/src/utils.rs new file mode 100644 index 00000000..f8840c69 --- /dev/null +++ b/math/math/src/utils.rs @@ -0,0 +1,4 @@ +/* + Appellation: utils + Contrib: FL03 +*/ diff --git a/math/math/tests/default.rs b/math/math/tests/default.rs new file mode 100644 index 00000000..0cac1eb5 --- /dev/null +++ b/math/math/tests/default.rs @@ -0,0 +1,8 @@ +#[cfg(test)] +#[test] +fn compiles() { + let f = |x: usize, y: usize| x + y; + + assert_eq!(f(10, 10), 20); + assert_ne!(f(1, 1), 3); +} diff --git a/math/Cargo.toml b/math/num/Cargo.toml similarity index 96% rename from math/Cargo.toml rename to math/num/Cargo.toml index 3990bb49..152a7b8a 100644 --- a/math/Cargo.toml +++ b/math/num/Cargo.toml @@ -6,7 +6,7 @@ edition.workspace = true homepage.workspace = true keywords.workspace = true license.workspace = true -name = "concision-math" +name = "concision-num" readme.workspace = true repository.workspace = true version.workspace = true @@ -23,6 +23,8 @@ test = true [build-dependencies] [dependencies] + + concision-core.workspace = true ndarray.workspace = true diff --git a/math/src/factorials.rs b/math/num/src/factorials.rs similarity index 100% rename from math/src/factorials.rs rename to math/num/src/factorials.rs diff --git a/math/src/lib.rs b/math/num/src/lib.rs similarity index 75% rename from math/src/lib.rs rename to math/num/src/lib.rs index 8d1dd740..6cab992d 100644 --- a/math/src/lib.rs +++ b/math/num/src/lib.rs @@ -6,10 +6,7 @@ pub use self::{factorials::*, primitives::*, specs::*, utils::*}; -pub mod calculus; -pub mod linalg; pub mod num; -pub mod statistics; pub(crate) mod factorials; @@ -20,11 +17,9 @@ pub(crate) mod specs; pub(crate) mod utils; pub mod prelude { - pub use crate::calculus::*; - pub use crate::linalg::*; + pub use crate::factorials::*; pub use crate::num::*; pub use crate::primitives::*; pub use crate::specs::*; - pub use crate::statistics::*; pub use crate::utils::*; } diff --git a/math/src/num/arithmetic.rs b/math/num/src/num/arithmetic.rs similarity index 100% rename from math/src/num/arithmetic.rs rename to math/num/src/num/arithmetic.rs diff --git a/math/src/num/complex.rs b/math/num/src/num/complex.rs similarity index 100% rename from math/src/num/complex.rs rename to math/num/src/num/complex.rs diff --git a/math/src/num/mod.rs b/math/num/src/num/mod.rs similarity index 100% rename from math/src/num/mod.rs rename to math/num/src/num/mod.rs diff --git a/math/num/src/primitives.rs b/math/num/src/primitives.rs new file mode 100644 index 00000000..f16ac25f --- /dev/null +++ b/math/num/src/primitives.rs @@ -0,0 +1,14 @@ +/* + Appellation: primitives + Contrib: FL03 +*/ +pub use self::{constants::*, statics::*, types::*}; + +/// Collection of constants used throughout the system +mod constants {} + +/// Collection of static references used throughout +mod statics {} + +/// Collection of types used throughout the system +mod types {} diff --git a/math/src/specs.rs b/math/num/src/specs.rs similarity index 100% rename from math/src/specs.rs rename to math/num/src/specs.rs diff --git a/math/src/utils.rs b/math/num/src/utils.rs similarity index 100% rename from math/src/utils.rs rename to math/num/src/utils.rs diff --git a/math/num/tests/default.rs b/math/num/tests/default.rs new file mode 100644 index 00000000..0cac1eb5 --- /dev/null +++ b/math/num/tests/default.rs @@ -0,0 +1,8 @@ +#[cfg(test)] +#[test] +fn compiles() { + let f = |x: usize, y: usize| x + y; + + assert_eq!(f(10, 10), 20); + assert_ne!(f(1, 1), 3); +} diff --git a/math/src/linalg/mod.rs b/math/src/linalg/mod.rs deleted file mode 100644 index 1b4dc1a0..00000000 --- a/math/src/linalg/mod.rs +++ /dev/null @@ -1,13 +0,0 @@ -/* - Appellation: linalg - Contrib: FL03 -*/ -//! # linalg -//! -//! This module implements the linear algebra functions and logic required to build -//! efficient neural networks. -pub use self::utils::*; - -pub mod vs; - -pub(crate) mod utils {} diff --git a/math/statistics/Cargo.toml b/math/statistics/Cargo.toml new file mode 100644 index 00000000..00ecabf3 --- /dev/null +++ b/math/statistics/Cargo.toml @@ -0,0 +1,45 @@ +[package] +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "concision-statistics" +readme.workspace = true +repository.workspace = true +version.workspace = true + +[features] +default = [] + +[lib] +bench = true +crate-type = ["cdylib", "rlib"] +doctest = false +test = true + +[build-dependencies] + +[dependencies] +concision-core.workspace = true + +ndarray.workspace = true +num.workspace = true +serde.workspace = true +serde_json.workspace = true +smart-default.workspace = true +strum.workspace = true + +[dev-dependencies] +concision-num = { features = [], path = "../num", version = "0.1.12" } +rand = "0.8" + +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] + +[target.wasm32-unknown-unknown] + +[target.wasm32-wasi] \ No newline at end of file diff --git a/math/src/statistics/deviation.rs b/math/statistics/src/calc/deviation.rs similarity index 95% rename from math/src/statistics/deviation.rs rename to math/statistics/src/calc/deviation.rs index a38fb9f0..e19fcd15 100644 --- a/math/src/statistics/deviation.rs +++ b/math/statistics/src/calc/deviation.rs @@ -51,7 +51,8 @@ impl StandardDeviation { mod tests { use super::*; - use crate::prelude::{RoundTo, Statistics}; + use crate::prelude::Statistics; + use concision_num::RoundTo; #[test] fn test_std() { diff --git a/math/src/statistics/mod.rs b/math/statistics/src/calc/mod.rs similarity index 62% rename from math/src/statistics/mod.rs rename to math/statistics/src/calc/mod.rs index df4be1be..4023625f 100644 --- a/math/src/statistics/mod.rs +++ b/math/statistics/src/calc/mod.rs @@ -5,54 +5,8 @@ //! # statistics pub use self::{deviation::*, utils::*}; -pub mod regression; - pub(crate) mod deviation; -pub trait Statistics -where - T: num::Float + std::iter::Sum, - Self: Clone + IntoIterator, -{ - fn covariance(&self, other: &Self) -> T { - let dx = self.deviation(); - let dy = other.deviation(); - dx.iter().zip(dy.iter()).map(|(&x, &y)| x * y).sum::() / T::from(dx.len()).unwrap() - } - - fn deviation(&self) -> Vec { - let mean = self.mean(); - self.clone().into_iter().map(|x| x - mean).collect() - } - fn len(&self) -> usize { - Vec::from_iter(self.clone().into_iter()).len() - } - /// [Statistics::mean] calculates the mean or average of the data - fn mean(&self) -> T { - self.clone().into_iter().sum::() / T::from(self.len()).unwrap() - } - /// [Statistics::std] calculates the standard deviation of the data - fn std(&self) -> T { - let mean = self.mean(); - let mut res = self - .clone() - .into_iter() - .map(|x| (x - mean).powi(2)) - .sum::(); - res = res / T::from(self.len()).unwrap(); - res.sqrt() - } - - fn variance(&self) -> T { - let dev = self.deviation(); - dev.iter().map(|&x| x * x).sum::() / T::from(dev.len()).unwrap() - } -} - -impl Statistics for Vec where T: num::Float + std::iter::Sum {} - -impl Statistics for ndarray::Array1 where T: num::Float + std::iter::Sum {} - pub(crate) mod utils { use std::iter::Sum; @@ -81,6 +35,7 @@ pub(crate) mod utils { #[cfg(test)] mod tests { use super::*; + use crate::Statistics; use rand::Rng; fn random_vec() -> Vec { diff --git a/math/statistics/src/lib.rs b/math/statistics/src/lib.rs new file mode 100644 index 00000000..b751fda1 --- /dev/null +++ b/math/statistics/src/lib.rs @@ -0,0 +1,26 @@ +/* + Appellation: concision-statistics + Contrib: FL03 +*/ +//! # concision-statistics + +pub use self::{primitives::*, specs::*, utils::*}; + +pub mod calc; +pub mod regression; + +// pub(crate) use concision_num as num; + +pub(crate) mod primitives; +pub(crate) mod specs; +pub(crate) mod utils; + +pub mod prelude { + + pub use crate::calc::*; + pub use crate::regression::*; + + pub use crate::primitives::*; + pub use crate::specs::*; + pub use crate::utils::*; +} diff --git a/math/statistics/src/primitives.rs b/math/statistics/src/primitives.rs new file mode 100644 index 00000000..f16ac25f --- /dev/null +++ b/math/statistics/src/primitives.rs @@ -0,0 +1,14 @@ +/* + Appellation: primitives + Contrib: FL03 +*/ +pub use self::{constants::*, statics::*, types::*}; + +/// Collection of constants used throughout the system +mod constants {} + +/// Collection of static references used throughout +mod statics {} + +/// Collection of types used throughout the system +mod types {} diff --git a/math/src/statistics/regression/linear.rs b/math/statistics/src/regression/linear.rs similarity index 96% rename from math/src/statistics/regression/linear.rs rename to math/statistics/src/regression/linear.rs index ec5f0818..10c0612f 100644 --- a/math/src/statistics/regression/linear.rs +++ b/math/statistics/src/regression/linear.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use super::Regression; -use crate::statistics::{covariance, deviation, mean, variance}; +use crate::calc::{covariance, deviation, mean, variance}; #[derive(Clone, Copy, Debug, Default, PartialEq, PartialOrd)] pub struct LinearRegression { diff --git a/math/src/statistics/regression/mod.rs b/math/statistics/src/regression/mod.rs similarity index 100% rename from math/src/statistics/regression/mod.rs rename to math/statistics/src/regression/mod.rs diff --git a/math/statistics/src/specs.rs b/math/statistics/src/specs.rs new file mode 100644 index 00000000..d178f942 --- /dev/null +++ b/math/statistics/src/specs.rs @@ -0,0 +1,48 @@ +/* + Appellation: specs + Contrib: FL03 +*/ + +pub trait Statistics +where + T: num::Float + std::iter::Sum, + Self: Clone + IntoIterator, +{ + fn covariance(&self, other: &Self) -> T { + let dx = self.deviation(); + let dy = other.deviation(); + dx.iter().zip(dy.iter()).map(|(&x, &y)| x * y).sum::() / T::from(dx.len()).unwrap() + } + + fn deviation(&self) -> Vec { + let mean = self.mean(); + self.clone().into_iter().map(|x| x - mean).collect() + } + fn len(&self) -> usize { + Vec::from_iter(self.clone().into_iter()).len() + } + /// [Statistics::mean] calculates the mean or average of the data + fn mean(&self) -> T { + self.clone().into_iter().sum::() / T::from(self.len()).unwrap() + } + /// [Statistics::std] calculates the standard deviation of the data + fn std(&self) -> T { + let mean = self.mean(); + let mut res = self + .clone() + .into_iter() + .map(|x| (x - mean).powi(2)) + .sum::(); + res = res / T::from(self.len()).unwrap(); + res.sqrt() + } + + fn variance(&self) -> T { + let dev = self.deviation(); + dev.iter().map(|&x| x * x).sum::() / T::from(dev.len()).unwrap() + } +} + +impl Statistics for Vec where T: num::Float + std::iter::Sum {} + +impl Statistics for ndarray::Array1 where T: num::Float + std::iter::Sum {} diff --git a/math/statistics/src/utils.rs b/math/statistics/src/utils.rs new file mode 100644 index 00000000..f8840c69 --- /dev/null +++ b/math/statistics/src/utils.rs @@ -0,0 +1,4 @@ +/* + Appellation: utils + Contrib: FL03 +*/ diff --git a/math/statistics/tests/default.rs b/math/statistics/tests/default.rs new file mode 100644 index 00000000..0cac1eb5 --- /dev/null +++ b/math/statistics/tests/default.rs @@ -0,0 +1,8 @@ +#[cfg(test)] +#[test] +fn compiles() { + let f = |x: usize, y: usize| x + y; + + assert_eq!(f(10, 10), 20); + assert_ne!(f(1, 1), 3); +} diff --git a/ml/neural/src/lib.rs b/ml/neural/src/lib.rs index 34cd613c..84c3f9db 100644 --- a/ml/neural/src/lib.rs +++ b/ml/neural/src/lib.rs @@ -12,11 +12,16 @@ pub(crate) mod utils; pub mod layers; pub mod neurons; pub mod nn; +pub mod prop; // pub(crate) use concision_core as core; pub mod prelude { + pub use crate::layers::*; pub use crate::neurons::*; + pub use crate::nn::*; + pub use crate::prop::*; + pub use crate::primitives::*; pub use crate::specs::*; pub use crate::utils::*; diff --git a/ml/neural/src/prop/mod.rs b/ml/neural/src/prop/mod.rs new file mode 100644 index 00000000..92a3afd0 --- /dev/null +++ b/ml/neural/src/prop/mod.rs @@ -0,0 +1,15 @@ +/* + Appellation: prop + Contrib: FL03 +*/ +//! # Propagation +//! +//! This module describes the propagation of data through a neural network. +pub use self::utils::*; + + + +pub(crate) mod utils { + + +} diff --git a/ml/transformers/src/attention/mod.rs b/ml/transformers/src/attention/mod.rs index 36efcfb3..ca64ebc3 100644 --- a/ml/transformers/src/attention/mod.rs +++ b/ml/transformers/src/attention/mod.rs @@ -7,6 +7,8 @@ pub use self::{head::*, utils::*}; pub(crate) mod head; +pub trait Attention {} + pub(crate) mod utils {} #[cfg(test)] diff --git a/ml/transformers/src/codec/decode.rs b/ml/transformers/src/codec/decode.rs deleted file mode 100644 index 3224d978..00000000 --- a/ml/transformers/src/codec/decode.rs +++ /dev/null @@ -1,6 +0,0 @@ -/* - Appellation: encode - Contrib: FL03 -*/ - -pub struct Decode {} diff --git a/ml/transformers/src/codec/decode/decoder.rs b/ml/transformers/src/codec/decode/decoder.rs new file mode 100644 index 00000000..1c157163 --- /dev/null +++ b/ml/transformers/src/codec/decode/decoder.rs @@ -0,0 +1,6 @@ +/* + Appellation: decoder + Contrib: FL03 +*/ + +pub struct Decoder {} diff --git a/ml/transformers/src/codec/decode/mod.rs b/ml/transformers/src/codec/decode/mod.rs new file mode 100644 index 00000000..ac1d07cf --- /dev/null +++ b/ml/transformers/src/codec/decode/mod.rs @@ -0,0 +1,22 @@ +/* + Appellation: decode + Contrib: FL03 +*/ +//! # Decode +pub use self::{decoder::*, utils::*}; + +pub(crate) mod decoder; + +pub trait Decode { + +} + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests { + // use super::*; + + #[test] + fn test_decoder() {} +} diff --git a/ml/transformers/src/codec/encode.rs b/ml/transformers/src/codec/encode.rs deleted file mode 100644 index 3d58906d..00000000 --- a/ml/transformers/src/codec/encode.rs +++ /dev/null @@ -1,6 +0,0 @@ -/* - Appellation: encode - Contrib: FL03 -*/ - -pub struct Encode {} diff --git a/ml/transformers/src/codec/encode/encoder.rs b/ml/transformers/src/codec/encode/encoder.rs new file mode 100644 index 00000000..1270bf6b --- /dev/null +++ b/ml/transformers/src/codec/encode/encoder.rs @@ -0,0 +1,6 @@ +/* + Appellation: encoder + Contrib: FL03 +*/ + +pub struct Encoder {} diff --git a/ml/transformers/src/codec/encode/mod.rs b/ml/transformers/src/codec/encode/mod.rs new file mode 100644 index 00000000..f251ec9e --- /dev/null +++ b/ml/transformers/src/codec/encode/mod.rs @@ -0,0 +1,20 @@ +/* + Appellation: encode + Contrib: FL03 +*/ +//! # Encode +pub use self::{encoder::*, utils::*}; + +pub(crate) mod encoder; + +pub trait Encode {} + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests { + // use super::*; + + #[test] + fn test_encoder() {} +} diff --git a/ml/transformers/src/codec/mod.rs b/ml/transformers/src/codec/mod.rs index 03c5b5d6..38f5114a 100644 --- a/ml/transformers/src/codec/mod.rs +++ b/ml/transformers/src/codec/mod.rs @@ -1,12 +1,14 @@ /* - Appellation: attention + Appellation: codec Contrib: FL03 */ -//! # Attention -pub use self::{decode::*, encode::*, utils::*}; +//! # Codec +pub use self::utils::*; -pub(crate) mod decode; -pub(crate) mod encode; +pub mod decode; +pub mod encode; + +pub trait Codec {} pub(crate) mod utils {} diff --git a/ml/transformers/src/lib.rs b/ml/transformers/src/lib.rs index cac299cd..0d0c79a6 100644 --- a/ml/transformers/src/lib.rs +++ b/ml/transformers/src/lib.rs @@ -11,10 +11,12 @@ pub(crate) mod utils; pub mod attention; pub mod codec; +pub mod transform; pub mod prelude { pub use crate::attention::*; pub use crate::codec::*; + pub use crate::transform::*; pub use crate::primitives::*; pub use crate::specs::*; diff --git a/ml/transformers/src/primitives.rs b/ml/transformers/src/primitives.rs index 3e1b7491..657a296c 100644 --- a/ml/transformers/src/primitives.rs +++ b/ml/transformers/src/primitives.rs @@ -5,15 +5,12 @@ pub use self::{constants::*, statics::*, types::*}; /// Collection of constants used throughout the system -pub(crate) mod constants {} +pub(crate) mod constants { + pub const DEFAULT_EMBEDDING_SIZE: usize = 512; +} /// Collection of static references used throughout pub(crate) mod statics {} /// Collection of types used throughout the system -pub(crate) mod types { - /// - pub type BoxError = Box; - /// - pub type BoxResult = std::result::Result; -} +pub(crate) mod types {} diff --git a/ml/transformers/src/transform/mod.rs b/ml/transformers/src/transform/mod.rs new file mode 100644 index 00000000..44c67180 --- /dev/null +++ b/ml/transformers/src/transform/mod.rs @@ -0,0 +1,20 @@ +/* + Appellation: transform + Contrib: FL03 +*/ +//! # Transform +pub use self::{transformer::*, utils::*}; + +pub(crate) mod transformer; + +pub trait Transform {} + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests { + // use super::*; + + #[test] + fn test_transformer() {} +} diff --git a/ml/transformers/src/transform/transformer.rs b/ml/transformers/src/transform/transformer.rs new file mode 100644 index 00000000..a15b51e4 --- /dev/null +++ b/ml/transformers/src/transform/transformer.rs @@ -0,0 +1,11 @@ +/* + Appellation: transformer + Contrib: FL03 +*/ +use super::Transform; +use crate::codec::Codec; + +#[derive(Clone, Debug, Default)] +pub struct Transformer; + +impl Transform for Transformer {} diff --git a/ml/transformers/src/utils.rs b/ml/transformers/src/utils.rs index c57e8121..752dabaf 100644 --- a/ml/transformers/src/utils.rs +++ b/ml/transformers/src/utils.rs @@ -1,12 +1,4 @@ /* Appellation: utils Contrib: FL03 - Description: ... Summary ... */ - -pub fn now() -> u128 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() -} From f0da7cce65ce3bde0779128692869a01506f6a9d Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 20 Oct 2023 13:38:05 -0500 Subject: [PATCH 022/118] update Signed-off-by: FL03 --- data/src/primitives.rs | 14 +++-------- data/src/utils.rs | 8 ------ ml/neural/Cargo.toml | 1 + ml/neural/src/neurons/activate/mod.rs | 4 ++- ml/neural/src/neurons/activate/softmax.rs | 30 +++++++++++++++++++++++ ml/neural/src/prop/mod.rs | 7 +----- ml/transformers/src/attention/head.rs | 12 +++++++-- ml/transformers/src/codec/decode/mod.rs | 4 +-- 8 files changed, 49 insertions(+), 31 deletions(-) create mode 100644 ml/neural/src/neurons/activate/softmax.rs diff --git a/data/src/primitives.rs b/data/src/primitives.rs index 3e1b7491..859023bb 100644 --- a/data/src/primitives.rs +++ b/data/src/primitives.rs @@ -4,16 +4,8 @@ */ pub use self::{constants::*, statics::*, types::*}; -/// Collection of constants used throughout the system -pub(crate) mod constants {} +mod constants {} -/// Collection of static references used throughout -pub(crate) mod statics {} +mod statics {} -/// Collection of types used throughout the system -pub(crate) mod types { - /// - pub type BoxError = Box; - /// - pub type BoxResult = std::result::Result; -} +mod types {} diff --git a/data/src/utils.rs b/data/src/utils.rs index c57e8121..752dabaf 100644 --- a/data/src/utils.rs +++ b/data/src/utils.rs @@ -1,12 +1,4 @@ /* Appellation: utils Contrib: FL03 - Description: ... Summary ... */ - -pub fn now() -> u128 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() -} diff --git a/ml/neural/Cargo.toml b/ml/neural/Cargo.toml index ae0b6aef..a0707028 100644 --- a/ml/neural/Cargo.toml +++ b/ml/neural/Cargo.toml @@ -35,6 +35,7 @@ smart-default.workspace = true strum.workspace = true [dev-dependencies] +concision-math = { features = [], path = "../../math/math", version = "0.1.12" } [package.metadata.docs.rs] all-features = true diff --git a/ml/neural/src/neurons/activate/mod.rs b/ml/neural/src/neurons/activate/mod.rs index 0e5ec803..c212589c 100644 --- a/ml/neural/src/neurons/activate/mod.rs +++ b/ml/neural/src/neurons/activate/mod.rs @@ -5,7 +5,9 @@ //! # activate //! //! This module contains the activation functions for the neurons. -pub use self::utils::*; +pub use self::{softmax::*, utils::*}; + +pub(crate) mod softmax; pub type ActivationFn = fn(T) -> T; diff --git a/ml/neural/src/neurons/activate/softmax.rs b/ml/neural/src/neurons/activate/softmax.rs new file mode 100644 index 00000000..22b2adff --- /dev/null +++ b/ml/neural/src/neurons/activate/softmax.rs @@ -0,0 +1,30 @@ +/* + Appellation: softmax + Contrib: FL03 +*/ +use ndarray::prelude::Array1; + +pub fn softmax(args: Array1) -> Array1 +where + T: num::Float, +{ + let mut res = Array1::zeros(args.len()); + let denom = args.mapv(|x| x.exp()).sum(); + for (i, x) in args.iter().enumerate() { + res[i] = x.exp() / denom; + } + res +} + +#[cfg(test)] +mod tests { + use super::*; + use concision_math::prelude::RoundTo; + + #[test] + fn test_softmax() { + let args = Array1::from(vec![1.0, 2.0, 3.0]); + let res = softmax(args).mapv(|i| i.round_to(8)); + assert_eq!(res, Array1::from(vec![0.09003057, 0.24472847, 0.66524096])); + } +} diff --git a/ml/neural/src/prop/mod.rs b/ml/neural/src/prop/mod.rs index 92a3afd0..29340655 100644 --- a/ml/neural/src/prop/mod.rs +++ b/ml/neural/src/prop/mod.rs @@ -7,9 +7,4 @@ //! This module describes the propagation of data through a neural network. pub use self::utils::*; - - -pub(crate) mod utils { - - -} +pub(crate) mod utils {} diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index 147cc341..ca707eec 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -4,15 +4,23 @@ */ use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, PartialEq, PartialOrd, Serialize)] #[serde(rename_all = "lowercase")] pub struct AttentionHead { + keys: Vec, + queries: Vec, + values: Vec, pos: usize, } impl AttentionHead { pub fn new() -> Self { - Self { pos: 0 } + Self { + keys: Vec::new(), + queries: Vec::new(), + values: Vec::new(), + pos: 0, + } } } diff --git a/ml/transformers/src/codec/decode/mod.rs b/ml/transformers/src/codec/decode/mod.rs index ac1d07cf..fcc6d4bb 100644 --- a/ml/transformers/src/codec/decode/mod.rs +++ b/ml/transformers/src/codec/decode/mod.rs @@ -7,9 +7,7 @@ pub use self::{decoder::*, utils::*}; pub(crate) mod decoder; -pub trait Decode { - -} +pub trait Decode {} pub(crate) mod utils {} From 8922fd264523b99d3d3a1218b5a73b2a7b853675 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 21 Oct 2023 08:03:03 -0500 Subject: [PATCH 023/118] update Signed-off-by: FL03 --- .github/dependabot.yml | 4 +- .github/workflows/publish.yml | 2 +- Cargo.toml | 3 +- concision/Cargo.toml | 12 +- concision/src/lib.rs | 4 - math/linalg/Cargo.toml | 44 ----- math/linalg/src/lib.rs | 22 --- math/linalg/src/primitives.rs | 14 -- math/linalg/src/specs.rs | 4 - math/linalg/src/utils.rs | 4 - math/linalg/src/vs/mod.rs | 37 ---- math/linalg/src/vs/space.rs | 26 --- math/linalg/tests/default.rs | 8 - math/math/Cargo.toml | 48 ----- math/math/examples/simple.rs | 16 -- math/math/src/calculus/derivatives/mod.rs | 8 - math/math/src/calculus/derivatives/nderive.rs | 21 --- math/math/src/calculus/mod.rs | 7 - math/math/src/lib.rs | 33 ---- math/math/src/primitives.rs | 14 -- math/math/src/specs.rs | 4 - math/math/src/utils.rs | 4 - math/math/tests/default.rs | 8 - math/num/Cargo.toml | 46 ----- math/num/src/factorials.rs | 54 ------ math/num/src/lib.rs | 25 --- math/num/src/num/arithmetic.rs | 21 --- math/num/src/num/complex.rs | 167 ------------------ math/num/src/num/mod.rs | 36 ---- math/num/src/primitives.rs | 14 -- math/num/src/specs.rs | 31 ---- math/num/src/utils.rs | 9 - math/num/tests/default.rs | 8 - math/statistics/Cargo.toml | 45 ----- math/statistics/src/calc/deviation.rs | 64 ------- math/statistics/src/calc/mod.rs | 85 --------- math/statistics/src/lib.rs | 26 --- math/statistics/src/primitives.rs | 14 -- math/statistics/src/regression/linear.rs | 66 ------- math/statistics/src/regression/mod.rs | 19 -- math/statistics/src/specs.rs | 48 ----- math/statistics/src/utils.rs | 4 - math/statistics/tests/default.rs | 8 - ml/neural/Cargo.toml | 2 +- ml/neural/src/neurons/activate/softmax.rs | 2 +- ml/transformers/src/codec/encode/embed.rs | 14 ++ ml/transformers/src/codec/encode/mod.rs | 3 +- 47 files changed, 26 insertions(+), 1132 deletions(-) delete mode 100644 math/linalg/Cargo.toml delete mode 100644 math/linalg/src/lib.rs delete mode 100644 math/linalg/src/primitives.rs delete mode 100644 math/linalg/src/specs.rs delete mode 100644 math/linalg/src/utils.rs delete mode 100644 math/linalg/src/vs/mod.rs delete mode 100644 math/linalg/src/vs/space.rs delete mode 100644 math/linalg/tests/default.rs delete mode 100644 math/math/Cargo.toml delete mode 100644 math/math/examples/simple.rs delete mode 100644 math/math/src/calculus/derivatives/mod.rs delete mode 100644 math/math/src/calculus/derivatives/nderive.rs delete mode 100644 math/math/src/calculus/mod.rs delete mode 100644 math/math/src/lib.rs delete mode 100644 math/math/src/primitives.rs delete mode 100644 math/math/src/specs.rs delete mode 100644 math/math/src/utils.rs delete mode 100644 math/math/tests/default.rs delete mode 100644 math/num/Cargo.toml delete mode 100644 math/num/src/factorials.rs delete mode 100644 math/num/src/lib.rs delete mode 100644 math/num/src/num/arithmetic.rs delete mode 100644 math/num/src/num/complex.rs delete mode 100644 math/num/src/num/mod.rs delete mode 100644 math/num/src/primitives.rs delete mode 100644 math/num/src/specs.rs delete mode 100644 math/num/src/utils.rs delete mode 100644 math/num/tests/default.rs delete mode 100644 math/statistics/Cargo.toml delete mode 100644 math/statistics/src/calc/deviation.rs delete mode 100644 math/statistics/src/calc/mod.rs delete mode 100644 math/statistics/src/lib.rs delete mode 100644 math/statistics/src/primitives.rs delete mode 100644 math/statistics/src/regression/linear.rs delete mode 100644 math/statistics/src/regression/mod.rs delete mode 100644 math/statistics/src/specs.rs delete mode 100644 math/statistics/src/utils.rs delete mode 100644 math/statistics/tests/default.rs create mode 100644 ml/transformers/src/codec/encode/embed.rs diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 44be9222..6c555a01 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -29,10 +29,10 @@ updates: schedule: interval: daily - package-ecosystem: cargo - directory: /math + directory: /ml/neural schedule: interval: daily - package-ecosystem: cargo - directory: /ml/nn + directory: /ml/transformers schedule: interval: daily \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 669e205c..309ff0a0 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - feature: [ nn ] + feature: [ neural, transformers ] env: PACKAGE_NAME: ${{ github.event.repository.name }}-${{ matrix.feature }} steps: diff --git a/Cargo.toml b/Cargo.toml index 7d241638..d30e6577 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,8 @@ version = "0.1.12" # TODO - Update the cargo package version [workspace.dependencies] concision-core = { path = "core", version = "0.1.12" } +computare = { features = ["full"], branch = "v0.1.0", git = "https://github.com/FL03/computare", version = "0.1.0" } + anyhow = "1" lazy_static = "1" ndarray = { features = ["serde-1"], version = "0.15" } @@ -33,7 +35,6 @@ members = [ "data", "derive", "macros", - "math/*", "ml/*", ] diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 6cabb535..7302ef18 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -13,14 +13,13 @@ version.workspace = true [features] default = [ - "core", - "math", + "core", + "ml", ] full = [ "core", - "derive", - "math", + "derive", "ml", ] @@ -41,10 +40,6 @@ macros = [ "concision-macros" ] -math = [ - "concision-math" -] - ml = [ "neural", "transformers" @@ -71,7 +66,6 @@ concision-core = { features = [], optional = true, path = "../core", version = " concision-data = { features = [], optional = true, path = "../data", version = "0.1.12" } concision-derive = { features = [], optional = true, path = "../derive", version = "0.1.12" } concision-macros = { features = [], optional = true, path = "../macros", version = "0.1.12" } -concision-math = { features = [], optional = true, path = "../math/math", version = "0.1.12" } concision-neural = { features = [], optional = true, path = "../ml/neural", version = "0.1.12" } concision-transformers = { features = [], optional = true, path = "../ml/transformers", version = "0.1.12" } diff --git a/concision/src/lib.rs b/concision/src/lib.rs index 11d3b21b..1c8ffebc 100644 --- a/concision/src/lib.rs +++ b/concision/src/lib.rs @@ -15,8 +15,6 @@ pub use concision_data as data; pub use concision_derive::*; #[cfg(feature = "macros")] pub use concision_macros::*; -#[cfg(feature = "math")] -pub use concision_math as math; #[cfg(feature = "nn")] pub use concision_nn as nn; #[cfg(feature = "transformers")] @@ -31,8 +29,6 @@ pub mod prelude { pub use concision_derive::*; #[cfg(feature = "macros")] pub use concision_macros::*; - #[cfg(feature = "math")] - pub use concision_math::prelude::*; #[cfg(feature = "nn")] pub use concision_nn::prelude::*; #[cfg(feature = "transformers")] diff --git a/math/linalg/Cargo.toml b/math/linalg/Cargo.toml deleted file mode 100644 index d960eec6..00000000 --- a/math/linalg/Cargo.toml +++ /dev/null @@ -1,44 +0,0 @@ -[package] -authors.workspace = true -categories.workspace = true -description.workspace = true -edition.workspace = true -homepage.workspace = true -keywords.workspace = true -license.workspace = true -name = "concision-linalg" -readme.workspace = true -repository.workspace = true -version.workspace = true - -[features] -default = [] - -[lib] -bench = true -crate-type = ["cdylib", "rlib"] -doctest = false -test = true - -[build-dependencies] - -[dependencies] -concision-core.workspace = true - -ndarray.workspace = true -num.workspace = true -serde.workspace = true -serde_json.workspace = true -smart-default.workspace = true -strum.workspace = true - -[dev-dependencies] -rand = "0.8" - -[package.metadata.docs.rs] -all-features = true -rustc-args = ["--cfg", "docsrs"] - -[target.wasm32-unknown-unknown] - -[target.wasm32-wasi] \ No newline at end of file diff --git a/math/linalg/src/lib.rs b/math/linalg/src/lib.rs deleted file mode 100644 index 2f483ed9..00000000 --- a/math/linalg/src/lib.rs +++ /dev/null @@ -1,22 +0,0 @@ -/* - Appellation: concision-linalg - Contrib: FL03 -*/ -//! # concision-linalg - -pub use self::{primitives::*, specs::*, utils::*}; - -pub mod vs; - -// pub(crate) use concision_core as core; - -pub(crate) mod primitives; -pub(crate) mod specs; -pub(crate) mod utils; - -pub mod prelude { - pub use crate::primitives::*; - pub use crate::specs::*; - pub use crate::utils::*; - pub use crate::vs::*; -} diff --git a/math/linalg/src/primitives.rs b/math/linalg/src/primitives.rs deleted file mode 100644 index f16ac25f..00000000 --- a/math/linalg/src/primitives.rs +++ /dev/null @@ -1,14 +0,0 @@ -/* - Appellation: primitives - Contrib: FL03 -*/ -pub use self::{constants::*, statics::*, types::*}; - -/// Collection of constants used throughout the system -mod constants {} - -/// Collection of static references used throughout -mod statics {} - -/// Collection of types used throughout the system -mod types {} diff --git a/math/linalg/src/specs.rs b/math/linalg/src/specs.rs deleted file mode 100644 index 8abf79cb..00000000 --- a/math/linalg/src/specs.rs +++ /dev/null @@ -1,4 +0,0 @@ -/* - Appellation: specs - Contrib: FL03 -*/ diff --git a/math/linalg/src/utils.rs b/math/linalg/src/utils.rs deleted file mode 100644 index f8840c69..00000000 --- a/math/linalg/src/utils.rs +++ /dev/null @@ -1,4 +0,0 @@ -/* - Appellation: utils - Contrib: FL03 -*/ diff --git a/math/linalg/src/vs/mod.rs b/math/linalg/src/vs/mod.rs deleted file mode 100644 index b5f7ba9f..00000000 --- a/math/linalg/src/vs/mod.rs +++ /dev/null @@ -1,37 +0,0 @@ -/* - Appellation: vs - Contrib: FL03 -*/ -//! # Vector Space -//! -//! A vector space is defined to be a set V, whose elements are called vectors, on which are defined two operations, -//! called addition and multiplication by scalars (real numbers), subject to the ten axioms listed below. -//! -//! ## Axioms -//! -//! 1. Closure under addition -//! 2. Closure under scalar multiplication -//! 3. Commutativity of addition -//! 4. Associativity of addition -//! 5. Additive identity -//! 6. Additive inverse -//! 7. Distributivity of scalar multiplication with respect to vector addition -//! 8. Distributivity of scalar multiplication with respect to field addition -//! 9. Compatibility of scalar multiplication with field multiplication -//! 10. Identity element of scalar multiplication -pub use self::{space::*, utils::*}; - -pub(crate) mod space; - -pub(crate) mod utils {} - -#[cfg(test)] -mod tests { - #[test] - fn test() { - let f = |x: usize, y: usize| x + y; - let actual = f(4, 4); - let expected: usize = 8; - assert_eq!(actual, expected) - } -} diff --git a/math/linalg/src/vs/space.rs b/math/linalg/src/vs/space.rs deleted file mode 100644 index dfaa0730..00000000 --- a/math/linalg/src/vs/space.rs +++ /dev/null @@ -1,26 +0,0 @@ -/* - Appellation: space - Contrib: FL03 -*/ - -pub trait Subspace: VectorSpace {} - -pub struct Space { - pub data: Vec, - pub shape: Vec, -} - -pub trait VectorSpace { - type Dim; -} - -#[cfg(test)] -mod tests { - #[test] - fn test() { - let f = |x: usize, y: usize| x + y; - let actual = f(4, 4); - let expected: usize = 8; - assert_eq!(actual, expected) - } -} diff --git a/math/linalg/tests/default.rs b/math/linalg/tests/default.rs deleted file mode 100644 index 0cac1eb5..00000000 --- a/math/linalg/tests/default.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[cfg(test)] -#[test] -fn compiles() { - let f = |x: usize, y: usize| x + y; - - assert_eq!(f(10, 10), 20); - assert_ne!(f(1, 1), 3); -} diff --git a/math/math/Cargo.toml b/math/math/Cargo.toml deleted file mode 100644 index 94e5dd11..00000000 --- a/math/math/Cargo.toml +++ /dev/null @@ -1,48 +0,0 @@ -[package] -authors.workspace = true -categories.workspace = true -description.workspace = true -edition.workspace = true -homepage.workspace = true -keywords.workspace = true -license.workspace = true -name = "concision-math" -readme.workspace = true -repository.workspace = true -version.workspace = true - -[features] -default = [] - -[lib] -bench = true -crate-type = ["cdylib", "rlib"] -doctest = false -test = true - -[build-dependencies] - -[dependencies] -concision-linalg = { features = [], path = "../linalg", version = "0.1.12" } -concision-num = { features = [], path = "../num", version = "0.1.12" } -concision-statistics = { features = [], path = "../statistics", version = "0.1.12" } - -concision-core.workspace = true - -ndarray.workspace = true -num.workspace = true -serde.workspace = true -serde_json.workspace = true -smart-default.workspace = true -strum.workspace = true - -[dev-dependencies] -rand = "0.8" - -[package.metadata.docs.rs] -all-features = true -rustc-args = ["--cfg", "docsrs"] - -[target.wasm32-unknown-unknown] - -[target.wasm32-wasi] \ No newline at end of file diff --git a/math/math/examples/simple.rs b/math/math/examples/simple.rs deleted file mode 100644 index 38a262ea..00000000 --- a/math/math/examples/simple.rs +++ /dev/null @@ -1,16 +0,0 @@ -extern crate concision_math as math; - -use math::prelude::{complex::C, Numerical}; - -// Define a holomorphic function that squares its input. -fn square(z: C) -> C { - z.clone() * z -} - -fn main() { - let c = C::from((1.0, 1.0)); - let res = square(c); - assert_eq!(res.clone(), C::from((0.0, 2.0))); - - println!("{:?}", res); -} diff --git a/math/math/src/calculus/derivatives/mod.rs b/math/math/src/calculus/derivatives/mod.rs deleted file mode 100644 index ac004eab..00000000 --- a/math/math/src/calculus/derivatives/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -/* - Appellation: derivatives - Contrib: FL03 - Description: ... Summary ... -*/ -pub use self::nderive::*; - -pub(crate) mod nderive; diff --git a/math/math/src/calculus/derivatives/nderive.rs b/math/math/src/calculus/derivatives/nderive.rs deleted file mode 100644 index 9049f614..00000000 --- a/math/math/src/calculus/derivatives/nderive.rs +++ /dev/null @@ -1,21 +0,0 @@ -/* - Appellation: num - Contrib: FL03 - Description: ... Summary ... -*/ - -pub fn central(x: T, f: Box T>, h: T) -> T { - (f(x + h) - f(x)) / h -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_central() { - let f = |x: f64| x * x; - let res = central(5.0, Box::new(f), 0.001); - assert_eq!(res.round(), 10.0); - } -} diff --git a/math/math/src/calculus/mod.rs b/math/math/src/calculus/mod.rs deleted file mode 100644 index 18d2ab7c..00000000 --- a/math/math/src/calculus/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -/* - Appellation: calculus - Contrib: FL03 - Description: ... Summary ... -*/ - -pub mod derivatives; diff --git a/math/math/src/lib.rs b/math/math/src/lib.rs deleted file mode 100644 index 3f7db8ba..00000000 --- a/math/math/src/lib.rs +++ /dev/null @@ -1,33 +0,0 @@ -/* - Appellation: concision-math - Contrib: FL03 -*/ -//! # concision-math - -pub use self::{primitives::*, specs::*, utils::*}; - -pub mod calculus; - -pub use concision_linalg as linalg; - -pub use concision_num as num; - -pub use concision_statistics as stats; - -// pub(crate) use concision_core as core; - -pub(crate) mod primitives; -pub(crate) mod specs; -pub(crate) mod utils; - -pub mod prelude { - - pub use concision_linalg::prelude::*; - pub use concision_num::prelude::*; - pub use concision_statistics::prelude::*; - - pub use crate::calculus::*; - pub use crate::primitives::*; - pub use crate::specs::*; - pub use crate::utils::*; -} diff --git a/math/math/src/primitives.rs b/math/math/src/primitives.rs deleted file mode 100644 index f16ac25f..00000000 --- a/math/math/src/primitives.rs +++ /dev/null @@ -1,14 +0,0 @@ -/* - Appellation: primitives - Contrib: FL03 -*/ -pub use self::{constants::*, statics::*, types::*}; - -/// Collection of constants used throughout the system -mod constants {} - -/// Collection of static references used throughout -mod statics {} - -/// Collection of types used throughout the system -mod types {} diff --git a/math/math/src/specs.rs b/math/math/src/specs.rs deleted file mode 100644 index 8abf79cb..00000000 --- a/math/math/src/specs.rs +++ /dev/null @@ -1,4 +0,0 @@ -/* - Appellation: specs - Contrib: FL03 -*/ diff --git a/math/math/src/utils.rs b/math/math/src/utils.rs deleted file mode 100644 index f8840c69..00000000 --- a/math/math/src/utils.rs +++ /dev/null @@ -1,4 +0,0 @@ -/* - Appellation: utils - Contrib: FL03 -*/ diff --git a/math/math/tests/default.rs b/math/math/tests/default.rs deleted file mode 100644 index 0cac1eb5..00000000 --- a/math/math/tests/default.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[cfg(test)] -#[test] -fn compiles() { - let f = |x: usize, y: usize| x + y; - - assert_eq!(f(10, 10), 20); - assert_ne!(f(1, 1), 3); -} diff --git a/math/num/Cargo.toml b/math/num/Cargo.toml deleted file mode 100644 index 152a7b8a..00000000 --- a/math/num/Cargo.toml +++ /dev/null @@ -1,46 +0,0 @@ -[package] -authors.workspace = true -categories.workspace = true -description.workspace = true -edition.workspace = true -homepage.workspace = true -keywords.workspace = true -license.workspace = true -name = "concision-num" -readme.workspace = true -repository.workspace = true -version.workspace = true - -[features] -default = [] - -[lib] -bench = true -crate-type = ["cdylib", "rlib"] -doctest = false -test = true - -[build-dependencies] - -[dependencies] - - -concision-core.workspace = true - -ndarray.workspace = true -num.workspace = true -serde.workspace = true -serde_json.workspace = true -smart-default.workspace = true -strum.workspace = true - -[dev-dependencies] -rand = "0.8" - -[package.metadata.docs.rs] -all-features = true -rustc-args = ["--cfg", "docsrs"] - -[target.wasm32-unknown-unknown] - -[target.wasm32-wasi] \ No newline at end of file diff --git a/math/num/src/factorials.rs b/math/num/src/factorials.rs deleted file mode 100644 index ef051db7..00000000 --- a/math/num/src/factorials.rs +++ /dev/null @@ -1,54 +0,0 @@ -/* - Appellation: factorials - Contrib: FL03 -*/ -use std::ops::{Mul, Sub}; -use std::str::FromStr; - -#[derive(Clone, Debug, Default, Eq, Ord, PartialEq, PartialOrd)] -pub struct Factorial + Sub>(T); - -impl + Sub> Factorial -where - ::Err: std::fmt::Debug, -{ - pub fn new(data: T) -> Self { - Self(Self::compute(data)) - } - pub fn compute(data: T) -> T { - let x = data.to_string(); - let b: T = "1".parse().ok().unwrap(); - match x.as_str() { - "0" | "1" => b.clone(), - _ => Self::compute(data.clone() - b) * data, - } - } - pub fn data(&self) -> &T { - &self.0 - } - pub fn from_args(data: Vec) -> Vec { - data.iter() - .map(|i| Self::new(i.clone())) - .collect::>() - } -} - -pub fn factorial(data: T) -> T { - if data.is_zero() || data.is_one() { - return T::one(); - } - factorial(data - T::one()) * data -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_factorial() { - assert_eq!(Factorial::new(0).data().clone(), 1); - assert_eq!(factorial(3), 6); - assert_eq!(factorial(4.0), 24.0); - assert_eq!(factorial(5.0), 120.0); - } -} diff --git a/math/num/src/lib.rs b/math/num/src/lib.rs deleted file mode 100644 index 6cab992d..00000000 --- a/math/num/src/lib.rs +++ /dev/null @@ -1,25 +0,0 @@ -/* - Appellation: concision-math - Contrib: FL03 -*/ -//! # concision-math - -pub use self::{factorials::*, primitives::*, specs::*, utils::*}; - -pub mod num; - -pub(crate) mod factorials; - -// pub(crate) use concision_core as core; - -pub(crate) mod primitives; -pub(crate) mod specs; -pub(crate) mod utils; - -pub mod prelude { - pub use crate::factorials::*; - pub use crate::num::*; - pub use crate::primitives::*; - pub use crate::specs::*; - pub use crate::utils::*; -} diff --git a/math/num/src/num/arithmetic.rs b/math/num/src/num/arithmetic.rs deleted file mode 100644 index 55853287..00000000 --- a/math/num/src/num/arithmetic.rs +++ /dev/null @@ -1,21 +0,0 @@ -/* - Appellation: arithmetic - Contrib: FL03 -*/ -use std::ops; - -pub trait Addition: ops::Add + ops::AddAssign + Sized {} - -impl Addition for T where T: ops::Add + ops::AddAssign + Sized {} - -pub trait Division: ops::Div + ops::DivAssign + Sized {} - -impl Division for T where T: ops::Div + ops::DivAssign + Sized {} - -pub trait Multiplication: ops::Mul + ops::MulAssign + Sized {} - -impl Multiplication for T where T: ops::Mul + ops::MulAssign + Sized {} - -pub trait Subtraction: ops::Sub + ops::SubAssign + Sized {} - -impl Subtraction for T where T: ops::Sub + ops::SubAssign + Sized {} diff --git a/math/num/src/num/complex.rs b/math/num/src/num/complex.rs deleted file mode 100644 index 8c0ee0ad..00000000 --- a/math/num/src/num/complex.rs +++ /dev/null @@ -1,167 +0,0 @@ -/* - Appellation: complex - Contrib: FL03 -*/ -use super::Numerical; -use serde::{Deserialize, Serialize}; -use std::ops; - -// Define a trait for complex numbers. -pub trait Complex { - fn new(re: T, im: T) -> Self; - fn re(&self) -> T; - fn im(&self) -> T; -} - -// Implement the Complex trait for the tuple (f64, f64). -impl Complex for (T, T) { - fn new(re: T, im: T) -> Self { - (re, im) - } - - fn re(&self) -> T { - self.0 - } - - fn im(&self) -> T { - self.1 - } -} - -impl Complex for [T; 2] { - fn new(re: T, im: T) -> Self { - [re, im] - } - - fn re(&self) -> T { - self[0] - } - - fn im(&self) -> T { - self[1] - } -} - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, PartialOrd, Serialize)] -pub struct C(T, T); - -impl Complex for C { - fn new(re: T, im: T) -> Self { - Self(re, im) - } - fn re(&self) -> T { - self.0 - } - fn im(&self) -> T { - self.1 - } -} - -impl From<(T, T)> for C { - fn from(data: (T, T)) -> Self { - Self::new(data.0, data.1) - } -} - -impl From> for (T, T) { - fn from(data: C) -> (T, T) { - (data.re(), data.im()) - } -} - -// Implement the Add and Mul traits for complex numbers. -impl ops::Add for C { - type Output = Self; - - fn add(self, other: Self) -> Self { - Self::from((self.re() + other.re(), self.im() + other.im())) - } -} - -impl ops::AddAssign for C -where - T: Numerical + ops::AddAssign, -{ - fn add_assign(&mut self, other: Self) { - self.0 += other.0; - self.1 += other.1; - } -} - -impl ops::Div for C { - type Output = Self; - - fn div(self, other: Self) -> Self { - let denom = other.re() * other.re() + other.im() * other.im(); - Self::from(( - (self.re() * other.re() + self.im() * other.im()) / denom, - (self.im() * other.re() - self.re() * other.im()) / denom, - )) - } -} - -impl ops::DivAssign for C { - fn div_assign(&mut self, other: Self) { - let denom = other.re() * other.re() + other.im() * other.im(); - let re = (self.re() * other.re() + self.im() * other.im()) / denom; - let im = (self.im() * other.re() - self.re() * other.im()) / denom; - self.0 = re; - self.1 = im; - } -} - -impl ops::Mul for C { - type Output = Self; - - fn mul(self, other: Self) -> Self { - Self::from(( - self.re() * other.re() - self.im() * other.im(), - self.re() * other.im() + self.im() * other.re(), - )) - } -} - -impl ops::MulAssign for C { - fn mul_assign(&mut self, other: Self) { - let re = self.re() * other.re() - self.im() * other.im(); - let im = self.re() * other.im() + self.im() * other.re(); - self.0 = re; - self.1 = im; - } -} - -impl ops::Sub for C { - type Output = Self; - - fn sub(self, other: Self) -> Self { - Self::from((self.re() - other.re(), self.im() - other.im())) - } -} - -impl ops::SubAssign for C -where - T: Numerical + ops::SubAssign, -{ - fn sub_assign(&mut self, other: Self) { - self.0 -= other.0; - self.1 -= other.1; - } -} - -#[cfg(test)] -mod tests { - use super::*; - // Define a holomorphic function that squares its input. - fn square(z: C) -> C { - z.clone() * z - } - - #[test] - fn test_square() { - // Create a complex number (1 + i) and square it. - let mut a = C::from((1.0, 1.0)); - a = square(a.clone()); - let b = C::new(0.0, 2.0); - assert_eq!(&a, &b); - } -} diff --git a/math/num/src/num/mod.rs b/math/num/src/num/mod.rs deleted file mode 100644 index e220f63d..00000000 --- a/math/num/src/num/mod.rs +++ /dev/null @@ -1,36 +0,0 @@ -/* - Appellation: num - Contrib: FL03 -*/ -pub use self::{arithmetic::*, utils::*}; - -pub(crate) mod arithmetic; - -pub mod complex; - -use std::ops::{self, Div, Mul, Sub}; - -pub trait Binary: num::One + num::Zero {} - -impl Binary for T where T: num::One + num::Zero {} - -pub trait Arithmetic: - ops::Add + Div + Mul + Sub + Sized -{ -} - -impl Arithmetic for T where - T: ops::Add - + Div - + Mul - + Sub - + Sized -{ -} - -/// [Numerical] is a basic trait describing numerical objects -pub trait Numerical: Arithmetic + Copy + PartialEq {} - -impl Numerical for T where T: Arithmetic + Copy + PartialEq {} - -pub(crate) mod utils {} diff --git a/math/num/src/primitives.rs b/math/num/src/primitives.rs deleted file mode 100644 index f16ac25f..00000000 --- a/math/num/src/primitives.rs +++ /dev/null @@ -1,14 +0,0 @@ -/* - Appellation: primitives - Contrib: FL03 -*/ -pub use self::{constants::*, statics::*, types::*}; - -/// Collection of constants used throughout the system -mod constants {} - -/// Collection of static references used throughout -mod statics {} - -/// Collection of types used throughout the system -mod types {} diff --git a/math/num/src/specs.rs b/math/num/src/specs.rs deleted file mode 100644 index 31f77629..00000000 --- a/math/num/src/specs.rs +++ /dev/null @@ -1,31 +0,0 @@ -/* - Appellation: specs - Contrib: FL03 -*/ - -pub trait RoundTo { - fn round_to(&self, decimals: usize) -> Self; -} - -impl RoundTo for T -where - T: num::Float, -{ - fn round_to(&self, decimals: usize) -> Self { - let val = T::from(self.clone()).expect("Failed to convert to type T"); - let factor = T::from(10).expect("").powi(decimals as i32); - (val * factor).round() / factor - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_round_to() { - let num = 1.23456789_f64; - assert_eq!(num.round_to(2), 1.23_f64); - assert_eq!(num.round_to(3), 1.235_f64); - } -} diff --git a/math/num/src/utils.rs b/math/num/src/utils.rs deleted file mode 100644 index e880e121..00000000 --- a/math/num/src/utils.rs +++ /dev/null @@ -1,9 +0,0 @@ -/* - Appellation: utils - Contrib: FL03 -*/ - -pub fn round(num: T, decimals: usize) -> T { - let factor = T::from(10.0).expect("").powi(decimals as i32); - (num * factor).round() / factor -} diff --git a/math/num/tests/default.rs b/math/num/tests/default.rs deleted file mode 100644 index 0cac1eb5..00000000 --- a/math/num/tests/default.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[cfg(test)] -#[test] -fn compiles() { - let f = |x: usize, y: usize| x + y; - - assert_eq!(f(10, 10), 20); - assert_ne!(f(1, 1), 3); -} diff --git a/math/statistics/Cargo.toml b/math/statistics/Cargo.toml deleted file mode 100644 index 00ecabf3..00000000 --- a/math/statistics/Cargo.toml +++ /dev/null @@ -1,45 +0,0 @@ -[package] -authors.workspace = true -categories.workspace = true -description.workspace = true -edition.workspace = true -homepage.workspace = true -keywords.workspace = true -license.workspace = true -name = "concision-statistics" -readme.workspace = true -repository.workspace = true -version.workspace = true - -[features] -default = [] - -[lib] -bench = true -crate-type = ["cdylib", "rlib"] -doctest = false -test = true - -[build-dependencies] - -[dependencies] -concision-core.workspace = true - -ndarray.workspace = true -num.workspace = true -serde.workspace = true -serde_json.workspace = true -smart-default.workspace = true -strum.workspace = true - -[dev-dependencies] -concision-num = { features = [], path = "../num", version = "0.1.12" } -rand = "0.8" - -[package.metadata.docs.rs] -all-features = true -rustc-args = ["--cfg", "docsrs"] - -[target.wasm32-unknown-unknown] - -[target.wasm32-wasi] \ No newline at end of file diff --git a/math/statistics/src/calc/deviation.rs b/math/statistics/src/calc/deviation.rs deleted file mode 100644 index e19fcd15..00000000 --- a/math/statistics/src/calc/deviation.rs +++ /dev/null @@ -1,64 +0,0 @@ -/* - Appellation: deviation - Contrib: FL03 -*/ -use super::utils::*; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, PartialOrd, Serialize)] -pub struct StandardDeviation { - pub deviation: f64, - pub mean: f64, - pub variance: f64, -} - -impl StandardDeviation { - pub fn new(x: &[f64]) -> StandardDeviation { - let mean = mean(x); - let variance = x.iter().map(|&x| x * x).sum::() / x.len() as f64 - mean * mean; - let deviation = variance.sqrt(); - StandardDeviation { - deviation, - mean, - variance, - } - } - - pub fn compute(&mut self, x: &[f64]) -> f64 { - let mean = x.iter().sum::() / x.len() as f64; - let variance = x.iter().map(|&x| x * x).sum::() / x.len() as f64 - mean * mean; - let deviation = variance.sqrt(); - self.deviation = deviation; - self.mean = mean; - self.variance = variance; - deviation - } - - pub fn deviation(&self) -> f64 { - self.deviation - } - - pub fn mean(&self) -> f64 { - self.mean - } - - pub fn variance(&self) -> f64 { - self.variance - } -} - -#[cfg(test)] -mod tests { - use super::*; - - use crate::prelude::Statistics; - use concision_num::RoundTo; - - #[test] - fn test_std() { - let x = vec![1.0, 2.0, 4.0, 9.0, 3.0, 4.0, 5.0]; - let sd = StandardDeviation::new(&x); - - assert_eq!(x.std().round_to(5), sd.deviation().round_to(5)); - } -} diff --git a/math/statistics/src/calc/mod.rs b/math/statistics/src/calc/mod.rs deleted file mode 100644 index 4023625f..00000000 --- a/math/statistics/src/calc/mod.rs +++ /dev/null @@ -1,85 +0,0 @@ -/* - Appellation: statistics - Contrib: FL03 -*/ -//! # statistics -pub use self::{deviation::*, utils::*}; - -pub(crate) mod deviation; - -pub(crate) mod utils { - use std::iter::Sum; - - /// Covariance is the average of the products of the deviations from the mean. - pub fn covariance(x: &[T], y: &[T]) -> T { - let dx = deviation(&x); - let dy = deviation(&y); - dx.iter().zip(dy.iter()).map(|(&x, &y)| x * y).sum::() / T::from(dx.len()).unwrap() - } - /// Deviation is the distance from the mean. - pub fn deviation(x: &[T]) -> Vec { - let mean = mean(x); - x.iter().map(|&x| x - mean).collect() - } - /// Mean is the average of the data. - pub fn mean(x: &[T]) -> T { - x.iter().cloned().sum::() / T::from(x.len()).unwrap() - } - /// Variance is the average of the squared deviations from the mean. - pub fn variance(x: &[T]) -> T { - let dev = deviation(&x); - dev.iter().map(|&x| x * x).sum::() / T::from(dev.len()).unwrap() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::Statistics; - use rand::Rng; - - fn random_vec() -> Vec { - let mut rng = rand::thread_rng(); - let mut v = Vec::new(); - for _ in 0..100 { - v.push(rng.gen_range(0.0..25.0)); - } - v - } - - #[test] - fn test_statistics() { - let x = random_vec(); - let y = random_vec(); - assert_eq!(covariance(&x, &y), x.covariance(&y)); - assert_eq!(deviation(&x), x.deviation()); - assert_eq!(mean(&x), x.mean()); - assert_eq!(variance(&x), x.variance()); - } - - #[test] - fn test_covariance() { - let x: Vec = (1..=5).map(|i| i as f64).collect(); - let y = vec![1.0, 2.0, 3.0, 4.0, 5.0]; - assert_eq!(covariance(&x, &y), 2.0); - assert_eq!(x.covariance(&y), 2.0); - } - - #[test] - fn test_deviation() { - let x = vec![1.0, 2.0, 3.0, 4.0, 5.0]; - assert_eq!(deviation(&x), vec![-2.0, -1.0, 0.0, 1.0, 2.0]); - } - - #[test] - fn test_mean() { - let x = vec![1.0, 2.0, 3.0, 4.0, 5.0]; - assert_eq!(mean(&x), 3.0); - } - - #[test] - fn test_variance() { - let x = vec![1.0, 2.0, 3.0, 4.0, 5.0]; - assert_eq!(variance(&x), 2.0); - } -} diff --git a/math/statistics/src/lib.rs b/math/statistics/src/lib.rs deleted file mode 100644 index b751fda1..00000000 --- a/math/statistics/src/lib.rs +++ /dev/null @@ -1,26 +0,0 @@ -/* - Appellation: concision-statistics - Contrib: FL03 -*/ -//! # concision-statistics - -pub use self::{primitives::*, specs::*, utils::*}; - -pub mod calc; -pub mod regression; - -// pub(crate) use concision_num as num; - -pub(crate) mod primitives; -pub(crate) mod specs; -pub(crate) mod utils; - -pub mod prelude { - - pub use crate::calc::*; - pub use crate::regression::*; - - pub use crate::primitives::*; - pub use crate::specs::*; - pub use crate::utils::*; -} diff --git a/math/statistics/src/primitives.rs b/math/statistics/src/primitives.rs deleted file mode 100644 index f16ac25f..00000000 --- a/math/statistics/src/primitives.rs +++ /dev/null @@ -1,14 +0,0 @@ -/* - Appellation: primitives - Contrib: FL03 -*/ -pub use self::{constants::*, statics::*, types::*}; - -/// Collection of constants used throughout the system -mod constants {} - -/// Collection of static references used throughout -mod statics {} - -/// Collection of types used throughout the system -mod types {} diff --git a/math/statistics/src/regression/linear.rs b/math/statistics/src/regression/linear.rs deleted file mode 100644 index 10c0612f..00000000 --- a/math/statistics/src/regression/linear.rs +++ /dev/null @@ -1,66 +0,0 @@ -/* - Appellation: linear - Contrib: FL03 -*/ -use super::Regression; -use crate::calc::{covariance, deviation, mean, variance}; - -#[derive(Clone, Copy, Debug, Default, PartialEq, PartialOrd)] -pub struct LinearRegression { - intercept: f64, - slope: f64, -} - -impl LinearRegression { - pub fn new(x: &[f64], y: &[f64]) -> LinearRegression { - let (x_mean, y_mean) = (mean(x), mean(y)); - let (x_dev, y_dev) = (deviation(x), deviation(y)); - let slope = covariance(&x_dev, &y_dev) / variance(&x_dev); - let intercept = y_mean - slope * x_mean; - LinearRegression { intercept, slope } - } - - pub fn intercept(&self) -> f64 { - self.intercept - } - - pub fn predict(&self, x: &[f64]) -> Vec { - x.iter().map(|&x| self.slope * x + self.intercept).collect() - } - - pub fn slope(&self) -> f64 { - self.slope - } -} - -impl Regression for LinearRegression { - type Item = f64; - - fn fit(&mut self, args: &[Self::Item], target: &[Self::Item]) { - let (x_mean, y_mean) = (mean(args), mean(target)); - let (x_dev, y_dev) = (deviation(args), deviation(target)); - self.slope = covariance(&x_dev, &y_dev) / variance(&x_dev); - self.intercept = y_mean - self.slope * x_mean; - } - - fn predict(&self, args: &[Self::Item]) -> Vec { - args.iter() - .map(|&x| self.slope * x + self.intercept) - .collect() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_linear_regression() { - let x = vec![1.0, 2.0, 3.0, 4.0, 5.0]; - let y = vec![1.0, 2.0, 3.0, 4.0, 5.0]; - let lr = LinearRegression::new(&x, &y); - assert_eq!(lr.slope, 1.0); - assert_eq!(lr.intercept, 0.0); - assert_eq!(lr.predict(&x), y); - } -} diff --git a/math/statistics/src/regression/mod.rs b/math/statistics/src/regression/mod.rs deleted file mode 100644 index 9566f194..00000000 --- a/math/statistics/src/regression/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -/* - Appellation: regression - Contrib: FL03 -*/ -//! # regression -//! -pub use self::utils::*; - -pub mod linear; - -pub trait Regression { - type Item: num::Float; - - fn fit(&mut self, args: &[Self::Item], target: &[Self::Item]); - - fn predict(&self, args: &[Self::Item]) -> Vec; -} - -pub(crate) mod utils {} diff --git a/math/statistics/src/specs.rs b/math/statistics/src/specs.rs deleted file mode 100644 index d178f942..00000000 --- a/math/statistics/src/specs.rs +++ /dev/null @@ -1,48 +0,0 @@ -/* - Appellation: specs - Contrib: FL03 -*/ - -pub trait Statistics -where - T: num::Float + std::iter::Sum, - Self: Clone + IntoIterator, -{ - fn covariance(&self, other: &Self) -> T { - let dx = self.deviation(); - let dy = other.deviation(); - dx.iter().zip(dy.iter()).map(|(&x, &y)| x * y).sum::() / T::from(dx.len()).unwrap() - } - - fn deviation(&self) -> Vec { - let mean = self.mean(); - self.clone().into_iter().map(|x| x - mean).collect() - } - fn len(&self) -> usize { - Vec::from_iter(self.clone().into_iter()).len() - } - /// [Statistics::mean] calculates the mean or average of the data - fn mean(&self) -> T { - self.clone().into_iter().sum::() / T::from(self.len()).unwrap() - } - /// [Statistics::std] calculates the standard deviation of the data - fn std(&self) -> T { - let mean = self.mean(); - let mut res = self - .clone() - .into_iter() - .map(|x| (x - mean).powi(2)) - .sum::(); - res = res / T::from(self.len()).unwrap(); - res.sqrt() - } - - fn variance(&self) -> T { - let dev = self.deviation(); - dev.iter().map(|&x| x * x).sum::() / T::from(dev.len()).unwrap() - } -} - -impl Statistics for Vec where T: num::Float + std::iter::Sum {} - -impl Statistics for ndarray::Array1 where T: num::Float + std::iter::Sum {} diff --git a/math/statistics/src/utils.rs b/math/statistics/src/utils.rs deleted file mode 100644 index f8840c69..00000000 --- a/math/statistics/src/utils.rs +++ /dev/null @@ -1,4 +0,0 @@ -/* - Appellation: utils - Contrib: FL03 -*/ diff --git a/math/statistics/tests/default.rs b/math/statistics/tests/default.rs deleted file mode 100644 index 0cac1eb5..00000000 --- a/math/statistics/tests/default.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[cfg(test)] -#[test] -fn compiles() { - let f = |x: usize, y: usize| x + y; - - assert_eq!(f(10, 10), 20); - assert_ne!(f(1, 1), 3); -} diff --git a/ml/neural/Cargo.toml b/ml/neural/Cargo.toml index a0707028..5aed8472 100644 --- a/ml/neural/Cargo.toml +++ b/ml/neural/Cargo.toml @@ -35,7 +35,7 @@ smart-default.workspace = true strum.workspace = true [dev-dependencies] -concision-math = { features = [], path = "../../math/math", version = "0.1.12" } +computare.workspace = true [package.metadata.docs.rs] all-features = true diff --git a/ml/neural/src/neurons/activate/softmax.rs b/ml/neural/src/neurons/activate/softmax.rs index 22b2adff..587e0182 100644 --- a/ml/neural/src/neurons/activate/softmax.rs +++ b/ml/neural/src/neurons/activate/softmax.rs @@ -19,7 +19,7 @@ where #[cfg(test)] mod tests { use super::*; - use concision_math::prelude::RoundTo; + use computare::prelude::RoundTo; #[test] fn test_softmax() { diff --git a/ml/transformers/src/codec/encode/embed.rs b/ml/transformers/src/codec/encode/embed.rs new file mode 100644 index 00000000..7219df08 --- /dev/null +++ b/ml/transformers/src/codec/encode/embed.rs @@ -0,0 +1,14 @@ +/* + Appellation: embed + Contrib: FL03 +*/ + +pub trait Embed {} + +#[derive(Clone)] +pub struct Embedding { + pub(crate) embedding: Vec, + pub(crate) position: Vec, +} + + diff --git a/ml/transformers/src/codec/encode/mod.rs b/ml/transformers/src/codec/encode/mod.rs index f251ec9e..d2a78b58 100644 --- a/ml/transformers/src/codec/encode/mod.rs +++ b/ml/transformers/src/codec/encode/mod.rs @@ -3,8 +3,9 @@ Contrib: FL03 */ //! # Encode -pub use self::{encoder::*, utils::*}; +pub use self::{embed::*, encoder::*, utils::*}; +pub(crate) mod embed; pub(crate) mod encoder; pub trait Encode {} From 0801fed604ebb6dddae6528db397b9da8bdcff02 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 21 Oct 2023 10:03:31 -0500 Subject: [PATCH 024/118] update Signed-off-by: FL03 --- concision/Cargo.toml | 2 ++ concision/examples/basic.rs | 16 +++------ data/src/flows/direction.rs | 33 +++++++++++++++++ data/src/flows/mod.rs | 3 +- ml/neural/src/arch/architecture.rs | 4 +++ ml/neural/src/arch/mod.rs | 14 ++++++++ ml/neural/src/layers/kinds.rs | 17 +++++++++ ml/neural/src/lib.rs | 1 + .../src/neurons/activate/common/binary.rs | 20 +++++++++++ .../src/neurons/activate/common/linear.rs | 4 +++ ml/neural/src/neurons/activate/common/mod.rs | 12 +++++++ .../{softmax.rs => common/nonlinear.rs} | 24 ++++++++++--- ml/neural/src/neurons/activate/mod.rs | 27 +++++++------- ml/neural/src/neurons/neuron.rs | 36 ++++++++----------- ml/neural/src/neurons/node.rs | 25 +++++++------ ml/neural/src/nn/loss/regress.rs | 26 ++++++++++++++ ml/neural/src/nn/mod.rs | 15 +++++++- ml/neural/src/nn/network.rs | 9 +++++ ml/neural/src/prop/kinds.rs | 9 +++++ ml/neural/src/prop/mod.rs | 5 ++- ml/neural/src/prop/propagation.rs | 4 +++ ml/neural/src/specs.rs | 2 -- ml/transformers/src/codec/encode/embed.rs | 19 ++++++++-- 23 files changed, 254 insertions(+), 73 deletions(-) create mode 100644 data/src/flows/direction.rs create mode 100644 ml/neural/src/arch/architecture.rs create mode 100644 ml/neural/src/arch/mod.rs create mode 100644 ml/neural/src/neurons/activate/common/binary.rs create mode 100644 ml/neural/src/neurons/activate/common/linear.rs create mode 100644 ml/neural/src/neurons/activate/common/mod.rs rename ml/neural/src/neurons/activate/{softmax.rs => common/nonlinear.rs} (57%) create mode 100644 ml/neural/src/nn/network.rs create mode 100644 ml/neural/src/prop/kinds.rs create mode 100644 ml/neural/src/prop/propagation.rs diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 7302ef18..570e7ed5 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -14,11 +14,13 @@ version.workspace = true [features] default = [ "core", + "data", "ml", ] full = [ "core", + "data", "derive", "ml", ] diff --git a/concision/examples/basic.rs b/concision/examples/basic.rs index af3ae1d4..76f97644 100644 --- a/concision/examples/basic.rs +++ b/concision/examples/basic.rs @@ -1,17 +1,9 @@ extern crate concision; -use concision::math::prelude::complex::C; -use concision::prelude::Numerical; +use concision::prelude::BoxResult; -// Define a holomorphic function that squares its input. -fn square(z: C) -> C { - z.clone() * z -} - -fn main() { - let c = C::from((1.0, 1.0)); - let res = square(c); - assert_eq!(res.clone(), C::from((0.0, 2.0))); +fn main() -> BoxResult { + println!("Welcome to concision!"); - println!("{:?}", res); + Ok(()) } diff --git a/data/src/flows/direction.rs b/data/src/flows/direction.rs new file mode 100644 index 00000000..68bff080 --- /dev/null +++ b/data/src/flows/direction.rs @@ -0,0 +1,33 @@ +/* + Appellation: direction + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum Direction { + #[default] + Forward, + Backward, +} diff --git a/data/src/flows/mod.rs b/data/src/flows/mod.rs index 9d06cd2c..08de4074 100644 --- a/data/src/flows/mod.rs +++ b/data/src/flows/mod.rs @@ -3,8 +3,9 @@ Contrib: FL03 */ //! # Flows -pub use self::{flow::*, utils::*}; +pub use self::{direction::*, flow::*, utils::*}; +pub(crate) mod direction; pub(crate) mod flow; pub(crate) mod utils {} diff --git a/ml/neural/src/arch/architecture.rs b/ml/neural/src/arch/architecture.rs new file mode 100644 index 00000000..ac0460ef --- /dev/null +++ b/ml/neural/src/arch/architecture.rs @@ -0,0 +1,4 @@ +/* + Appellation: architecture + Contrib: FL03 +*/ diff --git a/ml/neural/src/arch/mod.rs b/ml/neural/src/arch/mod.rs new file mode 100644 index 00000000..d92c10b6 --- /dev/null +++ b/ml/neural/src/arch/mod.rs @@ -0,0 +1,14 @@ +/* + Appellation: arch + Contrib: FL03 +*/ +//! # Architecture +//! +//! This module describes the architecture of various components of the neural network. +pub use self::{architecture::*, utils::*}; + +pub(crate) mod architecture; + +pub trait Arch {} + +pub(crate) mod utils {} diff --git a/ml/neural/src/layers/kinds.rs b/ml/neural/src/layers/kinds.rs index 6d6451a1..c5c18ffe 100644 --- a/ml/neural/src/layers/kinds.rs +++ b/ml/neural/src/layers/kinds.rs @@ -32,3 +32,20 @@ pub enum LayerType { Hidden(usize), Output, } + +pub struct Position { + pub index: usize, + pub kind: LayerType, +} + +impl Position { + pub fn new(index: usize, kind: LayerType) -> Self { + Self { index, kind } + } + pub fn new_input() -> Self { + Self { + index: 0, + kind: LayerType::Input, + } + } +} diff --git a/ml/neural/src/lib.rs b/ml/neural/src/lib.rs index 84c3f9db..8a8efe17 100644 --- a/ml/neural/src/lib.rs +++ b/ml/neural/src/lib.rs @@ -9,6 +9,7 @@ pub(crate) mod primitives; pub(crate) mod specs; pub(crate) mod utils; +pub mod arch; pub mod layers; pub mod neurons; pub mod nn; diff --git a/ml/neural/src/neurons/activate/common/binary.rs b/ml/neural/src/neurons/activate/common/binary.rs new file mode 100644 index 00000000..d6e3f99a --- /dev/null +++ b/ml/neural/src/neurons/activate/common/binary.rs @@ -0,0 +1,20 @@ +/* + Appellation: binary + Contrib: FL03 +*/ +use crate::neurons::activate::Activator; + +pub struct Heavyside; + +impl Activator for Heavyside +where + T: num::Float, +{ + fn rho(x: T) -> T { + if x > T::zero() { + T::one() + } else { + T::zero() + } + } +} diff --git a/ml/neural/src/neurons/activate/common/linear.rs b/ml/neural/src/neurons/activate/common/linear.rs new file mode 100644 index 00000000..50b9b5c5 --- /dev/null +++ b/ml/neural/src/neurons/activate/common/linear.rs @@ -0,0 +1,4 @@ +/* + Appellation: linear + Contrib: FL03 +*/ diff --git a/ml/neural/src/neurons/activate/common/mod.rs b/ml/neural/src/neurons/activate/common/mod.rs new file mode 100644 index 00000000..bee8ddd7 --- /dev/null +++ b/ml/neural/src/neurons/activate/common/mod.rs @@ -0,0 +1,12 @@ +/* + Appellation: common + Contrib: FL03 +*/ +//! # common +pub use self::{binary::*, linear::*, nonlinear::*, nonlinear::*, utils::*}; + +pub(crate) mod binary; +pub(crate) mod linear; +pub(crate) mod nonlinear; + +pub(crate) mod utils {} diff --git a/ml/neural/src/neurons/activate/softmax.rs b/ml/neural/src/neurons/activate/common/nonlinear.rs similarity index 57% rename from ml/neural/src/neurons/activate/softmax.rs rename to ml/neural/src/neurons/activate/common/nonlinear.rs index 587e0182..e3fe5b69 100644 --- a/ml/neural/src/neurons/activate/softmax.rs +++ b/ml/neural/src/neurons/activate/common/nonlinear.rs @@ -1,5 +1,5 @@ /* - Appellation: softmax + Appellation: nonlinear Contrib: FL03 */ use ndarray::prelude::Array1; @@ -8,12 +8,26 @@ pub fn softmax(args: Array1) -> Array1 where T: num::Float, { - let mut res = Array1::zeros(args.len()); let denom = args.mapv(|x| x.exp()).sum(); - for (i, x) in args.iter().enumerate() { - res[i] = x.exp() / denom; + args.mapv(|x| x.exp() / denom) +} + +pub struct Softmax { + args: Array1, +} + +impl Softmax { + pub fn new(args: Array1) -> Self { + Self { args } + } + + pub(crate) fn denom(&self) -> f64 { + self.args.mapv(|x| x.exp()).sum() + } + + pub fn compute(&self) -> Array1 { + self.args.mapv(|x| x.exp() / self.denom()) } - res } #[cfg(test)] diff --git a/ml/neural/src/neurons/activate/mod.rs b/ml/neural/src/neurons/activate/mod.rs index c212589c..070a1c67 100644 --- a/ml/neural/src/neurons/activate/mod.rs +++ b/ml/neural/src/neurons/activate/mod.rs @@ -5,27 +5,26 @@ //! # activate //! //! This module contains the activation functions for the neurons. -pub use self::{softmax::*, utils::*}; +pub use self::{common::*, utils::*}; -pub(crate) mod softmax; +pub(crate) mod common; pub type ActivationFn = fn(T) -> T; -pub trait Activate { - fn activate(&mut self) -> T; +pub trait Activable { + fn activate(&self, args: &ndarray::Array1) -> T; } -pub trait ActivateWith { - fn activate_with(&mut self, args: &[T]) -> T; +pub trait ActivateMethod { + fn activate(&self, x: T) -> T; } -pub(crate) mod utils { - - pub fn heavyside(x: f64) -> f64 { - if x > 0.0 { - 1.0 - } else { - 0.0 - } +pub trait Activator { + fn activate(&self, x: T) -> T { + Self::rho(x) } + + fn rho(x: T) -> T; } + +pub(crate) mod utils {} diff --git a/ml/neural/src/neurons/neuron.rs b/ml/neural/src/neurons/neuron.rs index d42ff4a6..4a9786b6 100644 --- a/ml/neural/src/neurons/neuron.rs +++ b/ml/neural/src/neurons/neuron.rs @@ -2,7 +2,7 @@ Appellation: neuron Contrib: FL03 */ -use super::activate::ActivationFn; +use super::activate::{ActivationFn, Activator}; use ndarray::prelude::Array1; /// Artificial Neuron @@ -27,14 +27,7 @@ impl Neuron { } pub fn compute(&self, args: &Array1) -> f64 { - let dot = args.dot(&self.weights); - self.rho()(dot - self.bias()) - } - - pub fn process(&self, args: impl AsRef<[f64]>) -> f64 { - let data = Array1::from(args.as_ref().to_vec()); - let dot = data.dot(&self.weights); - self.rho()(dot - self.bias()) + self.rho()(args.dot(&self.weights) - self.bias()) } pub fn rho(&self) -> ActivationFn { @@ -57,37 +50,36 @@ impl Neuron { #[cfg(test)] mod tests { use super::*; - use crate::neurons::activate::heavyside; + use crate::neurons::activate::{Activator, Heavyside}; use ndarray::array; fn _artificial( - args: &[f64], + args: &Array1, bias: Option, - rho: ActivationFn, + rho: impl Activator, weights: &Array1, ) -> f64 { - let data = Array1::from(args.to_vec()); - rho(data.dot(weights) - bias.unwrap_or_default()) + rho.activate(args.dot(weights) - bias.unwrap_or_default()) } #[test] fn test_neuron() { let bias = 0.0; - let a_data = [10.0, 10.0, 6.0, 1.0, 8.0]; + let a_data = array![10.0, 10.0, 6.0, 1.0, 8.0]; let a_weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; - let a = Neuron::new(heavyside, bias, a_weights.clone()); + let a = Neuron::new(Heavyside::rho, bias, a_weights.clone()); - let exp = _artificial(&a_data, Some(bias), heavyside, &a_weights); - assert_eq!(a.process(&a_data), exp); + let exp = _artificial(&a_data, Some(bias), Heavyside, &a_weights); + assert_eq!(a.compute(&a_data), exp); - let b_data = [0.0, 9.0, 3.0, 5.0, 3.0]; + let b_data = array![0.0, 9.0, 3.0, 5.0, 3.0]; let b_weights = array![2.0, 8.0, 8.0, 0.0, 3.0]; - let b = Neuron::new(heavyside, bias, b_weights.clone()); + let b = Neuron::new(Heavyside::rho, bias, b_weights.clone()); - let exp = _artificial(&b_data, Some(bias), heavyside, &b_weights); - assert_eq!(b.process(&b_data), exp); + let exp = _artificial(&b_data, Some(bias), Heavyside, &b_weights); + assert_eq!(b.compute(&b_data), exp); // assert_eq!(a.dot() + b.dot(), 252.0); } diff --git a/ml/neural/src/neurons/node.rs b/ml/neural/src/neurons/node.rs index e0127a65..4b34f364 100644 --- a/ml/neural/src/neurons/node.rs +++ b/ml/neural/src/neurons/node.rs @@ -50,37 +50,36 @@ impl Node { #[cfg(test)] mod tests { use super::*; - use crate::neurons::activate::{heavyside, ActivationFn}; + use crate::neurons::activate::{Activator, Heavyside}; use ndarray::array; fn _artificial( - args: &[f64], + args: &Array1, bias: Option, - rho: ActivationFn, + rho: impl Activator, weights: &Array1, ) -> f64 { - let data = Array1::from(args.to_vec()); - rho(data.dot(weights) - bias.unwrap_or_default()) + rho.activate(args.dot(weights) - bias.unwrap_or_default()) } #[test] fn test_node() { let bias = 0.0; - let a_data = [10.0, 10.0, 6.0, 1.0, 8.0]; + let a_data = array![10.0, 10.0, 6.0, 1.0, 8.0]; let a_weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; - let a = Neuron::new(heavyside, bias, a_weights.clone()); - let node_a = Node::new(a.clone()).with_data(Array1::from(a_data.to_vec())); + let a = Neuron::new(Heavyside::rho, bias, a_weights.clone()); + let node_a = Node::new(a.clone()).with_data(a_data.clone()); - let exp = _artificial(&a_data, Some(bias), heavyside, &a_weights); + let exp = _artificial(&a_data, Some(bias), Heavyside, &a_weights); assert_eq!(node_a.process(), exp); - let b_data = [0.0, 9.0, 3.0, 5.0, 3.0]; + let b_data = array![0.0, 9.0, 3.0, 5.0, 3.0]; let b_weights = array![2.0, 8.0, 8.0, 0.0, 3.0]; - let b = Neuron::new(heavyside, bias, b_weights.clone()); - let node_b = Node::new(b.clone()).with_data(Array1::from(b_data.to_vec())); - let exp = _artificial(&b_data, Some(bias), heavyside, &b_weights); + let b = Neuron::new(Heavyside::rho, bias, b_weights.clone()); + let node_b = Node::new(b.clone()).with_data(b_data.clone()); + let exp = _artificial(&b_data, Some(bias), Heavyside, &b_weights); assert_eq!(node_b.process(), exp); assert_eq!(node_a.dot() + node_b.dot(), 252.0); diff --git a/ml/neural/src/nn/loss/regress.rs b/ml/neural/src/nn/loss/regress.rs index 86defa8f..00d7f393 100644 --- a/ml/neural/src/nn/loss/regress.rs +++ b/ml/neural/src/nn/loss/regress.rs @@ -45,3 +45,29 @@ impl Loss for HuberLoss { loss / pred.len() as f64 } } + +pub struct MeanAbsoluteError; + +impl Loss for MeanAbsoluteError { + fn loss(&self, pred: &[f64], target: &[f64]) -> f64 { + let mut res = 0.0; + for (p, t) in pred.iter().zip(target.iter()) { + res += (p - t).abs(); + } + res /= pred.len() as f64; + res + } +} + +pub struct MeanSquaredError; + +impl Loss for MeanSquaredError { + fn loss(&self, pred: &[f64], target: &[f64]) -> f64 { + let mut res = 0.0; + for (p, t) in pred.iter().zip(target.iter()) { + res += (p - t).powi(2); + } + res /= pred.len() as f64; + res + } +} diff --git a/ml/neural/src/nn/mod.rs b/ml/neural/src/nn/mod.rs index fceb374f..e4c4d54d 100644 --- a/ml/neural/src/nn/mod.rs +++ b/ml/neural/src/nn/mod.rs @@ -3,8 +3,21 @@ Contrib: FL03 */ //! # Neural Network -pub use self::utils::*; +pub use self::{network::*, utils::*}; pub mod loss; +pub(crate) mod network; + +use crate::layers::Layer; +use crate::Trainable; + +pub trait NeuralNet: Trainable { + fn depth(&self) -> usize { + self.layers().len() + } + + fn layers(&self) -> &[Layer]; +} + pub(crate) mod utils {} diff --git a/ml/neural/src/nn/network.rs b/ml/neural/src/nn/network.rs new file mode 100644 index 00000000..dd062ecf --- /dev/null +++ b/ml/neural/src/nn/network.rs @@ -0,0 +1,9 @@ +/* + Appellation: network + Contrib: FL03 +*/ +use crate::layers::Layer; + +pub struct NeuralNetwork { + pub(crate) layers: Vec, +} diff --git a/ml/neural/src/prop/kinds.rs b/ml/neural/src/prop/kinds.rs new file mode 100644 index 00000000..26970c47 --- /dev/null +++ b/ml/neural/src/prop/kinds.rs @@ -0,0 +1,9 @@ +/* + Appellation: kinds + Contrib: FL03 +*/ + +pub enum PropagationKind { + Forward, + Backward, +} diff --git a/ml/neural/src/prop/mod.rs b/ml/neural/src/prop/mod.rs index 29340655..5a5148a6 100644 --- a/ml/neural/src/prop/mod.rs +++ b/ml/neural/src/prop/mod.rs @@ -5,6 +5,9 @@ //! # Propagation //! //! This module describes the propagation of data through a neural network. -pub use self::utils::*; +pub use self::{kinds::*, propagation::*, utils::*}; + +pub(crate) mod kinds; +pub(crate) mod propagation; pub(crate) mod utils {} diff --git a/ml/neural/src/prop/propagation.rs b/ml/neural/src/prop/propagation.rs new file mode 100644 index 00000000..fb65f594 --- /dev/null +++ b/ml/neural/src/prop/propagation.rs @@ -0,0 +1,4 @@ +/* + Appellation: propagation + Contrib: FL03 +*/ diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index 7b91c0ef..dcb9462a 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -3,8 +3,6 @@ Contrib: FL03 */ -pub trait NeuralNetwork: Trainable {} - pub trait Trainable { fn train(&mut self, args: &[f64]) -> f64; } diff --git a/ml/transformers/src/codec/encode/embed.rs b/ml/transformers/src/codec/encode/embed.rs index 7219df08..2b0e2e5b 100644 --- a/ml/transformers/src/codec/encode/embed.rs +++ b/ml/transformers/src/codec/encode/embed.rs @@ -7,8 +7,23 @@ pub trait Embed {} #[derive(Clone)] pub struct Embedding { - pub(crate) embedding: Vec, - pub(crate) position: Vec, + pub(crate) embedding: Vec, + pub(crate) position: Vec, } +impl Embedding { + pub fn new(embedding: Vec, position: Vec) -> Self { + Self { + embedding, + position, + } + } + pub fn embedding(&self) -> &Vec { + &self.embedding + } + + pub fn position(&self) -> &Vec { + &self.position + } +} From e545b38129b02923ff2f4316bc0bb29860253bab Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 21 Oct 2023 12:52:41 -0500 Subject: [PATCH 025/118] update Signed-off-by: FL03 --- ml/transformers/Cargo.toml | 1 + ml/transformers/src/attention/head.rs | 21 ++++++++++++--------- ml/transformers/src/lib.rs | 2 ++ ml/transformers/src/primitives.rs | 4 ++++ 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/ml/transformers/Cargo.toml b/ml/transformers/Cargo.toml index 138e8529..c7848202 100644 --- a/ml/transformers/Cargo.toml +++ b/ml/transformers/Cargo.toml @@ -24,6 +24,7 @@ test = true [build-dependencies] [dependencies] +concision-neural = { path = "../neural" } anyhow.workspace = true ndarray.workspace = true num.workspace = true diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index ca707eec..6016f42b 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -4,22 +4,25 @@ */ use serde::{Deserialize, Serialize}; +#[derive(Clone, Debug, Default, Deserialize, PartialEq, PartialOrd, Serialize)] +pub struct Weights { + key: Vec, + query: Vec, + value: Vec, +} + #[derive(Clone, Debug, Default, Deserialize, PartialEq, PartialOrd, Serialize)] #[serde(rename_all = "lowercase")] pub struct AttentionHead { - keys: Vec, - queries: Vec, - values: Vec, - pos: usize, + data: Vec, + dim: usize, } impl AttentionHead { - pub fn new() -> Self { + pub fn new(dim: usize) -> Self { Self { - keys: Vec::new(), - queries: Vec::new(), - values: Vec::new(), - pos: 0, + data: Vec::new(), + dim, } } } diff --git a/ml/transformers/src/lib.rs b/ml/transformers/src/lib.rs index 0d0c79a6..c2335679 100644 --- a/ml/transformers/src/lib.rs +++ b/ml/transformers/src/lib.rs @@ -13,6 +13,8 @@ pub mod attention; pub mod codec; pub mod transform; +pub(crate) use concision_neural as neural; + pub mod prelude { pub use crate::attention::*; pub use crate::codec::*; diff --git a/ml/transformers/src/primitives.rs b/ml/transformers/src/primitives.rs index 657a296c..b7131c91 100644 --- a/ml/transformers/src/primitives.rs +++ b/ml/transformers/src/primitives.rs @@ -7,6 +7,10 @@ pub use self::{constants::*, statics::*, types::*}; /// Collection of constants used throughout the system pub(crate) mod constants { pub const DEFAULT_EMBEDDING_SIZE: usize = 512; + + pub const DEFAULT_ATTENTION_HEADS: usize = 8; + + pub const DEFAULT_ATTENTION_HEAD_SIZE: usize = 64; } /// Collection of static references used throughout From 6f694bbc6ec8f96213b08eefc5b46a33f5df236f Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 21 Oct 2023 14:54:55 -0500 Subject: [PATCH 026/118] update Signed-off-by: FL03 --- ml/transformers/src/attention/head.rs | 6 ++++++ ml/transformers/src/codec/encode/mod.rs | 25 +++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index 6016f42b..5df51fe1 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -11,6 +11,12 @@ pub struct Weights { value: Vec, } +pub struct AttentionParams { + pub(crate) depth: usize, // embedding size + pub(crate) heads: usize, // number of attention heads + pub(crate) dropout: f64, +} + #[derive(Clone, Debug, Default, Deserialize, PartialEq, PartialOrd, Serialize)] #[serde(rename_all = "lowercase")] pub struct AttentionHead { diff --git a/ml/transformers/src/codec/encode/mod.rs b/ml/transformers/src/codec/encode/mod.rs index d2a78b58..0f717c9f 100644 --- a/ml/transformers/src/codec/encode/mod.rs +++ b/ml/transformers/src/codec/encode/mod.rs @@ -10,12 +10,33 @@ pub(crate) mod encoder; pub trait Encode {} -pub(crate) mod utils {} +pub(crate) mod utils { + use ndarray::Array2; + + pub fn get_position_encoding(seq_len: usize, d: usize, n: f64) -> Array2 { + let mut p = Array2::zeros((seq_len, d)); + for k in 0..seq_len { + for i in 0..d / 2 { + let denominator = f64::powf(n, 2.0 * i as f64 / d as f64); + p[[k, 2 * i]] = (k as f64 / denominator).sin(); + p[[k, 2 * i + 1]] = (k as f64 / denominator).cos(); + } + } + p + } +} #[cfg(test)] mod tests { - // use super::*; + use super::*; + use ndarray::array; #[test] fn test_encoder() {} + + #[test] + fn test_positional_encoding() { + let p = get_position_encoding(4, 4, 1000.); + assert_eq!(p.row(0), array![0.0, 1.0, 0.0, 1.0]); + } } From 5aab96f587ee982d05a629c7463c02636527e776 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 22 Oct 2023 02:14:09 -0500 Subject: [PATCH 027/118] update Signed-off-by: FL03 --- concision/Cargo.toml | 9 ++- concision/src/lib.rs | 12 ++- ml/neural/src/layers/layer.rs | 20 +++++ .../neurons/activate/{common => }/binary.rs | 0 ml/neural/src/neurons/activate/common/mod.rs | 12 --- .../src/neurons/activate/common/nonlinear.rs | 44 ----------- ml/neural/src/neurons/activate/mod.rs | 7 +- ml/neural/src/neurons/activate/nonlinear.rs | 63 +++++++++++++++ ml/neural/src/neurons/mod.rs | 63 +++++++++++++++ ml/neural/src/neurons/neuron.rs | 60 ++++----------- ml/neural/src/neurons/node.rs | 41 +--------- ml/neural/src/nn/network.rs | 2 +- ml/nlp/Cargo.toml | 43 +++++++++++ ml/nlp/benches/default.rs | 52 +++++++++++++ ml/nlp/src/embed/embedding.rs | 21 +++++ ml/nlp/src/embed/mod.rs | 18 +++++ ml/nlp/src/lib.rs | 20 +++++ ml/nlp/src/primitives.rs | 11 +++ .../common/linear.rs => nlp/src/specs.rs} | 2 +- ml/nlp/src/utils.rs | 4 + ml/nlp/tests/default.rs | 8 ++ ml/transformers/src/attention/head.rs | 40 ++++++++-- ml/transformers/src/attention/mod.rs | 2 + ml/transformers/src/attention/weights.rs | 77 +++++++++++++++++++ ml/transformers/src/codec/encode/embed.rs | 29 ------- ml/transformers/src/codec/encode/mod.rs | 3 +- 26 files changed, 471 insertions(+), 192 deletions(-) rename ml/neural/src/neurons/activate/{common => }/binary.rs (100%) delete mode 100644 ml/neural/src/neurons/activate/common/mod.rs delete mode 100644 ml/neural/src/neurons/activate/common/nonlinear.rs create mode 100644 ml/neural/src/neurons/activate/nonlinear.rs create mode 100644 ml/nlp/Cargo.toml create mode 100644 ml/nlp/benches/default.rs create mode 100644 ml/nlp/src/embed/embedding.rs create mode 100644 ml/nlp/src/embed/mod.rs create mode 100644 ml/nlp/src/lib.rs create mode 100644 ml/nlp/src/primitives.rs rename ml/{neural/src/neurons/activate/common/linear.rs => nlp/src/specs.rs} (59%) create mode 100644 ml/nlp/src/utils.rs create mode 100644 ml/nlp/tests/default.rs create mode 100644 ml/transformers/src/attention/weights.rs delete mode 100644 ml/transformers/src/codec/encode/embed.rs diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 570e7ed5..ca65a13c 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -44,13 +44,18 @@ macros = [ ml = [ "neural", - "transformers" + "nlp", + "transformers", ] neural = [ "concision-neural" ] +nlp = [ + "concision-nlp" +] + transformers = [ "concision-transformers" ] @@ -68,7 +73,9 @@ concision-core = { features = [], optional = true, path = "../core", version = " concision-data = { features = [], optional = true, path = "../data", version = "0.1.12" } concision-derive = { features = [], optional = true, path = "../derive", version = "0.1.12" } concision-macros = { features = [], optional = true, path = "../macros", version = "0.1.12" } + concision-neural = { features = [], optional = true, path = "../ml/neural", version = "0.1.12" } +concision-nlp = { features = [], optional = true, path = "../ml/nlp", version = "0.1.12" } concision-transformers = { features = [], optional = true, path = "../ml/transformers", version = "0.1.12" } [dev-dependencies] diff --git a/concision/src/lib.rs b/concision/src/lib.rs index 1c8ffebc..f5113520 100644 --- a/concision/src/lib.rs +++ b/concision/src/lib.rs @@ -15,8 +15,10 @@ pub use concision_data as data; pub use concision_derive::*; #[cfg(feature = "macros")] pub use concision_macros::*; -#[cfg(feature = "nn")] -pub use concision_nn as nn; +#[cfg(feature = "neural")] +pub use concision_neural as neural; +#[cfg(feature = "nlp")] +pub use concision_nlp as nlp; #[cfg(feature = "transformers")] pub use concision_transformers as transformers; @@ -29,8 +31,10 @@ pub mod prelude { pub use concision_derive::*; #[cfg(feature = "macros")] pub use concision_macros::*; - #[cfg(feature = "nn")] - pub use concision_nn::prelude::*; + #[cfg(feature = "neural")] + pub use concision_neural::prelude::*; + #[cfg(feature = "nlp")] + pub use concision_nlp::prelude::*; #[cfg(feature = "transformers")] pub use concision_transformers::prelude::*; } diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 6ddc5b8f..ada2986c 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -3,9 +3,29 @@ Contrib: FL03 */ use super::LayerType; +use crate::neurons::activate::Activator; use crate::neurons::Node; +use ndarray::Array2; use std::ops; +pub trait L +where + T: num::Float + 'static, +{ + type Activator: Activator; + + fn bias(&self) -> &Array2; + + fn weights(&self) -> &Array2; + + fn activator(&self) -> &Self::Activator; + + fn activate(&self, args: &Array2) -> Array2 { + let z = args.dot(self.weights()) + self.bias(); + z.mapv(|x| self.activator().activate(x)) + } +} + #[derive(Clone, Debug, Default, PartialEq)] pub struct Layer { layer: LayerType, diff --git a/ml/neural/src/neurons/activate/common/binary.rs b/ml/neural/src/neurons/activate/binary.rs similarity index 100% rename from ml/neural/src/neurons/activate/common/binary.rs rename to ml/neural/src/neurons/activate/binary.rs diff --git a/ml/neural/src/neurons/activate/common/mod.rs b/ml/neural/src/neurons/activate/common/mod.rs deleted file mode 100644 index bee8ddd7..00000000 --- a/ml/neural/src/neurons/activate/common/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -/* - Appellation: common - Contrib: FL03 -*/ -//! # common -pub use self::{binary::*, linear::*, nonlinear::*, nonlinear::*, utils::*}; - -pub(crate) mod binary; -pub(crate) mod linear; -pub(crate) mod nonlinear; - -pub(crate) mod utils {} diff --git a/ml/neural/src/neurons/activate/common/nonlinear.rs b/ml/neural/src/neurons/activate/common/nonlinear.rs deleted file mode 100644 index e3fe5b69..00000000 --- a/ml/neural/src/neurons/activate/common/nonlinear.rs +++ /dev/null @@ -1,44 +0,0 @@ -/* - Appellation: nonlinear - Contrib: FL03 -*/ -use ndarray::prelude::Array1; - -pub fn softmax(args: Array1) -> Array1 -where - T: num::Float, -{ - let denom = args.mapv(|x| x.exp()).sum(); - args.mapv(|x| x.exp() / denom) -} - -pub struct Softmax { - args: Array1, -} - -impl Softmax { - pub fn new(args: Array1) -> Self { - Self { args } - } - - pub(crate) fn denom(&self) -> f64 { - self.args.mapv(|x| x.exp()).sum() - } - - pub fn compute(&self) -> Array1 { - self.args.mapv(|x| x.exp() / self.denom()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use computare::prelude::RoundTo; - - #[test] - fn test_softmax() { - let args = Array1::from(vec![1.0, 2.0, 3.0]); - let res = softmax(args).mapv(|i| i.round_to(8)); - assert_eq!(res, Array1::from(vec![0.09003057, 0.24472847, 0.66524096])); - } -} diff --git a/ml/neural/src/neurons/activate/mod.rs b/ml/neural/src/neurons/activate/mod.rs index 070a1c67..300b4ae9 100644 --- a/ml/neural/src/neurons/activate/mod.rs +++ b/ml/neural/src/neurons/activate/mod.rs @@ -5,14 +5,15 @@ //! # activate //! //! This module contains the activation functions for the neurons. -pub use self::{common::*, utils::*}; +pub use self::{binary::*, nonlinear::*, utils::*}; -pub(crate) mod common; +pub(crate) mod binary; +pub(crate) mod nonlinear; pub type ActivationFn = fn(T) -> T; pub trait Activable { - fn activate(&self, args: &ndarray::Array1) -> T; + fn activate(&self, rho: impl Activator) -> T; } pub trait ActivateMethod { diff --git a/ml/neural/src/neurons/activate/nonlinear.rs b/ml/neural/src/neurons/activate/nonlinear.rs new file mode 100644 index 00000000..a5dd7193 --- /dev/null +++ b/ml/neural/src/neurons/activate/nonlinear.rs @@ -0,0 +1,63 @@ +/* + Appellation: nonlinear + Contrib: FL03 +*/ +use super::Activator; +use ndarray::prelude::Array1; + +pub fn softmax(args: Array1) -> Array1 +where + T: num::Float, +{ + let denom = args.mapv(|x| x.exp()).sum(); + args.mapv(|x| x.exp() / denom) +} + +pub struct Sigmoid; + +impl Sigmoid { + pub fn compute(x: f64) -> f64 { + 1.0 / (1.0 + (-x).exp()) + } +} + +impl Activator for Sigmoid +where + T: num::Float, +{ + fn rho(x: T) -> T { + T::one() / (T::one() + (-x).exp()) + } +} + +pub struct Softmax; + +impl Softmax { + pub fn new() -> Self { + Self + } +} + +impl Activator> for Softmax +where + T: num::Float, +{ + fn rho(x: Array1) -> Array1 { + let denom = x.mapv(|x| x.exp()).sum(); + x.mapv(|x| x.exp() / denom) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use computare::prelude::RoundTo; + + #[test] + fn test_softmax() { + let exp = Array1::from(vec![0.09003057, 0.24472847, 0.66524096]); + let args = Array1::from(vec![1.0, 2.0, 3.0]); + let res = Softmax::rho(args).mapv(|i| i.round_to(8)); + assert_eq!(res, exp); + } +} diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index 2818ca7e..fe89bbee 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -13,3 +13,66 @@ pub mod activate; pub trait Weight {} pub(crate) mod utils {} + +#[cfg(test)] +mod tests { + use super::activate::{Activator, Softmax}; + use super::*; + use ndarray::{array, Array1}; + + fn _artificial( + args: &Array1, + bias: Option>, + rho: impl Activator>, + weights: &Array1, + ) -> Array1 { + rho.activate( + args.dot(weights) - bias.unwrap_or_else(|| Array1::::zeros(args.shape()[0])), + ) + } + + #[test] + fn test_neuron() { + let bias = ndarray::Array1::::zeros(4); + + let a_data = array![10.0, 10.0, 6.0, 1.0, 8.0]; + let a_weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; + let a = Neuron::new(Softmax::rho, bias.clone(), a_weights.clone()); + + let exp = _artificial(&a_data, Some(bias.clone()), Softmax, &a_weights); + assert_eq!(a.compute(&a_data), exp); + + let b_data = array![0.0, 9.0, 3.0, 5.0, 3.0]; + let b_weights = array![2.0, 8.0, 8.0, 0.0, 3.0]; + + let b = Neuron::new(Softmax::rho, bias.clone(), b_weights.clone()); + + let exp = _artificial(&b_data, Some(bias), Softmax, &b_weights); + assert_eq!(b.compute(&b_data), exp); + + // assert_eq!(a.dot() + b.dot(), 252.0); + } + + #[test] + fn test_node() { + let bias = ndarray::Array1::::zeros(4); + + let a_data = array![10.0, 10.0, 6.0, 1.0, 8.0]; + let a_weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; + let a = Neuron::new(Softmax::rho, bias.clone(), a_weights.clone()); + let node_a = Node::new(a.clone()).with_data(a_data.clone()); + + let exp = _artificial(&a_data, Some(bias.clone()), Softmax, &a_weights); + assert_eq!(node_a.process(), exp); + + let b_data = array![0.0, 9.0, 3.0, 5.0, 3.0]; + let b_weights = array![2.0, 8.0, 8.0, 0.0, 3.0]; + + let b = Neuron::new(Softmax::rho, bias.clone(), b_weights.clone()); + let node_b = Node::new(b.clone()).with_data(b_data.clone()); + let exp = _artificial(&b_data, Some(bias), Softmax, &b_weights); + assert_eq!(node_b.process(), exp); + + assert_eq!(node_a.dot() + node_b.dot(), 252.0); + } +} diff --git a/ml/neural/src/neurons/neuron.rs b/ml/neural/src/neurons/neuron.rs index 4a9786b6..e620e68f 100644 --- a/ml/neural/src/neurons/neuron.rs +++ b/ml/neural/src/neurons/neuron.rs @@ -2,19 +2,23 @@ Appellation: neuron Contrib: FL03 */ -use super::activate::{ActivationFn, Activator}; +use super::activate::ActivationFn; use ndarray::prelude::Array1; /// Artificial Neuron #[derive(Clone, Debug, PartialEq)] pub struct Neuron { - activation: ActivationFn, - bias: f64, + activation: ActivationFn>, + bias: Array1, weights: Array1, } impl Neuron { - pub fn new(activation: ActivationFn, bias: f64, weights: Array1) -> Self { + pub fn new( + activation: ActivationFn>, + bias: Array1, + weights: Array1, + ) -> Self { Self { activation, bias, @@ -22,15 +26,15 @@ impl Neuron { } } - pub fn bias(&self) -> f64 { - self.bias + pub fn bias(&self) -> &Array1 { + &self.bias } - pub fn compute(&self, args: &Array1) -> f64 { + pub fn compute(&self, args: &Array1) -> Array1 { self.rho()(args.dot(&self.weights) - self.bias()) } - pub fn rho(&self) -> ActivationFn { + pub fn rho(&self) -> ActivationFn> { self.activation } @@ -38,7 +42,7 @@ impl Neuron { &self.weights } - pub fn set_bias(&mut self, bias: f64) { + pub fn set_bias(&mut self, bias: Array1) { self.bias = bias; } @@ -46,41 +50,3 @@ impl Neuron { self.weights = weights; } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::neurons::activate::{Activator, Heavyside}; - use ndarray::array; - - fn _artificial( - args: &Array1, - bias: Option, - rho: impl Activator, - weights: &Array1, - ) -> f64 { - rho.activate(args.dot(weights) - bias.unwrap_or_default()) - } - - #[test] - fn test_neuron() { - let bias = 0.0; - - let a_data = array![10.0, 10.0, 6.0, 1.0, 8.0]; - let a_weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; - let a = Neuron::new(Heavyside::rho, bias, a_weights.clone()); - - let exp = _artificial(&a_data, Some(bias), Heavyside, &a_weights); - assert_eq!(a.compute(&a_data), exp); - - let b_data = array![0.0, 9.0, 3.0, 5.0, 3.0]; - let b_weights = array![2.0, 8.0, 8.0, 0.0, 3.0]; - - let b = Neuron::new(Heavyside::rho, bias, b_weights.clone()); - - let exp = _artificial(&b_data, Some(bias), Heavyside, &b_weights); - assert_eq!(b.compute(&b_data), exp); - - // assert_eq!(a.dot() + b.dot(), 252.0); - } -} diff --git a/ml/neural/src/neurons/node.rs b/ml/neural/src/neurons/node.rs index 4b34f364..5dcb3bf3 100644 --- a/ml/neural/src/neurons/node.rs +++ b/ml/neural/src/neurons/node.rs @@ -33,7 +33,7 @@ impl Node { &self.neuron } - pub fn process(&self) -> f64 { + pub fn process(&self) -> Array1 { self.neuron.compute(&self.data) } @@ -46,42 +46,3 @@ impl Node { self } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::neurons::activate::{Activator, Heavyside}; - use ndarray::array; - - fn _artificial( - args: &Array1, - bias: Option, - rho: impl Activator, - weights: &Array1, - ) -> f64 { - rho.activate(args.dot(weights) - bias.unwrap_or_default()) - } - - #[test] - fn test_node() { - let bias = 0.0; - - let a_data = array![10.0, 10.0, 6.0, 1.0, 8.0]; - let a_weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; - let a = Neuron::new(Heavyside::rho, bias, a_weights.clone()); - let node_a = Node::new(a.clone()).with_data(a_data.clone()); - - let exp = _artificial(&a_data, Some(bias), Heavyside, &a_weights); - assert_eq!(node_a.process(), exp); - - let b_data = array![0.0, 9.0, 3.0, 5.0, 3.0]; - let b_weights = array![2.0, 8.0, 8.0, 0.0, 3.0]; - - let b = Neuron::new(Heavyside::rho, bias, b_weights.clone()); - let node_b = Node::new(b.clone()).with_data(b_data.clone()); - let exp = _artificial(&b_data, Some(bias), Heavyside, &b_weights); - assert_eq!(node_b.process(), exp); - - assert_eq!(node_a.dot() + node_b.dot(), 252.0); - } -} diff --git a/ml/neural/src/nn/network.rs b/ml/neural/src/nn/network.rs index dd062ecf..29c1c417 100644 --- a/ml/neural/src/nn/network.rs +++ b/ml/neural/src/nn/network.rs @@ -5,5 +5,5 @@ use crate::layers::Layer; pub struct NeuralNetwork { - pub(crate) layers: Vec, + pub layers: Vec, } diff --git a/ml/nlp/Cargo.toml b/ml/nlp/Cargo.toml new file mode 100644 index 00000000..37433b6a --- /dev/null +++ b/ml/nlp/Cargo.toml @@ -0,0 +1,43 @@ +[package] +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "concision-nlp" +readme.workspace = true +repository.workspace = true +version.workspace = true + +[features] +default = [] + + +[lib] +bench = true +crate-type = ["cdylib", "rlib"] +doctest = false +test = true + +[build-dependencies] + +[dependencies] +anyhow.workspace = true +ndarray.workspace = true +num.workspace = true +serde.workspace = true +serde_json.workspace = true +smart-default.workspace = true +strum.workspace = true + +[dev-dependencies] + +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] + +[target.wasm32-unknown-unknown] + +[target.wasm32-wasi] diff --git a/ml/nlp/benches/default.rs b/ml/nlp/benches/default.rs new file mode 100644 index 00000000..937f2387 --- /dev/null +++ b/ml/nlp/benches/default.rs @@ -0,0 +1,52 @@ +// bench.rs +#![feature(test)] + +extern crate test; + +use std::mem::replace; +use test::Bencher; + +// bench: find the `BENCH_SIZE` first terms of the fibonacci sequence +static BENCH_SIZE: usize = 20; + +// recursive fibonacci +fn fibonacci(n: usize) -> u32 { + if n < 2 { + 1 + } else { + fibonacci(n - 1) + fibonacci(n - 2) + } +} + +// iterative fibonacci +struct Fibonacci { + curr: u32, + next: u32, +} + +impl Iterator for Fibonacci { + type Item = u32; + fn next(&mut self) -> Option { + let new_next = self.curr + self.next; + let new_curr = replace(&mut self.next, new_next); + + Some(replace(&mut self.curr, new_curr)) + } +} + +fn fibonacci_sequence() -> Fibonacci { + Fibonacci { curr: 1, next: 1 } +} + +// function to benchmark must be annotated with `#[bench]` +#[bench] +fn recursive_fibonacci(b: &mut Bencher) { + // exact code to benchmark must be passed as a closure to the iter + // method of Bencher + b.iter(|| (0..BENCH_SIZE).map(fibonacci).collect::>()) +} + +#[bench] +fn iterative_fibonacci(b: &mut Bencher) { + b.iter(|| fibonacci_sequence().take(BENCH_SIZE).collect::>()) +} diff --git a/ml/nlp/src/embed/embedding.rs b/ml/nlp/src/embed/embedding.rs new file mode 100644 index 00000000..d6ba8071 --- /dev/null +++ b/ml/nlp/src/embed/embedding.rs @@ -0,0 +1,21 @@ +/* + Appellation: embedding + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "lowercase")] +pub struct Embedding; + +impl Embedding { + pub fn new() -> Self { + Self + } +} + +impl std::fmt::Display for Embedding { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", serde_json::to_string(self).unwrap()) + } +} diff --git a/ml/nlp/src/embed/mod.rs b/ml/nlp/src/embed/mod.rs new file mode 100644 index 00000000..1576c35b --- /dev/null +++ b/ml/nlp/src/embed/mod.rs @@ -0,0 +1,18 @@ +/* + Appellation: embed + Contrib: FL03 +*/ +//! # Embedding +//! +//! Natural language processing applications rely heavily on the use of embeddings. +//! Embeddings are generally broken down into two categories: contextual and static. +//! Popular static solutions such as word2vec and GloVe are used to create a static +//! embedding of a word or phrase. Contextual embeddings, on the other hand, are +//! generated by a model that takes into account the context of the word or phrase +//! in question. This module provides a variety of embedding solutions for use in +//! natural language processing applications. +pub use self::{embedding::*, utils::*}; + +pub(crate) mod embedding; + +pub(crate) mod utils {} diff --git a/ml/nlp/src/lib.rs b/ml/nlp/src/lib.rs new file mode 100644 index 00000000..3a81de3f --- /dev/null +++ b/ml/nlp/src/lib.rs @@ -0,0 +1,20 @@ +/* + Appellation: concision-nlp + Contrib: FL03 +*/ +//! # Natural Language Processing +pub use self::{primitives::*, specs::*, utils::*}; + +pub(crate) mod primitives; +pub(crate) mod specs; +pub(crate) mod utils; + +pub mod embed; + +pub mod prelude { + pub use crate::embed::*; + + pub use crate::primitives::*; + pub use crate::specs::*; + pub use crate::utils::*; +} diff --git a/ml/nlp/src/primitives.rs b/ml/nlp/src/primitives.rs new file mode 100644 index 00000000..859023bb --- /dev/null +++ b/ml/nlp/src/primitives.rs @@ -0,0 +1,11 @@ +/* + Appellation: primitives + Contrib: FL03 +*/ +pub use self::{constants::*, statics::*, types::*}; + +mod constants {} + +mod statics {} + +mod types {} diff --git a/ml/neural/src/neurons/activate/common/linear.rs b/ml/nlp/src/specs.rs similarity index 59% rename from ml/neural/src/neurons/activate/common/linear.rs rename to ml/nlp/src/specs.rs index 50b9b5c5..1d8faa71 100644 --- a/ml/neural/src/neurons/activate/common/linear.rs +++ b/ml/nlp/src/specs.rs @@ -1,4 +1,4 @@ /* - Appellation: linear + Appellation: specs Contrib: FL03 */ diff --git a/ml/nlp/src/utils.rs b/ml/nlp/src/utils.rs new file mode 100644 index 00000000..752dabaf --- /dev/null +++ b/ml/nlp/src/utils.rs @@ -0,0 +1,4 @@ +/* + Appellation: utils + Contrib: FL03 +*/ diff --git a/ml/nlp/tests/default.rs b/ml/nlp/tests/default.rs new file mode 100644 index 00000000..0cac1eb5 --- /dev/null +++ b/ml/nlp/tests/default.rs @@ -0,0 +1,8 @@ +#[cfg(test)] +#[test] +fn compiles() { + let f = |x: usize, y: usize| x + y; + + assert_eq!(f(10, 10), 20); + assert_ne!(f(1, 1), 3); +} diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index 5df51fe1..5f5735ec 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -2,19 +2,43 @@ Appellation: head Contrib: FL03 */ +use crate::neural::prelude::activate::{Activator, Softmax}; +use ndarray::Array1; use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, Default, Deserialize, PartialEq, PartialOrd, Serialize)] -pub struct Weights { - key: Vec, - query: Vec, - value: Vec, +fn compute_attention(query: &Array1, key: &Array1, value: &Array1) -> Array1 { + let dk = query.shape()[0] as f64; + + let inner = (query.clone() * key.t().clone()) / dk.sqrt(); + value * Softmax::rho(inner) +} + +pub struct AttentionDim { + pub attention: usize, // The dimension of the key, query, and value vectors + pub heads: usize, // The number of attention heads + pub model: usize, // The dimension of the model (embedding size) +} + +impl AttentionDim { + pub fn new(attention: usize, heads: usize, model: usize) -> Self { + Self { + attention, + heads, + model, + } + } + + pub fn linear(model: usize, heads: usize) -> Self { + Self { + attention: model / heads, + heads, + model, + } + } } pub struct AttentionParams { - pub(crate) depth: usize, // embedding size - pub(crate) heads: usize, // number of attention heads - pub(crate) dropout: f64, + pub dim: AttentionDim, } #[derive(Clone, Debug, Default, Deserialize, PartialEq, PartialOrd, Serialize)] diff --git a/ml/transformers/src/attention/mod.rs b/ml/transformers/src/attention/mod.rs index ca64ebc3..6cf2579c 100644 --- a/ml/transformers/src/attention/mod.rs +++ b/ml/transformers/src/attention/mod.rs @@ -7,6 +7,8 @@ pub use self::{head::*, utils::*}; pub(crate) mod head; +pub type AttentionArray = ndarray::Array2; + pub trait Attention {} pub(crate) mod utils {} diff --git a/ml/transformers/src/attention/weights.rs b/ml/transformers/src/attention/weights.rs new file mode 100644 index 00000000..2d3db9e4 --- /dev/null +++ b/ml/transformers/src/attention/weights.rs @@ -0,0 +1,77 @@ +/* + Appellation: weights + Contrib: FL03 +*/ +use ndarray::Dim; +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; + +pub type WeightsArray = ndarray::Array2; + +pub type WeightDim = Dim<[usize; 2]>; + +fn compute_head_size(depth: usize, heads: usize) -> usize { + depth / heads +} + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum Weights { + #[serde(alias = "k")] + Key, + #[default] + #[serde(alias = "q")] + Query, + #[serde(alias = "v")] + Value, +} + +impl ndarray::Dimension for Weights { + const NDIM: usize = 3; + + fn ndim(&self) -> usize { + 3 + } + fn shape(&self) -> ndarray::IxDyn { + ndarray::IxDyn(&[1]) + } +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, PartialOrd, Serialize)] +pub struct Weight { + key: Vec, + query: Vec, + value: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_attention_weights() { + let w = Weights::Key; + assert_eq!(w.to_string(), "key"); + assert_eq!(Weights::Key, Weights::from_str("key").unwrap()); + assert_eq!(Weights::Key, Weights::from_str("k").unwrap()); + } +} \ No newline at end of file diff --git a/ml/transformers/src/codec/encode/embed.rs b/ml/transformers/src/codec/encode/embed.rs deleted file mode 100644 index 2b0e2e5b..00000000 --- a/ml/transformers/src/codec/encode/embed.rs +++ /dev/null @@ -1,29 +0,0 @@ -/* - Appellation: embed - Contrib: FL03 -*/ - -pub trait Embed {} - -#[derive(Clone)] -pub struct Embedding { - pub(crate) embedding: Vec, - pub(crate) position: Vec, -} - -impl Embedding { - pub fn new(embedding: Vec, position: Vec) -> Self { - Self { - embedding, - position, - } - } - - pub fn embedding(&self) -> &Vec { - &self.embedding - } - - pub fn position(&self) -> &Vec { - &self.position - } -} diff --git a/ml/transformers/src/codec/encode/mod.rs b/ml/transformers/src/codec/encode/mod.rs index 0f717c9f..03cd8a0a 100644 --- a/ml/transformers/src/codec/encode/mod.rs +++ b/ml/transformers/src/codec/encode/mod.rs @@ -3,9 +3,8 @@ Contrib: FL03 */ //! # Encode -pub use self::{embed::*, encoder::*, utils::*}; +pub use self::{encoder::*, utils::*}; -pub(crate) mod embed; pub(crate) mod encoder; pub trait Encode {} From 814ec6e99b41b842f69ce7ee6be9db34d41290b8 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 22 Oct 2023 02:15:15 -0500 Subject: [PATCH 028/118] update Signed-off-by: FL03 --- ml/transformers/src/attention/head.rs | 2 +- ml/transformers/src/transform/transformer.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index 5f5735ec..8b40c830 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -6,7 +6,7 @@ use crate::neural::prelude::activate::{Activator, Softmax}; use ndarray::Array1; use serde::{Deserialize, Serialize}; -fn compute_attention(query: &Array1, key: &Array1, value: &Array1) -> Array1 { +fn _attention(query: &Array1, key: &Array1, value: &Array1) -> Array1 { let dk = query.shape()[0] as f64; let inner = (query.clone() * key.t().clone()) / dk.sqrt(); diff --git a/ml/transformers/src/transform/transformer.rs b/ml/transformers/src/transform/transformer.rs index a15b51e4..41ce0d4e 100644 --- a/ml/transformers/src/transform/transformer.rs +++ b/ml/transformers/src/transform/transformer.rs @@ -3,7 +3,6 @@ Contrib: FL03 */ use super::Transform; -use crate::codec::Codec; #[derive(Clone, Debug, Default)] pub struct Transformer; From b77c54cb5725212f04217d2685151fb99e479d35 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 22 Oct 2023 13:27:14 -0500 Subject: [PATCH 029/118] update Signed-off-by: FL03 --- data/src/df/dataframe.rs | 13 ++--- data/src/lib.rs | 2 + data/src/tensors/mod.rs | 10 ++++ data/src/tensors/tensor.rs | 23 ++++++++ ml/neural/src/layers/layer.rs | 14 ++--- ml/neural/src/neurons/activate/binary.rs | 19 +++++-- ml/neural/src/neurons/activate/mod.rs | 18 +++++++ ml/neural/src/neurons/activate/nonlinear.rs | 46 ++++++++++++---- ml/neural/src/prop/kinds.rs | 9 ---- ml/neural/src/prop/mod.rs | 4 +- ml/neural/src/prop/modes.rs | 58 +++++++++++++++++++++ ml/neural/src/prop/propagation.rs | 5 ++ ml/nlp/src/embed/context/mod.rs | 10 ++++ ml/nlp/src/embed/mod.rs | 3 ++ ml/nlp/src/embed/words/mod.rs | 12 +++++ ml/nlp/src/embed/words/word2vec.rs | 16 ++++++ ml/transformers/Cargo.toml | 1 + ml/transformers/src/attention/head.rs | 14 +++-- ml/transformers/src/attention/mod.rs | 3 +- ml/transformers/src/attention/weights.rs | 13 +---- ml/transformers/src/ffn/mod.rs | 18 +++++++ ml/transformers/src/ffn/network.rs | 14 +++++ ml/transformers/src/lib.rs | 1 + ml/transformers/src/primitives.rs | 7 ++- 24 files changed, 274 insertions(+), 59 deletions(-) create mode 100644 data/src/tensors/mod.rs create mode 100644 data/src/tensors/tensor.rs delete mode 100644 ml/neural/src/prop/kinds.rs create mode 100644 ml/neural/src/prop/modes.rs create mode 100644 ml/nlp/src/embed/context/mod.rs create mode 100644 ml/nlp/src/embed/words/mod.rs create mode 100644 ml/nlp/src/embed/words/word2vec.rs create mode 100644 ml/transformers/src/ffn/mod.rs create mode 100644 ml/transformers/src/ffn/network.rs diff --git a/data/src/df/dataframe.rs b/data/src/df/dataframe.rs index 13a257e3..be7058a6 100644 --- a/data/src/df/dataframe.rs +++ b/data/src/df/dataframe.rs @@ -2,21 +2,22 @@ Appellation: dataframe Contrib: FL03 */ +use ndarray::{Array, Dimension}; use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)] #[serde(rename_all = "lowercase")] -pub struct DataFrame { - data: Vec, +pub struct DataFrame { + data: Array, } -impl DataFrame { +impl DataFrame { pub fn new() -> Self { - Self { data: Vec::new() } + Self { data: Default::default() } } } -impl std::fmt::Display for DataFrame { +impl std::fmt::Display for DataFrame where D: Serialize, T: Serialize { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "{}", serde_json::to_string(self).unwrap()) } diff --git a/data/src/lib.rs b/data/src/lib.rs index d8ca8940..f736b907 100644 --- a/data/src/lib.rs +++ b/data/src/lib.rs @@ -11,10 +11,12 @@ pub(crate) mod utils; pub mod df; pub mod flows; +pub mod tensors; pub mod prelude { pub use crate::df::*; pub use crate::flows::*; + pub use crate::tensors::*; pub use crate::primitives::*; pub use crate::specs::*; diff --git a/data/src/tensors/mod.rs b/data/src/tensors/mod.rs new file mode 100644 index 00000000..a0ff37c2 --- /dev/null +++ b/data/src/tensors/mod.rs @@ -0,0 +1,10 @@ +/* + Appellation: tensors + Contrib: FL03 +*/ +//! # Tensors +pub use self::{tensor::*, utils::*}; + +pub(crate) mod tensor; + +pub(crate) mod utils {} diff --git a/data/src/tensors/tensor.rs b/data/src/tensors/tensor.rs new file mode 100644 index 00000000..e5944170 --- /dev/null +++ b/data/src/tensors/tensor.rs @@ -0,0 +1,23 @@ +/* + Appellation: tensor + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "lowercase")] +pub struct Tensor { + data: Vec, +} + +impl Tensor { + pub fn new() -> Self { + Self { data: Vec::new() } + } +} + +impl std::fmt::Display for Tensor { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", serde_json::to_string(self).unwrap()) + } +} diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index ada2986c..51ce7e6c 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -13,17 +13,17 @@ where T: num::Float + 'static, { type Activator: Activator; + // + fn activate(&self, args: &Array2) -> Array2 { + let z = args.dot(self.weights()) - self.bias(); + z.mapv(|x| self.activator().activate(x)) + } + + fn activator(&self) -> &Self::Activator; fn bias(&self) -> &Array2; fn weights(&self) -> &Array2; - - fn activator(&self) -> &Self::Activator; - - fn activate(&self, args: &Array2) -> Array2 { - let z = args.dot(self.weights()) + self.bias(); - z.mapv(|x| self.activator().activate(x)) - } } #[derive(Clone, Debug, Default, PartialEq)] diff --git a/ml/neural/src/neurons/activate/binary.rs b/ml/neural/src/neurons/activate/binary.rs index d6e3f99a..a852091e 100644 --- a/ml/neural/src/neurons/activate/binary.rs +++ b/ml/neural/src/neurons/activate/binary.rs @@ -3,14 +3,13 @@ Contrib: FL03 */ use crate::neurons::activate::Activator; +use ndarray::Array; +use num::{One, Zero}; pub struct Heavyside; -impl Activator for Heavyside -where - T: num::Float, -{ - fn rho(x: T) -> T { +impl Heavyside { + pub fn heavyside(x: T) -> T { if x > T::zero() { T::one() } else { @@ -18,3 +17,13 @@ where } } } + +impl Activator> for Heavyside +where + D: ndarray::Dimension, + T: Clone + PartialOrd + One + Zero, +{ + fn rho(x: Array) -> Array { + x.mapv(|x| Self::heavyside(x)) + } +} diff --git a/ml/neural/src/neurons/activate/mod.rs b/ml/neural/src/neurons/activate/mod.rs index 300b4ae9..c6358643 100644 --- a/ml/neural/src/neurons/activate/mod.rs +++ b/ml/neural/src/neurons/activate/mod.rs @@ -12,6 +12,18 @@ pub(crate) mod nonlinear; pub type ActivationFn = fn(T) -> T; +pub struct Linear; + +impl Activator for Linear { + fn rho(x: T) -> T { + x + } +} + +pub trait Activate { + fn activate(&self, x: T) -> T; +} + pub trait Activable { fn activate(&self, rho: impl Activator) -> T; } @@ -28,4 +40,10 @@ pub trait Activator { fn rho(x: T) -> T; } +// impl Activator for F where F: Fn(T) -> T{ +// fn rho(x: T) -> T { +// F::call(&self, args) +// } +// } + pub(crate) mod utils {} diff --git a/ml/neural/src/neurons/activate/nonlinear.rs b/ml/neural/src/neurons/activate/nonlinear.rs index a5dd7193..89da75eb 100644 --- a/ml/neural/src/neurons/activate/nonlinear.rs +++ b/ml/neural/src/neurons/activate/nonlinear.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use super::Activator; -use ndarray::prelude::Array1; +use ndarray::prelude::{Array, Array1}; pub fn softmax(args: Array1) -> Array1 where @@ -13,23 +13,45 @@ where args.mapv(|x| x.exp() / denom) } -pub struct Sigmoid; +pub struct ReLU; -impl Sigmoid { - pub fn compute(x: f64) -> f64 { - 1.0 / (1.0 + (-x).exp()) +impl ReLU { + pub fn compute(x: T) -> T { + if x > T::zero() { + x + } else { + T::zero() + } } } -impl Activator for Sigmoid +impl Activator> for ReLU where + D: ndarray::Dimension, T: num::Float, { - fn rho(x: T) -> T { + fn rho(x: Array) -> Array { + x.mapv(|x| Self::compute(x)) + } +} + +pub struct Sigmoid; + +impl Sigmoid { + pub fn compute(x: T) -> T { T::one() / (T::one() + (-x).exp()) } } +impl Activator> for Sigmoid +where + D: ndarray::Dimension, + T: num::Float, +{ + fn rho(x: Array) -> Array { + x.mapv(|x| Self::compute(x)) + } +} pub struct Softmax; impl Softmax { @@ -38,11 +60,12 @@ impl Softmax { } } -impl Activator> for Softmax +impl Activator> for Softmax where + D: ndarray::Dimension, T: num::Float, { - fn rho(x: Array1) -> Array1 { + fn rho(x: Array) -> Array { let denom = x.mapv(|x| x.exp()).sum(); x.mapv(|x| x.exp() / denom) } @@ -52,11 +75,12 @@ where mod tests { use super::*; use computare::prelude::RoundTo; + use ndarray::array; #[test] fn test_softmax() { - let exp = Array1::from(vec![0.09003057, 0.24472847, 0.66524096]); - let args = Array1::from(vec![1.0, 2.0, 3.0]); + let exp = array![0.09003057, 0.24472847, 0.66524096]; + let args = array![1.0, 2.0, 3.0]; let res = Softmax::rho(args).mapv(|i| i.round_to(8)); assert_eq!(res, exp); } diff --git a/ml/neural/src/prop/kinds.rs b/ml/neural/src/prop/kinds.rs deleted file mode 100644 index 26970c47..00000000 --- a/ml/neural/src/prop/kinds.rs +++ /dev/null @@ -1,9 +0,0 @@ -/* - Appellation: kinds - Contrib: FL03 -*/ - -pub enum PropagationKind { - Forward, - Backward, -} diff --git a/ml/neural/src/prop/mod.rs b/ml/neural/src/prop/mod.rs index 5a5148a6..12d73f26 100644 --- a/ml/neural/src/prop/mod.rs +++ b/ml/neural/src/prop/mod.rs @@ -5,9 +5,9 @@ //! # Propagation //! //! This module describes the propagation of data through a neural network. -pub use self::{kinds::*, propagation::*, utils::*}; +pub use self::{modes::*, propagation::*, utils::*}; -pub(crate) mod kinds; +pub(crate) mod modes; pub(crate) mod propagation; pub(crate) mod utils {} diff --git a/ml/neural/src/prop/modes.rs b/ml/neural/src/prop/modes.rs new file mode 100644 index 00000000..3860b43e --- /dev/null +++ b/ml/neural/src/prop/modes.rs @@ -0,0 +1,58 @@ +/* + Appellation: kinds + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum PropagationMode { + #[default] + Backward = 0, + Forward = 1, +} + +impl PropagationMode { + pub fn backward() -> Self { + Self::Backward + } + + pub fn forward() -> Self { + Self::Forward + } +} + +impl From for PropagationMode { + fn from(value: usize) -> Self { + match value % 2 { + 1 => Self::Forward, + _ => Self::Backward, + } + } +} + +impl From for usize { + fn from(value: PropagationMode) -> Self { + value as usize + } +} diff --git a/ml/neural/src/prop/propagation.rs b/ml/neural/src/prop/propagation.rs index fb65f594..7048eb46 100644 --- a/ml/neural/src/prop/propagation.rs +++ b/ml/neural/src/prop/propagation.rs @@ -2,3 +2,8 @@ Appellation: propagation Contrib: FL03 */ +use super::PropagationMode; + +pub struct Propagator { + pub mode: PropagationMode, +} diff --git a/ml/nlp/src/embed/context/mod.rs b/ml/nlp/src/embed/context/mod.rs new file mode 100644 index 00000000..c93d20d4 --- /dev/null +++ b/ml/nlp/src/embed/context/mod.rs @@ -0,0 +1,10 @@ +/* + Appellation: context + Contrib: FL03 +*/ +//! # Contextual Embedding +//! + +pub use self::utils::*; + +pub(crate) mod utils {} diff --git a/ml/nlp/src/embed/mod.rs b/ml/nlp/src/embed/mod.rs index 1576c35b..c37d44b0 100644 --- a/ml/nlp/src/embed/mod.rs +++ b/ml/nlp/src/embed/mod.rs @@ -15,4 +15,7 @@ pub use self::{embedding::*, utils::*}; pub(crate) mod embedding; +pub mod context; +pub mod words; + pub(crate) mod utils {} diff --git a/ml/nlp/src/embed/words/mod.rs b/ml/nlp/src/embed/words/mod.rs new file mode 100644 index 00000000..ca206721 --- /dev/null +++ b/ml/nlp/src/embed/words/mod.rs @@ -0,0 +1,12 @@ +/* + Appellation: words + Contrib: FL03 +*/ +//! # Word Embedding +//! + +pub use self::utils::*; + +pub mod word2vec; + +pub(crate) mod utils {} diff --git a/ml/nlp/src/embed/words/word2vec.rs b/ml/nlp/src/embed/words/word2vec.rs new file mode 100644 index 00000000..7d123428 --- /dev/null +++ b/ml/nlp/src/embed/words/word2vec.rs @@ -0,0 +1,16 @@ +/* + Appellation: word2vec + Contrib: FL03 +*/ +//! # word2vec +//! +//! word2vec is a popular static embedding solution that is used in a variety of +//! natural language processing applications. This module provides a word2vec +//! implementation that can be used to generate static embeddings for words and +//! phrases. +//! +//! ## Methods +//! +//! ### Continuous Bag of Words (CBOW) +//! +//! ### Skip-Gram diff --git a/ml/transformers/Cargo.toml b/ml/transformers/Cargo.toml index c7848202..d3730131 100644 --- a/ml/transformers/Cargo.toml +++ b/ml/transformers/Cargo.toml @@ -26,6 +26,7 @@ test = true [dependencies] concision-neural = { path = "../neural" } anyhow.workspace = true +lazy_static.workspace = true ndarray.workspace = true num.workspace = true serde.workspace = true diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index 8b40c830..b7060616 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -3,16 +3,20 @@ Contrib: FL03 */ use crate::neural::prelude::activate::{Activator, Softmax}; -use ndarray::Array1; +use ndarray::{s, Array3, Array2}; use serde::{Deserialize, Serialize}; -fn _attention(query: &Array1, key: &Array1, value: &Array1) -> Array1 { - let dk = query.shape()[0] as f64; +fn _attention(qkv: &Array3) -> Array2 { + let query = qkv.slice(s![0, .., ..]).to_owned(); + let key = qkv.slice(s![1, .., ..]).to_owned(); + let value = qkv.slice(s![2, .., ..]).to_owned(); + let dk = qkv.shape()[1] as f64; - let inner = (query.clone() * key.t().clone()) / dk.sqrt(); - value * Softmax::rho(inner) + let inner = (query * key.t()) / dk.sqrt(); + Softmax::rho(inner) * value } + pub struct AttentionDim { pub attention: usize, // The dimension of the key, query, and value vectors pub heads: usize, // The number of attention heads diff --git a/ml/transformers/src/attention/mod.rs b/ml/transformers/src/attention/mod.rs index 6cf2579c..be28c18f 100644 --- a/ml/transformers/src/attention/mod.rs +++ b/ml/transformers/src/attention/mod.rs @@ -3,9 +3,10 @@ Contrib: FL03 */ //! # Attention -pub use self::{head::*, utils::*}; +pub use self::{head::*, utils::*, weights::*,}; pub(crate) mod head; +pub(crate) mod weights; pub type AttentionArray = ndarray::Array2; diff --git a/ml/transformers/src/attention/weights.rs b/ml/transformers/src/attention/weights.rs index 2d3db9e4..6bef6641 100644 --- a/ml/transformers/src/attention/weights.rs +++ b/ml/transformers/src/attention/weights.rs @@ -10,9 +10,6 @@ pub type WeightsArray = ndarray::Array2; pub type WeightDim = Dim<[usize; 2]>; -fn compute_head_size(depth: usize, heads: usize) -> usize { - depth / heads -} #[derive( Clone, @@ -45,16 +42,7 @@ pub enum Weights { Value, } -impl ndarray::Dimension for Weights { - const NDIM: usize = 3; - fn ndim(&self) -> usize { - 3 - } - fn shape(&self) -> ndarray::IxDyn { - ndarray::IxDyn(&[1]) - } -} #[derive(Clone, Debug, Default, Deserialize, PartialEq, PartialOrd, Serialize)] pub struct Weight { @@ -66,6 +54,7 @@ pub struct Weight { #[cfg(test)] mod tests { use super::*; + use std::str::FromStr; #[test] fn test_attention_weights() { diff --git a/ml/transformers/src/ffn/mod.rs b/ml/transformers/src/ffn/mod.rs new file mode 100644 index 00000000..2c45033a --- /dev/null +++ b/ml/transformers/src/ffn/mod.rs @@ -0,0 +1,18 @@ +/* + Appellation: decode + Contrib: FL03 +*/ +//! # Decode +pub use self::{network::*, utils::*}; + +pub(crate) mod network; + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests { + // use super::*; + + #[test] + fn test_ffn() {} +} diff --git a/ml/transformers/src/ffn/network.rs b/ml/transformers/src/ffn/network.rs new file mode 100644 index 00000000..4538e334 --- /dev/null +++ b/ml/transformers/src/ffn/network.rs @@ -0,0 +1,14 @@ +/* + Appellation: network + Contrib: FL03 +*/ +use crate::neural::neurons::activate::{Activator, ReLU}; +use ndarray::Array2; + +/// All vectors have a dimension of (nodes, elem) +pub fn ffn(data: Array2, bias: Array2, weights: Array2) -> Array2 { + let a = data * weights.row(0) + bias.row(0); + ReLU::rho(a) * weights.row(1) + bias.row(1) +} + +pub struct FFN {} diff --git a/ml/transformers/src/lib.rs b/ml/transformers/src/lib.rs index c2335679..30766699 100644 --- a/ml/transformers/src/lib.rs +++ b/ml/transformers/src/lib.rs @@ -11,6 +11,7 @@ pub(crate) mod utils; pub mod attention; pub mod codec; +pub mod ffn; pub mod transform; pub(crate) use concision_neural as neural; diff --git a/ml/transformers/src/primitives.rs b/ml/transformers/src/primitives.rs index b7131c91..e3259855 100644 --- a/ml/transformers/src/primitives.rs +++ b/ml/transformers/src/primitives.rs @@ -11,10 +11,15 @@ pub(crate) mod constants { pub const DEFAULT_ATTENTION_HEADS: usize = 8; pub const DEFAULT_ATTENTION_HEAD_SIZE: usize = 64; + + pub const DEFAULT_SAMPLE_SIZE: usize = 10000; + + } /// Collection of static references used throughout -pub(crate) mod statics {} +pub(crate) mod statics { +} /// Collection of types used throughout the system pub(crate) mod types {} From 0747cb200e84c43dc6cb15fe0a97b51834da68f4 Mon Sep 17 00:00:00 2001 From: FL03 Date: Tue, 24 Oct 2023 15:07:54 -0500 Subject: [PATCH 030/118] update Signed-off-by: FL03 --- ml/nlp/src/embed/embedding.rs | 22 ++++++++++++++++++---- ml/nlp/src/embed/mod.rs | 4 ++++ ml/transformers/src/attention/head.rs | 18 +++++------------- ml/transformers/src/attention/mod.rs | 17 ++++++++++++++++- 4 files changed, 43 insertions(+), 18 deletions(-) diff --git a/ml/nlp/src/embed/embedding.rs b/ml/nlp/src/embed/embedding.rs index d6ba8071..dfc4cc98 100644 --- a/ml/nlp/src/embed/embedding.rs +++ b/ml/nlp/src/embed/embedding.rs @@ -2,15 +2,20 @@ Appellation: embedding Contrib: FL03 */ +use ndarray::{Array2, Dim, IntoDimension}; use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] #[serde(rename_all = "lowercase")] -pub struct Embedding; +pub struct Embedding { + data: Array2, +} impl Embedding { - pub fn new() -> Self { - Self + pub fn new(dim: Dim<[usize; 2]>) -> Self { + Self { + data: Array2::zeros(dim), + } } } @@ -19,3 +24,12 @@ impl std::fmt::Display for Embedding { write!(f, "{}", serde_json::to_string(self).unwrap()) } } + +impl From for Embedding +where + D: IntoDimension>, +{ + fn from(dim: D) -> Self { + Self::new(dim.into_dimension()) + } +} diff --git a/ml/nlp/src/embed/mod.rs b/ml/nlp/src/embed/mod.rs index c37d44b0..6bc8499e 100644 --- a/ml/nlp/src/embed/mod.rs +++ b/ml/nlp/src/embed/mod.rs @@ -18,4 +18,8 @@ pub(crate) mod embedding; pub mod context; pub mod words; +pub trait Embed { + +} + pub(crate) mod utils {} diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index b7060616..2e5b2df3 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -2,39 +2,31 @@ Appellation: head Contrib: FL03 */ -use crate::neural::prelude::activate::{Activator, Softmax}; -use ndarray::{s, Array3, Array2}; use serde::{Deserialize, Serialize}; -fn _attention(qkv: &Array3) -> Array2 { - let query = qkv.slice(s![0, .., ..]).to_owned(); - let key = qkv.slice(s![1, .., ..]).to_owned(); - let value = qkv.slice(s![2, .., ..]).to_owned(); - let dk = qkv.shape()[1] as f64; - - let inner = (query * key.t()) / dk.sqrt(); - Softmax::rho(inner) * value -} pub struct AttentionDim { pub attention: usize, // The dimension of the key, query, and value vectors + pub batch: usize, // The batch size pub heads: usize, // The number of attention heads pub model: usize, // The dimension of the model (embedding size) } impl AttentionDim { - pub fn new(attention: usize, heads: usize, model: usize) -> Self { + pub fn new(attention: usize, batch: usize, heads: usize, model: usize) -> Self { Self { attention, + batch, heads, model, } } - pub fn linear(model: usize, heads: usize) -> Self { + pub fn linear(batch: usize, model: usize, heads: usize) -> Self { Self { attention: model / heads, + batch, heads, model, } diff --git a/ml/transformers/src/attention/mod.rs b/ml/transformers/src/attention/mod.rs index be28c18f..125c2436 100644 --- a/ml/transformers/src/attention/mod.rs +++ b/ml/transformers/src/attention/mod.rs @@ -3,6 +3,8 @@ Contrib: FL03 */ //! # Attention +//! +//! pub use self::{head::*, utils::*, weights::*,}; pub(crate) mod head; @@ -12,7 +14,20 @@ pub type AttentionArray = ndarray::Array2; pub trait Attention {} -pub(crate) mod utils {} +pub(crate) mod utils { + use crate::neural::prelude::activate::{Activator, Softmax}; + use ndarray::{s, Array3, Array2}; + + pub fn compute_attention(qkv: &Array3) -> Array2 { + let query = qkv.slice(s![0, .., ..]).to_owned(); + let key = qkv.slice(s![1, .., ..]).to_owned(); + let value = qkv.slice(s![2, .., ..]).to_owned(); + let dk = qkv.shape()[1] as f64; + + let inner = (query * key.t()) / dk.sqrt(); + Softmax::rho(inner) * value + } +} #[cfg(test)] mod tests { From 346de4eafe89da616b74291c8e95ce52b7b3ee77 Mon Sep 17 00:00:00 2001 From: FL03 Date: Wed, 25 Oct 2023 23:22:07 -0500 Subject: [PATCH 031/118] update Signed-off-by: FL03 --- Cargo.toml | 1 + data/src/df/dataframe.rs | 10 +- ml/neural/Cargo.toml | 1 + ml/neural/src/layers/layer.rs | 2 +- ml/neural/src/lib.rs | 4 + ml/neural/src/neurons/activate/mod.rs | 25 ++- ml/nlp/Cargo.toml | 1 + ml/nlp/src/embed/mod.rs | 2 +- ml/transformers/src/attention/context.rs | 181 ++++++++++++++++++ ml/transformers/src/attention/head.rs | 65 +++---- ml/transformers/src/attention/mod.rs | 69 +++++-- .../src/attention/multi/attention.rs | 4 + ml/transformers/src/attention/multi/mod.rs | 15 ++ ml/transformers/src/attention/params/dim.rs | 88 +++++++++ ml/transformers/src/attention/params/mod.rs | 25 +++ ml/transformers/src/attention/weights.rs | 66 ------- ml/transformers/src/primitives.rs | 5 +- 17 files changed, 430 insertions(+), 134 deletions(-) create mode 100644 ml/transformers/src/attention/context.rs create mode 100644 ml/transformers/src/attention/multi/attention.rs create mode 100644 ml/transformers/src/attention/multi/mod.rs create mode 100644 ml/transformers/src/attention/params/dim.rs create mode 100644 ml/transformers/src/attention/params/mod.rs delete mode 100644 ml/transformers/src/attention/weights.rs diff --git a/Cargo.toml b/Cargo.toml index d30e6577..32ad33ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ anyhow = "1" lazy_static = "1" ndarray = { features = ["serde-1"], version = "0.15" } num = { features = ["serde"], version = "0.4" } +petgraph = { features = ["serde-1"], version = "0.6" } serde = { features = ["derive"], version = "1" } serde_json = "1" smart-default = "0.7" diff --git a/data/src/df/dataframe.rs b/data/src/df/dataframe.rs index be7058a6..2c66c96d 100644 --- a/data/src/df/dataframe.rs +++ b/data/src/df/dataframe.rs @@ -13,11 +13,17 @@ pub struct DataFrame { impl DataFrame { pub fn new() -> Self { - Self { data: Default::default() } + Self { + data: Default::default(), + } } } -impl std::fmt::Display for DataFrame where D: Serialize, T: Serialize { +impl std::fmt::Display for DataFrame +where + D: Serialize, + T: Serialize, +{ fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "{}", serde_json::to_string(self).unwrap()) } diff --git a/ml/neural/Cargo.toml b/ml/neural/Cargo.toml index 5aed8472..30ab801c 100644 --- a/ml/neural/Cargo.toml +++ b/ml/neural/Cargo.toml @@ -29,6 +29,7 @@ concision-core.workspace = true anyhow.workspace = true ndarray.workspace = true num.workspace = true +petgraph.workspace = true serde.workspace = true serde_json.workspace = true smart-default.workspace = true diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 51ce7e6c..35cc7928 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -18,7 +18,7 @@ where let z = args.dot(self.weights()) - self.bias(); z.mapv(|x| self.activator().activate(x)) } - + fn activator(&self) -> &Self::Activator; fn bias(&self) -> &Array2; diff --git a/ml/neural/src/lib.rs b/ml/neural/src/lib.rs index 8a8efe17..ed779bea 100644 --- a/ml/neural/src/lib.rs +++ b/ml/neural/src/lib.rs @@ -3,6 +3,10 @@ Contrib: FL03 */ //! # concision-neural +//! +//! This library implements the neural network primitives and specifications. +//! +#![feature(fn_traits)] pub use self::{primitives::*, specs::*, utils::*}; pub(crate) mod primitives; diff --git a/ml/neural/src/neurons/activate/mod.rs b/ml/neural/src/neurons/activate/mod.rs index c6358643..eea0a127 100644 --- a/ml/neural/src/neurons/activate/mod.rs +++ b/ml/neural/src/neurons/activate/mod.rs @@ -14,6 +14,12 @@ pub type ActivationFn = fn(T) -> T; pub struct Linear; +impl Linear { + pub fn act() -> ActivationFn { + |x| x + } +} + impl Activator for Linear { fn rho(x: T) -> T { x @@ -24,12 +30,8 @@ pub trait Activate { fn activate(&self, x: T) -> T; } -pub trait Activable { - fn activate(&self, rho: impl Activator) -> T; -} - pub trait ActivateMethod { - fn activate(&self, x: T) -> T; + fn method() -> fn(T) -> T; } pub trait Activator { @@ -40,10 +42,13 @@ pub trait Activator { fn rho(x: T) -> T; } -// impl Activator for F where F: Fn(T) -> T{ -// fn rho(x: T) -> T { -// F::call(&self, args) -// } -// } +impl Activate for F +where + F: Fn(T) -> T, +{ + fn activate(&self, x: T) -> T { + self.call((x,)) + } +} pub(crate) mod utils {} diff --git a/ml/nlp/Cargo.toml b/ml/nlp/Cargo.toml index 37433b6a..d705fd8e 100644 --- a/ml/nlp/Cargo.toml +++ b/ml/nlp/Cargo.toml @@ -25,6 +25,7 @@ test = true [dependencies] anyhow.workspace = true +finalfusion = "0.18" ndarray.workspace = true num.workspace = true serde.workspace = true diff --git a/ml/nlp/src/embed/mod.rs b/ml/nlp/src/embed/mod.rs index 6bc8499e..f2b80d40 100644 --- a/ml/nlp/src/embed/mod.rs +++ b/ml/nlp/src/embed/mod.rs @@ -19,7 +19,7 @@ pub mod context; pub mod words; pub trait Embed { - + fn embed(&self) -> Embedding; } pub(crate) mod utils {} diff --git a/ml/transformers/src/attention/context.rs b/ml/transformers/src/attention/context.rs new file mode 100644 index 00000000..8052b6f0 --- /dev/null +++ b/ml/transformers/src/attention/context.rs @@ -0,0 +1,181 @@ +/* + Appellation: context + Contrib: FL03 +*/ +use ndarray::{Array2, IntoDimension, Ix2}; +use serde::{Deserialize, Serialize}; +use std::ops; +use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum QKV { + #[serde(alias = "k")] + Key, + #[default] + #[serde(alias = "q")] + Query, + #[serde(alias = "v")] + Value, +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +pub struct Context { + dim: Ix2, + pub key: Array2, + pub query: Array2, + pub value: Array2, +} + +impl Context { + pub fn new(dim: D) -> Self + where + D: IntoDimension, + { + let dim = dim.into_dimension(); + Self { + dim, + key: Array2::ones(dim), + query: Array2::ones(dim), + value: Array2::ones(dim), + } + } + + pub fn dim(&self) -> Ix2 { + self.dim + } + + pub fn matmul(&self) -> Array2 { + self.query.dot(&self.key.t()) + } + + pub fn with_weight(&mut self, query: &Array2, key: &Array2, value: &Array2) { + self.key = key.clone(); + self.query = query.clone(); + self.value = value.clone(); + } +} + +impl std::fmt::Display for Context { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", serde_json::to_string(self).unwrap()) + } +} + +impl From for Context +where + D: IntoDimension, +{ + fn from(dim: D) -> Self { + let dim = dim.into_dimension(); + let arr = Array2::ones(dim); + Self { + dim, + key: arr.clone(), + query: arr.clone(), + value: arr, + } + } +} + +impl From for (Array2, Array2, Array2) { + fn from(context: Context) -> Self { + (context.key, context.query, context.value) + } +} + +impl ops::Index for Context { + type Output = Array2; + + fn index(&self, index: QKV) -> &Self::Output { + use QKV::*; + match index { + Key => &self.key, + Query => &self.query, + Value => &self.value, + } + } +} + +impl ops::IndexMut for Context { + fn index_mut(&mut self, index: QKV) -> &mut Self::Output { + match index { + QKV::Key => &mut self.key, + QKV::Query => &mut self.query, + QKV::Value => &mut self.value, + } + } +} + +impl ops::Mul<&Array2> for Context { + type Output = Self; + + fn mul(self, rhs: &Array2) -> Self::Output { + let mut context = self.clone(); + context.key = context.key.dot(rhs); + context.query = context.query.dot(rhs); + context.value = context.value.dot(rhs); + context + } +} + +impl ops::Mul<&Array2> for &Context { + type Output = Context; + + fn mul(self, rhs: &Array2) -> Self::Output { + let mut context = self.clone(); + context.key = context.key.dot(rhs); + context.query = context.query.dot(rhs); + context.value = context.value.dot(rhs); + context + } +} + +impl ops::MulAssign> for Context { + fn mul_assign(&mut self, rhs: Array2) { + self.key = self.key.dot(&rhs); + self.query = self.query.dot(&rhs); + self.value = self.value.dot(&rhs); + } +} + +impl ops::MulAssign<&Array2> for Context { + fn mul_assign(&mut self, rhs: &Array2) { + self.key = self.key.dot(rhs); + self.query = self.query.dot(rhs); + self.value = self.value.dot(rhs); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::str::FromStr; + + #[test] + fn test_attention_weights() { + let w = QKV::Key; + assert_eq!(w.to_string(), "key"); + assert_eq!(QKV::Key, QKV::from_str("key").unwrap()); + assert_eq!(QKV::Key, QKV::from_str("k").unwrap()); + } +} diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index 2e5b2df3..4489f2a5 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -2,54 +2,47 @@ Appellation: head Contrib: FL03 */ +use super::params::AttentionDim; +use super::Context; +use crate::neural::neurons::activate::{Activator, Softmax}; +use ndarray::Array2; use serde::{Deserialize, Serialize}; - - -pub struct AttentionDim { - pub attention: usize, // The dimension of the key, query, and value vectors - pub batch: usize, // The batch size - pub heads: usize, // The number of attention heads - pub model: usize, // The dimension of the model (embedding size) +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub struct AttentionHead { + context: Context, + dim: AttentionDim, } -impl AttentionDim { - pub fn new(attention: usize, batch: usize, heads: usize, model: usize) -> Self { +impl AttentionHead { + pub fn new(dim: AttentionDim) -> Self { Self { - attention, - batch, - heads, - model, + context: Context::new(dim.head_dim()), + dim, } } - pub fn linear(batch: usize, model: usize, heads: usize) -> Self { - Self { - attention: model / heads, - batch, - heads, - model, - } + pub fn attention(&mut self, data: &Array2) -> Array2 { + self.context *= data; + + Softmax::rho(self.query().dot(&self.key().t()) * self.scale()) * self.value().clone() } -} -pub struct AttentionParams { - pub dim: AttentionDim, -} + pub fn scale(&self) -> f64 { + 1.0 / (self.dim.query_size() as f64).sqrt() + } -#[derive(Clone, Debug, Default, Deserialize, PartialEq, PartialOrd, Serialize)] -#[serde(rename_all = "lowercase")] -pub struct AttentionHead { - data: Vec, - dim: usize, -} + pub fn query(&self) -> &Array2 { + &self.context.query + } -impl AttentionHead { - pub fn new(dim: usize) -> Self { - Self { - data: Vec::new(), - dim, - } + pub fn key(&self) -> &Array2 { + &self.context.key + } + + pub fn value(&self) -> &Array2 { + &self.context.value } } diff --git a/ml/transformers/src/attention/mod.rs b/ml/transformers/src/attention/mod.rs index 125c2436..34ac5868 100644 --- a/ml/transformers/src/attention/mod.rs +++ b/ml/transformers/src/attention/mod.rs @@ -3,29 +3,70 @@ Contrib: FL03 */ //! # Attention -//! -//! -pub use self::{head::*, utils::*, weights::*,}; +//! +//! The attention mechanism is a key component of the transformer architecture. +//! +//! +pub use self::{context::*, head::*, utils::*}; + +pub(crate) mod context; pub(crate) mod head; -pub(crate) mod weights; -pub type AttentionArray = ndarray::Array2; +pub mod multi; +pub mod params; + +use ndarray::{Array, Array2}; +use std::ops; + +pub type BaseDim = ndarray::Dim<[usize; 4]>; + +/// (batch, sample, seq, model) +pub type InputArray = Array; + +pub type AttentionArray = Array2; + +pub trait Representations { + fn query(&self) -> &InputArray; + fn key(&self) -> &InputArray; + fn value(&self) -> &InputArray; +} + +pub trait Attention { + type Head: Head; + + fn head(&self) -> &Self::Head; + +} + +pub trait Head: ops::MulAssign> { + fn query(&self) -> &AttentionArray; + fn key(&self) -> &AttentionArray; + fn value(&self) -> &AttentionArray; + + +} + -pub trait Attention {} pub(crate) mod utils { use crate::neural::prelude::activate::{Activator, Softmax}; - use ndarray::{s, Array3, Array2}; + use ndarray::Array2; - pub fn compute_attention(qkv: &Array3) -> Array2 { - let query = qkv.slice(s![0, .., ..]).to_owned(); - let key = qkv.slice(s![1, .., ..]).to_owned(); - let value = qkv.slice(s![2, .., ..]).to_owned(); - let dk = qkv.shape()[1] as f64; + pub fn linear_layer( + data: &Array2, + weights: &Array2, + ) -> Array2 { + data.dot(weights) + } - let inner = (query * key.t()) / dk.sqrt(); - Softmax::rho(inner) * value + pub fn compute_attention( + query: &Array2, + key: &Array2, + value: &Array2, + ) -> Array2 { + let dk = query.shape()[1] as f64; + Softmax::rho(query.dot(&key.t()) / dk.sqrt()) * value } } diff --git a/ml/transformers/src/attention/multi/attention.rs b/ml/transformers/src/attention/multi/attention.rs new file mode 100644 index 00000000..34cd6fdb --- /dev/null +++ b/ml/transformers/src/attention/multi/attention.rs @@ -0,0 +1,4 @@ +/* + Appellation: attention + Contrib: FL03 +*/ diff --git a/ml/transformers/src/attention/multi/mod.rs b/ml/transformers/src/attention/multi/mod.rs new file mode 100644 index 00000000..d84274a1 --- /dev/null +++ b/ml/transformers/src/attention/multi/mod.rs @@ -0,0 +1,15 @@ +/* + Appellation: multi + Contrib: FL03 +*/ +pub use self::{attention::*, utils::*}; + +pub(crate) mod attention; + +pub trait Split { + fn split(&self) -> Vec + where + Self: Sized; +} + +pub(crate) mod utils {} diff --git a/ml/transformers/src/attention/params/dim.rs b/ml/transformers/src/attention/params/dim.rs new file mode 100644 index 00000000..ef7d13e7 --- /dev/null +++ b/ml/transformers/src/attention/params/dim.rs @@ -0,0 +1,88 @@ +/* + Appellation: dim + Contrib: FL03 +*/ +//! # Dimensions +//! +//! The dimensionality of the attention mechanism is defined by the following: +//! +//! - `attention`: The dimension of the key, query, and value vectors +//! - `batch`: The batch size +//! - `heads`: The number of attention heads +//! - `model`: The dimension of the model (embedding size) +use crate::{DEFAULT_ATTENTION_HEADS, DEFAULT_EMBEDDING_SIZE, DEFAULT_SAMPLE_SIZE}; +use serde::{Deserialize, Serialize}; + +pub enum AttentionDims { + Head { + query: usize, + seq: usize, + }, + Context { + batch: usize, + heads: usize, + model: usize, + seq: usize, + samples: usize, + }, +} + +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] +pub struct AttentionDim { + pub batch: usize, // The batch size + pub heads: usize, // The number of attention heads + pub model: usize, // The dimension of the model (embedding size) + pub seq: usize, // The sequence length + pub samples: usize, // The number of samples +} + +impl AttentionDim { + pub fn new(batch: usize, heads: usize, model: usize, seq: usize, samples: usize) -> Self { + Self { + batch, + heads, + model, + seq, + samples, + } + } + + pub fn std(batch: usize, seq: usize) -> Self { + Self::new( + batch, + DEFAULT_ATTENTION_HEADS, + DEFAULT_EMBEDDING_SIZE, + seq, + DEFAULT_SAMPLE_SIZE, + ) + } + + // The dimension of the key, query, and value vectors + pub fn query_size(&self) -> usize { + self.model / self.heads + } + + pub fn head_dim(&self) -> (usize, usize) { + (self.seq, self.query_size()) + } +} + +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] +pub struct HeadShape { + pub query: usize, + pub seq: usize, +} + +impl HeadShape { + pub fn new(query: usize, seq: usize) -> Self { + Self { query, seq } + } + + pub fn scale(&self) -> f64 { + 1.0 / (self.query as f64).sqrt() + } +} diff --git a/ml/transformers/src/attention/params/mod.rs b/ml/transformers/src/attention/params/mod.rs new file mode 100644 index 00000000..891cb844 --- /dev/null +++ b/ml/transformers/src/attention/params/mod.rs @@ -0,0 +1,25 @@ +/* + Appellation: params + Contrib: FL03 +*/ +//! # Attention Parameters +//! +//! ## Hyperparameters +pub use self::{dim::*, utils::*}; + +pub(crate) mod dim; + +use serde::{Deserialize, Serialize}; + +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] + +pub struct AttentionParams { + pub batch: usize, + pub dim: AttentionDim, + pub heads: usize, + pub samples: usize, +} + +pub(crate) mod utils {} diff --git a/ml/transformers/src/attention/weights.rs b/ml/transformers/src/attention/weights.rs deleted file mode 100644 index 6bef6641..00000000 --- a/ml/transformers/src/attention/weights.rs +++ /dev/null @@ -1,66 +0,0 @@ -/* - Appellation: weights - Contrib: FL03 -*/ -use ndarray::Dim; -use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; - -pub type WeightsArray = ndarray::Array2; - -pub type WeightDim = Dim<[usize; 2]>; - - -#[derive( - Clone, - Copy, - Debug, - Default, - Deserialize, - Display, - EnumIs, - EnumIter, - EnumString, - EnumVariantNames, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize, -)] -#[repr(usize)] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum Weights { - #[serde(alias = "k")] - Key, - #[default] - #[serde(alias = "q")] - Query, - #[serde(alias = "v")] - Value, -} - - - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, PartialOrd, Serialize)] -pub struct Weight { - key: Vec, - query: Vec, - value: Vec, -} - -#[cfg(test)] -mod tests { - use super::*; - use std::str::FromStr; - - #[test] - fn test_attention_weights() { - let w = Weights::Key; - assert_eq!(w.to_string(), "key"); - assert_eq!(Weights::Key, Weights::from_str("key").unwrap()); - assert_eq!(Weights::Key, Weights::from_str("k").unwrap()); - } -} \ No newline at end of file diff --git a/ml/transformers/src/primitives.rs b/ml/transformers/src/primitives.rs index e3259855..48a29aaa 100644 --- a/ml/transformers/src/primitives.rs +++ b/ml/transformers/src/primitives.rs @@ -13,13 +13,10 @@ pub(crate) mod constants { pub const DEFAULT_ATTENTION_HEAD_SIZE: usize = 64; pub const DEFAULT_SAMPLE_SIZE: usize = 10000; - - } /// Collection of static references used throughout -pub(crate) mod statics { -} +pub(crate) mod statics {} /// Collection of types used throughout the system pub(crate) mod types {} From a8524521f3341aa1497cb4add08a7b8d53bc39ef Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 27 Oct 2023 14:48:14 -0500 Subject: [PATCH 032/118] update Signed-off-by: FL03 --- core/Cargo.toml | 1 + core/src/errors/error.rs | 10 +- core/src/specs.rs | 13 ++ core/src/utils.rs | 15 +- data/src/lib.rs | 1 + data/src/linear/layer.rs | 23 +++ data/src/linear/mod.rs | 10 ++ ml/neural/src/layers/layer.rs | 2 +- ml/transformers/Cargo.toml | 2 + ml/transformers/src/attention/head.rs | 137 +++++++++++++-- ml/transformers/src/attention/mod.rs | 49 +++--- .../src/attention/multi/attention.rs | 2 + ml/transformers/src/attention/multi/mod.rs | 62 ++++++- ml/transformers/src/attention/ops/merge.rs | 38 +++++ ml/transformers/src/attention/ops/mod.rs | 90 ++++++++++ ml/transformers/src/attention/ops/split.rs | 40 +++++ ml/transformers/src/attention/params/dim.rs | 159 ++++++++++++++---- ml/transformers/src/attention/params/mod.rs | 73 +++++++- ml/transformers/src/attention/params/qkv.rs | 57 +++++++ .../src/attention/{context.rs => reps.rs} | 129 +++++++------- ml/transformers/src/ffn/network.rs | 21 ++- ml/transformers/src/lib.rs | 1 + 22 files changed, 780 insertions(+), 155 deletions(-) create mode 100644 data/src/linear/layer.rs create mode 100644 data/src/linear/mod.rs create mode 100644 ml/transformers/src/attention/ops/merge.rs create mode 100644 ml/transformers/src/attention/ops/mod.rs create mode 100644 ml/transformers/src/attention/ops/split.rs create mode 100644 ml/transformers/src/attention/params/qkv.rs rename ml/transformers/src/attention/{context.rs => reps.rs} (51%) diff --git a/core/Cargo.toml b/core/Cargo.toml index 79dea624..f8bf1980 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -25,6 +25,7 @@ test = true [dependencies] anyhow.workspace = true chrono = "0.4" +ndarray.workspace = true num.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/core/src/errors/error.rs b/core/src/errors/error.rs index a849d66a..1815d3e9 100644 --- a/core/src/errors/error.rs +++ b/core/src/errors/error.rs @@ -27,11 +27,13 @@ use strum::{Display, EnumIs, EnumIter, EnumVariantNames}; pub enum Errors { Async, Connection, - + Data, + Dimension, #[default] Error(String), Execution, IO, + Null, Process, Runtime, Syntax, @@ -138,3 +140,9 @@ impl From for Error { Self::new(Errors::Unknown, err.to_string()) } } + +impl From for Error { + fn from(err: ndarray::ShapeError) -> Self { + Self::new(Errors::Dimension, err.to_string()) + } +} diff --git a/core/src/specs.rs b/core/src/specs.rs index 1d8faa71..0ad8fffa 100644 --- a/core/src/specs.rs +++ b/core/src/specs.rs @@ -2,3 +2,16 @@ Appellation: specs Contrib: FL03 */ + +pub trait Pair { + fn pair(&self) -> (A, B); +} + +impl Pair for T +where + T: Clone + Into<(A, B)>, +{ + fn pair(&self) -> (A, B) { + self.clone().into() + } +} diff --git a/core/src/utils.rs b/core/src/utils.rs index c57e8121..47b00354 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -1,9 +1,22 @@ /* Appellation: utils Contrib: FL03 - Description: ... Summary ... */ +use ndarray::{concatenate, Array, Axis, RemoveAxis}; +// use num::Float; +pub fn concat_iter(axis: usize, iter: impl IntoIterator>) -> Array +where + D: RemoveAxis, + T: Clone, +{ + let mut arr = iter.into_iter().collect::>(); + let mut out = arr.pop().unwrap(); + for i in arr { + out = concatenate!(Axis(axis), out, i); + } + out +} pub fn now() -> u128 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) diff --git a/data/src/lib.rs b/data/src/lib.rs index f736b907..f02f2fcd 100644 --- a/data/src/lib.rs +++ b/data/src/lib.rs @@ -11,6 +11,7 @@ pub(crate) mod utils; pub mod df; pub mod flows; +pub mod linear; pub mod tensors; pub mod prelude { diff --git a/data/src/linear/layer.rs b/data/src/linear/layer.rs new file mode 100644 index 00000000..379b9047 --- /dev/null +++ b/data/src/linear/layer.rs @@ -0,0 +1,23 @@ +/* + Appellation: layer + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "lowercase")] +pub struct Layer { + data: Vec, +} + +impl Layer { + pub fn new() -> Self { + Self { data: Vec::new() } + } +} + +impl std::fmt::Display for Layer { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", serde_json::to_string(self).unwrap()) + } +} diff --git a/data/src/linear/mod.rs b/data/src/linear/mod.rs new file mode 100644 index 00000000..3087b792 --- /dev/null +++ b/data/src/linear/mod.rs @@ -0,0 +1,10 @@ +/* + Appellation: linear + Contrib: FL03 +*/ +//! # Linear Layers +pub use self::{layer::*, utils::*}; + +pub(crate) mod layer; + +pub(crate) mod utils {} diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 35cc7928..51ce7e6c 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -18,7 +18,7 @@ where let z = args.dot(self.weights()) - self.bias(); z.mapv(|x| self.activator().activate(x)) } - + fn activator(&self) -> &Self::Activator; fn bias(&self) -> &Array2; diff --git a/ml/transformers/Cargo.toml b/ml/transformers/Cargo.toml index d3730131..701ca784 100644 --- a/ml/transformers/Cargo.toml +++ b/ml/transformers/Cargo.toml @@ -24,7 +24,9 @@ test = true [build-dependencies] [dependencies] +concision-core.workspace = true concision-neural = { path = "../neural" } + anyhow.workspace = true lazy_static.workspace = true ndarray.workspace = true diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index 4489f2a5..870b484b 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -2,47 +2,100 @@ Appellation: head Contrib: FL03 */ -use super::params::AttentionDim; -use super::Context; +use super::params::{HeadShape, QKV}; +use super::{AttentionSpace, HeadSpace}; use crate::neural::neurons::activate::{Activator, Softmax}; -use ndarray::Array2; +use ndarray::{Array2, IntoDimension}; use serde::{Deserialize, Serialize}; +use std::ops; #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] #[serde(rename_all = "lowercase")] pub struct AttentionHead { - context: Context, - dim: AttentionDim, + dim: HeadShape, + mask: Array2, + + score: Array2, + weights: AttentionSpace, } impl AttentionHead { - pub fn new(dim: AttentionDim) -> Self { + pub fn new(dim: HeadShape) -> Self { Self { - context: Context::new(dim.head_dim()), dim, + mask: Array2::zeros((dim.sequence(), dim.sequence())), + score: Array2::zeros(dim.into_dimension()), + weights: AttentionSpace::new(dim), } } - pub fn attention(&mut self, data: &Array2) -> Array2 { - self.context *= data; + pub fn attention(&mut self, data: &Array2) -> Array2{ + // multiply the data by the wieghted query, key, and value matrices, respectively + let weighted = self.weights.clone() * data; + let (q, k, v) = weighted.qkv(); - Softmax::rho(self.query().dot(&self.key().t()) * self.scale()) * self.value().clone() + // compute the attention score + let score = { + let inner = (q.dot(&k.t()) + self.mask.clone()) * self.scale(); + Softmax::rho(inner).dot(&v) + }; + self.score = score.clone(); + score + } + + pub fn dim(&self) -> HeadShape { + self.dim + } + + pub fn mask(&self) -> &Array2 { + &self.mask + } + + pub fn mask_mut(&mut self) -> &mut Array2 { + &mut self.mask + } + + pub fn process(&mut self, data: &Array2) { + // multiply the data by the wieghted query, key, and value matrices, respectively + let weighted = self.weights.clone() * data; + let (q, k, v) = weighted.qkv(); + + // compute the attention score + self.score = { + let inner = (q.dot(&k.t()) + self.mask.clone()) * self.scale(); + Softmax::rho(inner).dot(&v) + }; } pub fn scale(&self) -> f64 { 1.0 / (self.dim.query_size() as f64).sqrt() } - pub fn query(&self) -> &Array2 { - &self.context.query + pub fn score(&self) -> &Array2 { + &self.score + } + + pub fn set_mask(&mut self, mask: Array2) { + self.mask = mask; + } + + pub fn with_mask(mut self, mask: Array2) -> Self { + self.mask = mask; + self + } +} + +impl HeadSpace for AttentionHead { + fn query(&self) -> &Array2 { + &self.weights.query } - pub fn key(&self) -> &Array2 { - &self.context.key + fn key(&self) -> &Array2 { + &self.weights.key } - pub fn value(&self) -> &Array2 { - &self.context.value + fn value(&self) -> &Array2 { + &self.weights.value } } @@ -51,3 +104,55 @@ impl std::fmt::Display for AttentionHead { write!(f, "{}", serde_json::to_string(self).unwrap()) } } + +impl ops::Index for AttentionHead { + type Output = Array2; + + fn index(&self, index: QKV) -> &Self::Output { + &self.weights[index] + } +} + +impl ops::IndexMut for AttentionHead { + fn index_mut(&mut self, index: QKV) -> &mut Self::Output { + &mut self.weights[index] + } +} + +impl ops::Mul> for AttentionHead { + type Output = AttentionHead; + + fn mul(self, rhs: Array2) -> Self::Output { + let mut head = self.clone(); + head.weights = self.weights * rhs; + head + } +} + +impl ops::Mul<&Array2> for AttentionHead { + type Output = AttentionHead; + + fn mul(self, rhs: &Array2) -> Self::Output { + let mut head = self.clone(); + head.weights = self.weights * rhs; + head + } +} + +impl ops::MulAssign> for AttentionHead { + fn mul_assign(&mut self, rhs: Array2) { + self.weights *= rhs; + } +} + +impl ops::MulAssign<&Array2> for AttentionHead { + fn mul_assign(&mut self, rhs: &Array2) { + self.weights *= rhs; + } +} + +impl ops::MulAssign<&Array2> for &mut AttentionHead { + fn mul_assign(&mut self, rhs: &Array2) { + self.weights *= rhs; + } +} diff --git a/ml/transformers/src/attention/mod.rs b/ml/transformers/src/attention/mod.rs index 34ac5868..626338b4 100644 --- a/ml/transformers/src/attention/mod.rs +++ b/ml/transformers/src/attention/mod.rs @@ -8,16 +8,17 @@ //! //! -pub use self::{context::*, head::*, utils::*}; +pub use self::{head::*, reps::*, utils::*}; -pub(crate) mod context; pub(crate) mod head; +pub(crate) mod reps; pub mod multi; +pub mod ops; pub mod params; use ndarray::{Array, Array2}; -use std::ops; +use std::ops::MulAssign; pub type BaseDim = ndarray::Dim<[usize; 4]>; @@ -26,47 +27,51 @@ pub type InputArray = Array; pub type AttentionArray = Array2; -pub trait Representations { - fn query(&self) -> &InputArray; - fn key(&self) -> &InputArray; - fn value(&self) -> &InputArray; -} - pub trait Attention { - type Head: Head; + type Head: HeadSpace; fn head(&self) -> &Self::Head; - } -pub trait Head: ops::MulAssign> { - fn query(&self) -> &AttentionArray; - fn key(&self) -> &AttentionArray; - fn value(&self) -> &AttentionArray; - - +pub trait HeadSpace: MulAssign> { + fn query(&self) -> &Array2; + fn key(&self) -> &Array2; + fn value(&self) -> &Array2; } +pub trait Context { + type Dim: ndarray::Dimension; + fn query(&self) -> &Array; + fn key(&self) -> &Array; + fn value(&self) -> &Array; +} pub(crate) mod utils { + use super::ops::Split; use crate::neural::prelude::activate::{Activator, Softmax}; - use ndarray::Array2; + use ndarray::ShapeError; + use ndarray::prelude::{Array2, Array3}; + pub fn linear_layer( data: &Array2, weights: &Array2, - ) -> Array2 { - data.dot(weights) + heads: usize, + ) -> Result, ShapeError> { + data.dot(weights).split(heads) } pub fn compute_attention( query: &Array2, key: &Array2, value: &Array2, + mask: Option>, ) -> Array2 { - let dk = query.shape()[1] as f64; - Softmax::rho(query.dot(&key.t()) / dk.sqrt()) * value + let (seq, dk) = query.dim(); + let mask = mask.unwrap_or_else(|| Array2::::zeros((seq, seq))); + let scale = 1.0 / (dk as f64).sqrt(); + Softmax::rho((query.dot(&key.t()) + mask) * scale).dot(value) } } diff --git a/ml/transformers/src/attention/multi/attention.rs b/ml/transformers/src/attention/multi/attention.rs index 34cd6fdb..259f5d65 100644 --- a/ml/transformers/src/attention/multi/attention.rs +++ b/ml/transformers/src/attention/multi/attention.rs @@ -2,3 +2,5 @@ Appellation: attention Contrib: FL03 */ + +pub struct MultiHeadAttention {} diff --git a/ml/transformers/src/attention/multi/mod.rs b/ml/transformers/src/attention/multi/mod.rs index d84274a1..8ec7e454 100644 --- a/ml/transformers/src/attention/multi/mod.rs +++ b/ml/transformers/src/attention/multi/mod.rs @@ -6,10 +6,62 @@ pub use self::{attention::*, utils::*}; pub(crate) mod attention; -pub trait Split { - fn split(&self) -> Vec - where - Self: Sized; +use crate::attention::params::MultiShape; +use crate::attention::AttentionHead; +// use crate::core::concat_iter; +use ndarray::prelude::{Array2, Array4}; +use ndarray::{concatenate, Axis}; + +pub trait MultiHead { + fn dim(&self) -> MultiShape; + + fn attention(&self) -> &[AttentionHead]; + + fn heads_mut(&mut self) -> &mut [AttentionHead]; + + fn process(&mut self, data: &Array2); + + fn score(&mut self, data: &Array2) -> Array2 { + let scores = self + .attention() + .iter() + .map(|head| head.score()) + .collect::>(); + let mut score: Array2 = scores[0].clone(); + for i in 1..scores.len() { + score = concatenate!(Axis(0), score, scores[i].clone()); + } + score + } +} + +pub(crate) mod utils { + use crate::attention::compute_attention; + use crate::attention::ops::Merge; + use ndarray::prelude::{Array2, Array3, Array4}; + use ndarray::{s, ShapeError}; + + pub fn multihead( + query: &Array4, + key: &Array4, + value: &Array4, + mask: Option>, + ) -> Result, ShapeError> { + let (batch, heads, seq, _) = query.dim(); + let mask = mask.unwrap_or_else(|| Array2::::zeros((seq, seq))); + let mut score = Array4::::zeros(query.dim()); + for i in 0..batch { + for h in 0..heads { + let q = query.slice(s![i, h, .., ..]).to_owned(); + let k = key.slice(s![i, h, .., ..]).to_owned(); + let v = value.slice(s![i, h, .., ..]).to_owned(); + let head = compute_attention(&q, &k, &v, Some(mask.clone())); + score.slice_mut(s![i, h, .., ..]).assign(&head); + } + } + score.merge() + } } -pub(crate) mod utils {} +#[cfg(test)] +mod tests {} diff --git a/ml/transformers/src/attention/ops/merge.rs b/ml/transformers/src/attention/ops/merge.rs new file mode 100644 index 00000000..e8c246db --- /dev/null +++ b/ml/transformers/src/attention/ops/merge.rs @@ -0,0 +1,38 @@ +/* + Appellation: split + Contrib: FL03 +*/ +use ndarray::prelude::{Array2, Array3, Array4}; +use ndarray::ShapeError; + +pub trait Merge { + type Error; + + fn merge(&self) -> Result; +} + +impl Merge> for Array3 { + type Error = ShapeError; + + fn merge(&self) -> Result, Self::Error> { + let (heads, seq, query) = self.dim(); + let mut tmp = self.clone(); + // swap the head and sequence axes + tmp.swap_axes(0, 1); + // reshape the qkv matrix into a 2d array + tmp.into_shape((seq, heads * query)) + } +} + +impl Merge> for Array4 { + type Error = ShapeError; + + fn merge(&self) -> Result, Self::Error> { + let (batch, heads, seq, query) = self.dim(); + let mut tmp = self.clone(); + // swap the head and sequence axes + tmp.swap_axes(1, 2); + // reshape the qkv matrix into a 2d array + tmp.into_shape((batch, seq, heads * query)) + } +} diff --git a/ml/transformers/src/attention/ops/mod.rs b/ml/transformers/src/attention/ops/mod.rs new file mode 100644 index 00000000..4d523d2a --- /dev/null +++ b/ml/transformers/src/attention/ops/mod.rs @@ -0,0 +1,90 @@ +/* + Appellation: ops + Contrib: FL03 +*/ +pub use self::{merge::*, split::*, utils::*}; + +pub(crate) mod merge; +pub(crate) mod split; + +pub(crate) mod utils { + use ndarray::prelude::{Array2, Array3, Array4}; + use ndarray::ShapeError; + // use num::Float; + + pub fn merge_heads(heads: &Array3) -> Result, ShapeError> + where + T: Clone, + { + let (n, seq, query) = heads.dim(); + let mut tmp = heads.clone(); + // swap the head and sequence axes + tmp.swap_axes(0, 1); + // reshape the qkv matrix into a 2d array + tmp.into_shape((seq, n * query)) + } + + pub fn split_heads(param: &Array2, num_heads: usize) -> Result, ShapeError> + where + T: Clone, + { + let dim = param.shape().last().unwrap() / num_heads; + // reshape the qkv matrix into a 3d array + let mut res = param + .clone() + .into_shape((param.shape()[0], num_heads, dim))?; + // swap the sequence and head axes + res.swap_axes(0, 1); + Ok(res) + } + + pub fn merge_batch(heads: &Array4) -> Result, ShapeError> + where + T: Clone, + { + let (batch, n, seq, query) = ( + heads.shape()[0], + heads.shape()[1], + heads.shape()[2], + heads.shape()[3], + ); + let mut tmp = heads.clone(); + // swap the head and sequence axes + tmp.swap_axes(1, 2); + // reshape the qkv matrix into a 2d array + tmp.into_shape((batch, seq, n * query)) + } + + pub fn split_batch(param: &Array3, num_heads: usize) -> Result, ShapeError> + where + T: Clone, + { + let dim = param.shape().last().unwrap() / num_heads; + // reshape the qkv matrix into a 3d array + let mut res = + param + .clone() + .into_shape((param.shape()[0], param.shape()[1], num_heads, dim))?; + // swap the sequence and head axes + res.swap_axes(1, 2); + Ok(res) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ndarray::Array; + + #[test] + fn reshape_ops() { + let dim_input: [usize; 3] = [2, 4, 6]; // (batch, seq, model) + let dim_split = [2, 2, 4, 3]; // (batch, heads, seq, model) + let data = Array::linspace(1., 48., 48).into_shape(dim_input).unwrap(); + + let a = split_batch(&data, 2).unwrap(); + assert_eq!(a.shape(), &dim_split); + let b = merge_batch(&a).unwrap(); + assert_eq!(b.shape(), &dim_input); + } +} diff --git a/ml/transformers/src/attention/ops/split.rs b/ml/transformers/src/attention/ops/split.rs new file mode 100644 index 00000000..a52154c5 --- /dev/null +++ b/ml/transformers/src/attention/ops/split.rs @@ -0,0 +1,40 @@ +/* + Appellation: split + Contrib: FL03 +*/ +use ndarray::prelude::{Array2, Array3, Array4}; +use ndarray::ShapeError; + +pub trait Split { + type Error; + + fn split(&self, heads: usize) -> Result; +} + +impl Split> for Array2 { + type Error = ShapeError; + + fn split(&self, heads: usize) -> Result, Self::Error> { + let (seq, model) = self.dim(); + let query = model / heads; + // reshape the qkv matrix into a 3d array + let mut res = self.clone().into_shape((seq, heads, query))?; + // swap the sequence and head axes + res.swap_axes(0, 1); + Ok(res) + } +} + +impl Split> for Array3 { + type Error = ShapeError; + + fn split(&self, heads: usize) -> Result, Self::Error> { + let (batch, seq, model) = self.dim(); + let query = model / heads; + // reshape the qkv matrix into a 3d array + let mut res = self.clone().into_shape((batch, seq, heads, query))?; + // swap the sequence and head axes + res.swap_axes(1, 2); + Ok(res) + } +} diff --git a/ml/transformers/src/attention/params/dim.rs b/ml/transformers/src/attention/params/dim.rs index ef7d13e7..803c8654 100644 --- a/ml/transformers/src/attention/params/dim.rs +++ b/ml/transformers/src/attention/params/dim.rs @@ -10,71 +10,124 @@ //! - `batch`: The batch size //! - `heads`: The number of attention heads //! - `model`: The dimension of the model (embedding size) -use crate::{DEFAULT_ATTENTION_HEADS, DEFAULT_EMBEDDING_SIZE, DEFAULT_SAMPLE_SIZE}; +use crate::{DEFAULT_ATTENTION_HEADS, DEFAULT_EMBEDDING_SIZE}; +use ndarray::IntoDimension; use serde::{Deserialize, Serialize}; +pub trait StructuredDim: IntoDimension {} + +pub trait BaseDimension: IntoDimension { + fn batch_size(&self) -> usize; + fn model_size(&self) -> usize; + fn seq_len(&self) -> usize; +} + +pub trait MultiHeadDimension: BaseDimension { + fn heads(&self) -> usize; +} + +impl BaseDimension for D +where + D: Clone + IntoDimension, +{ + fn batch_size(&self) -> usize { + self.clone().into_dimension()[0] + } + + fn model_size(&self) -> usize { + self.clone().into_dimension()[2] + } + + fn seq_len(&self) -> usize { + self.clone().into_dimension()[1] + } +} + pub enum AttentionDims { - Head { - query: usize, - seq: usize, - }, - Context { - batch: usize, - heads: usize, - model: usize, - seq: usize, - samples: usize, - }, + Base(BaseShape), // a 3d matrix (batch, seq, model) + Head(HeadShape), // a 2d matrix (seq, query) + MultiHead(MultiShape), // a 4d matrix (batch, heads, seq, query) } #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] -pub struct AttentionDim { - pub batch: usize, // The batch size - pub heads: usize, // The number of attention heads - pub model: usize, // The dimension of the model (embedding size) - pub seq: usize, // The sequence length - pub samples: usize, // The number of samples +pub struct BaseShape { + pub batch: usize, + pub seq: usize, + pub model: usize, +} + +impl BaseShape { + pub fn new(batch: usize, seq: usize, model: usize) -> Self { + Self { batch, seq, model } + } + + pub fn std(batch: usize, seq: usize) -> Self { + Self::new(batch, seq, DEFAULT_EMBEDDING_SIZE) + } + + pub fn batch_size(&self) -> usize { + self.batch + } + + pub fn model_size(&self) -> usize { + self.model + } + + pub fn seq_len(&self) -> usize { + self.seq + } } -impl AttentionDim { - pub fn new(batch: usize, heads: usize, model: usize, seq: usize, samples: usize) -> Self { +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] +pub struct MultiShape { + pub batch: usize, + pub heads: usize, + pub seq: usize, + pub query: usize, +} + +impl MultiShape { + pub fn new(batch: usize, heads: usize, seq: usize, query: usize) -> Self { Self { batch, heads, - model, seq, - samples, + query, } } pub fn std(batch: usize, seq: usize) -> Self { - Self::new( - batch, - DEFAULT_ATTENTION_HEADS, - DEFAULT_EMBEDDING_SIZE, - seq, - DEFAULT_SAMPLE_SIZE, - ) + Self::new(batch, DEFAULT_ATTENTION_HEADS, seq, DEFAULT_EMBEDDING_SIZE) } - // The dimension of the key, query, and value vectors - pub fn query_size(&self) -> usize { - self.model / self.heads + pub fn batch_size(&self) -> usize { + self.batch + } + + pub fn heads(&self) -> usize { + self.heads } - pub fn head_dim(&self) -> (usize, usize) { - (self.seq, self.query_size()) + pub fn model_size(&self) -> usize { + self.query + } + + pub fn seq_len(&self) -> usize { + self.seq } } +/// #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] pub struct HeadShape { - pub query: usize, - pub seq: usize, + pub query: usize, // cols + pub seq: usize, // rows } impl HeadShape { @@ -82,7 +135,41 @@ impl HeadShape { Self { query, seq } } + pub fn query_size(&self) -> usize { + self.query + } + + pub fn sequence(&self) -> usize { + self.seq + } + pub fn scale(&self) -> f64 { 1.0 / (self.query as f64).sqrt() } } + +impl From<(usize, usize)> for HeadShape { + fn from((seq, query): (usize, usize)) -> Self { + Self::new(query, seq) + } +} + +impl From<[usize; 2]> for HeadShape { + fn from(dim: [usize; 2]) -> Self { + Self::new(dim[1], dim[0]) + } +} + +impl From for (usize, usize) { + fn from(shape: HeadShape) -> Self { + (shape.seq, shape.query) + } +} + +impl IntoDimension for HeadShape { + type Dim = ndarray::Ix2; + + fn into_dimension(self) -> Self::Dim { + ndarray::Ix2(self.seq, self.query) + } +} diff --git a/ml/transformers/src/attention/params/mod.rs b/ml/transformers/src/attention/params/mod.rs index 891cb844..9cb6b0d2 100644 --- a/ml/transformers/src/attention/params/mod.rs +++ b/ml/transformers/src/attention/params/mod.rs @@ -5,21 +5,88 @@ //! # Attention Parameters //! //! ## Hyperparameters -pub use self::{dim::*, utils::*}; +pub use self::{dim::*, qkv::*, utils::*}; pub(crate) mod dim; +pub(crate) mod qkv; +use crate::{DEFAULT_ATTENTION_HEADS, DEFAULT_EMBEDDING_SIZE, DEFAULT_SAMPLE_SIZE}; use serde::{Deserialize, Serialize}; #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] -pub struct AttentionParams { +pub struct Hyperparameters { pub batch: usize, - pub dim: AttentionDim, pub heads: usize, + pub model: usize, pub samples: usize, + pub seq: usize, +} + +impl Hyperparameters { + pub fn new(batch: usize, heads: usize, model: usize, samples: usize, seq: usize) -> Self { + Self { + batch, + heads, + model, + samples, + seq, + } + } + + pub fn std(batch: usize, seq: usize) -> Self { + Self::new( + batch, + DEFAULT_ATTENTION_HEADS, + DEFAULT_EMBEDDING_SIZE, + DEFAULT_SAMPLE_SIZE, + seq, + ) + } + + pub fn batch_size(&self) -> usize { + self.batch + } + + pub fn heads(&self) -> usize { + self.heads + } + + pub fn model_size(&self) -> usize { + self.model + } + + pub fn samples(&self) -> usize { + self.samples + } + + pub fn seq_len(&self) -> usize { + self.seq + } + + pub fn query_size(&self) -> usize { + self.model / self.heads + } +} + +impl From for BaseShape { + fn from(hyper: Hyperparameters) -> Self { + Self::new(hyper.batch, hyper.seq, hyper.model) + } +} + +impl From for MultiShape { + fn from(hyper: Hyperparameters) -> Self { + Self::new(hyper.batch, hyper.heads, hyper.seq, hyper.model) + } +} + +impl From for HeadShape { + fn from(hyper: Hyperparameters) -> Self { + Self::new(hyper.seq, hyper.query_size()) + } } pub(crate) mod utils {} diff --git a/ml/transformers/src/attention/params/qkv.rs b/ml/transformers/src/attention/params/qkv.rs new file mode 100644 index 00000000..79f8f577 --- /dev/null +++ b/ml/transformers/src/attention/params/qkv.rs @@ -0,0 +1,57 @@ +/* + Appellation: qkv + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum QKV { + #[serde(alias = "k")] + #[strum(serialize = "k", serialize = "key")] + Key, + #[default] + #[serde(alias = "q")] + #[strum(serialize = "q", serialize = "query")] + Query, + #[serde(alias = "v")] + #[strum(serialize = "v", serialize = "val", serialize = "value")] + Value, +} + + + +#[cfg(test)] +mod tests { + use super::*; + use std::str::FromStr; + + #[test] + fn test_qkv() { + use QKV::Key; + let w = Key; + assert_eq!(w.to_string(), "key"); + assert_eq!(Key, QKV::from_str("key").unwrap()); + assert_eq!(Key, QKV::from_str("k").unwrap()); + } +} diff --git a/ml/transformers/src/attention/context.rs b/ml/transformers/src/attention/reps.rs similarity index 51% rename from ml/transformers/src/attention/context.rs rename to ml/transformers/src/attention/reps.rs index 8052b6f0..51dbfab1 100644 --- a/ml/transformers/src/attention/context.rs +++ b/ml/transformers/src/attention/reps.rs @@ -2,51 +2,24 @@ Appellation: context Contrib: FL03 */ +use super::params::QKV; use ndarray::{Array2, IntoDimension, Ix2}; use serde::{Deserialize, Serialize}; use std::ops; -use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; - -#[derive( - Clone, - Copy, - Debug, - Default, - Deserialize, - Display, - EnumIs, - EnumIter, - EnumString, - EnumVariantNames, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize, -)] -#[repr(usize)] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum QKV { - #[serde(alias = "k")] - Key, - #[default] - #[serde(alias = "q")] - Query, - #[serde(alias = "v")] - Value, + +pub trait LinearLayer { + fn matmul(&self, data: &Array2) -> Array2; } #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -pub struct Context { +pub struct AttentionSpace { dim: Ix2, pub key: Array2, pub query: Array2, pub value: Array2, } -impl Context { +impl AttentionSpace { pub fn new(dim: D) -> Self where D: IntoDimension, @@ -68,20 +41,38 @@ impl Context { self.query.dot(&self.key.t()) } - pub fn with_weight(&mut self, query: &Array2, key: &Array2, value: &Array2) { + pub fn qkv(&self) -> (Array2, Array2, Array2) { + (self.query.clone(), self.key.clone(), self.value.clone()) + } + + pub fn set_weight(&mut self, qkv: QKV, weight: Array2) { + match qkv { + QKV::Key => self.key = weight, + QKV::Query => self.query = weight, + QKV::Value => self.value = weight, + } + } + + pub fn set_weights(&mut self, qkv: impl IntoIterator, weight: Array2) { + for qkv in qkv { + self.set_weight(qkv, weight.clone()); + } + } + + pub fn with_weight(mut self, query: &Array2, key: &Array2, value: &Array2) { self.key = key.clone(); self.query = query.clone(); self.value = value.clone(); } } -impl std::fmt::Display for Context { +impl std::fmt::Display for AttentionSpace { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "{}", serde_json::to_string(self).unwrap()) } } -impl From for Context +impl From for AttentionSpace where D: IntoDimension, { @@ -97,13 +88,13 @@ where } } -impl From for (Array2, Array2, Array2) { - fn from(context: Context) -> Self { +impl From for (Array2, Array2, Array2) { + fn from(context: AttentionSpace) -> Self { (context.key, context.query, context.value) } } -impl ops::Index for Context { +impl ops::Index for AttentionSpace { type Output = Array2; fn index(&self, index: QKV) -> &Self::Output { @@ -116,7 +107,7 @@ impl ops::Index for Context { } } -impl ops::IndexMut for Context { +impl ops::IndexMut for AttentionSpace { fn index_mut(&mut self, index: QKV) -> &mut Self::Output { match index { QKV::Key => &mut self.key, @@ -126,31 +117,43 @@ impl ops::IndexMut for Context { } } -impl ops::Mul<&Array2> for Context { +impl ops::Mul> for AttentionSpace { + type Output = Self; + + fn mul(self, rhs: Array2) -> Self::Output { + let mut ctx = self.clone(); + ctx.key = ctx.key.dot(&rhs); + ctx.query = ctx.query.dot(&rhs); + ctx.value = ctx.value.dot(&rhs); + ctx + } +} + +impl ops::Mul<&Array2> for AttentionSpace { type Output = Self; fn mul(self, rhs: &Array2) -> Self::Output { - let mut context = self.clone(); - context.key = context.key.dot(rhs); - context.query = context.query.dot(rhs); - context.value = context.value.dot(rhs); - context + let mut ctx = self.clone(); + ctx.key = ctx.key.dot(rhs); + ctx.query = ctx.query.dot(rhs); + ctx.value = ctx.value.dot(rhs); + ctx } } -impl ops::Mul<&Array2> for &Context { - type Output = Context; +impl ops::Mul<&Array2> for &AttentionSpace { + type Output = AttentionSpace; fn mul(self, rhs: &Array2) -> Self::Output { - let mut context = self.clone(); - context.key = context.key.dot(rhs); - context.query = context.query.dot(rhs); - context.value = context.value.dot(rhs); - context + let mut ctx = self.clone(); + ctx.key = ctx.key.dot(rhs); + ctx.query = ctx.query.dot(rhs); + ctx.value = ctx.value.dot(rhs); + ctx } } -impl ops::MulAssign> for Context { +impl ops::MulAssign> for AttentionSpace { fn mul_assign(&mut self, rhs: Array2) { self.key = self.key.dot(&rhs); self.query = self.query.dot(&rhs); @@ -158,7 +161,7 @@ impl ops::MulAssign> for Context { } } -impl ops::MulAssign<&Array2> for Context { +impl ops::MulAssign<&Array2> for AttentionSpace { fn mul_assign(&mut self, rhs: &Array2) { self.key = self.key.dot(rhs); self.query = self.query.dot(rhs); @@ -166,16 +169,10 @@ impl ops::MulAssign<&Array2> for Context { } } -#[cfg(test)] -mod tests { - use super::*; - use std::str::FromStr; - - #[test] - fn test_attention_weights() { - let w = QKV::Key; - assert_eq!(w.to_string(), "key"); - assert_eq!(QKV::Key, QKV::from_str("key").unwrap()); - assert_eq!(QKV::Key, QKV::from_str("k").unwrap()); +impl ops::MulAssign<&Array2> for &mut AttentionSpace { + fn mul_assign(&mut self, rhs: &Array2) { + self.key = self.key.dot(rhs); + self.query = self.query.dot(rhs); + self.value = self.value.dot(rhs); } } diff --git a/ml/transformers/src/ffn/network.rs b/ml/transformers/src/ffn/network.rs index 4538e334..743b6a04 100644 --- a/ml/transformers/src/ffn/network.rs +++ b/ml/transformers/src/ffn/network.rs @@ -3,12 +3,25 @@ Contrib: FL03 */ use crate::neural::neurons::activate::{Activator, ReLU}; -use ndarray::Array2; +use ndarray::{Array1, Array2}; /// All vectors have a dimension of (nodes, elem) pub fn ffn(data: Array2, bias: Array2, weights: Array2) -> Array2 { - let a = data * weights.row(0) + bias.row(0); - ReLU::rho(a) * weights.row(1) + bias.row(1) + let a = data.dot(&weights) + bias.clone(); + ReLU::rho(a).dot(&weights) + bias } -pub struct FFN {} +pub struct FFN { + bias: Array2, + weights: Array2, +} + +impl FFN { + pub fn new(bias: Array2, weights: Array2) -> Self { + Self { bias, weights } + } + + pub fn forward(&self, data: Array2) -> Array2 { + ffn(data, self.bias.clone(), self.weights.clone()) + } +} diff --git a/ml/transformers/src/lib.rs b/ml/transformers/src/lib.rs index 30766699..b934c0ff 100644 --- a/ml/transformers/src/lib.rs +++ b/ml/transformers/src/lib.rs @@ -14,6 +14,7 @@ pub mod codec; pub mod ffn; pub mod transform; +pub(crate) use concision_core as core; pub(crate) use concision_neural as neural; pub mod prelude { From da1095058a1e0fff1dec9fd8c046164fbe77cddb Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 29 Oct 2023 11:09:54 -0500 Subject: [PATCH 033/118] update Signed-off-by: FL03 --- core/src/lib.rs | 2 + core/src/states/mod.rs | 11 ++ core/src/states/state.rs | 63 +++++++ core/src/states/weighted.rs | 31 +++ data/src/linear/features.rs | 36 ++++ data/src/linear/layer.rs | 45 +++-- data/src/linear/mod.rs | 17 +- ml/neural/src/layers/layer.rs | 8 +- ml/neural/src/primitives.rs | 4 + ml/neural/src/prop/mod.rs | 14 ++ ml/transformers/Cargo.toml | 1 + ml/transformers/src/attention/head.rs | 97 ++++------ ml/transformers/src/attention/mod.rs | 57 +++--- .../src/attention/multi/attention.rs | 15 +- ml/transformers/src/attention/multi/mod.rs | 73 ++++--- ml/transformers/src/attention/params/dim.rs | 139 +++++++++++--- .../src/attention/params/hyperparams.rs | 114 +++++++++++ ml/transformers/src/attention/params/mod.rs | 94 ++------- ml/transformers/src/attention/params/qkv.rs | 2 - ml/transformers/src/attention/reps.rs | 178 ------------------ ml/transformers/src/attention/weights.rs | 170 +++++++++++++++++ ml/transformers/src/codec/decode/mod.rs | 3 +- ml/transformers/src/codec/decode/params.rs | 4 + ml/transformers/src/codec/encode/encoder.rs | 18 +- ml/transformers/src/codec/encode/mod.rs | 4 +- ml/transformers/src/codec/encode/params.rs | 11 ++ ml/transformers/src/codec/encode/stack.rs | 25 +++ ml/transformers/src/ffn/mod.rs | 3 +- ml/transformers/src/ffn/network.rs | 31 +-- ml/transformers/src/ffn/params.rs | 34 ++++ ml/transformers/src/lib.rs | 6 +- .../src/{attention => }/ops/merge.rs | 0 .../src/{attention => }/ops/mod.rs | 12 +- ml/transformers/src/ops/norm.rs | 0 .../src/{attention => }/ops/split.rs | 0 ml/transformers/src/primitives.rs | 33 ++-- ml/transformers/src/transform/mod.rs | 3 +- ml/transformers/src/transform/params.rs | 14 ++ 38 files changed, 918 insertions(+), 454 deletions(-) create mode 100644 core/src/states/mod.rs create mode 100644 core/src/states/state.rs create mode 100644 core/src/states/weighted.rs create mode 100644 data/src/linear/features.rs create mode 100644 ml/transformers/src/attention/params/hyperparams.rs delete mode 100644 ml/transformers/src/attention/reps.rs create mode 100644 ml/transformers/src/attention/weights.rs create mode 100644 ml/transformers/src/codec/decode/params.rs create mode 100644 ml/transformers/src/codec/encode/params.rs create mode 100644 ml/transformers/src/codec/encode/stack.rs create mode 100644 ml/transformers/src/ffn/params.rs rename ml/transformers/src/{attention => }/ops/merge.rs (100%) rename ml/transformers/src/{attention => }/ops/mod.rs (91%) create mode 100644 ml/transformers/src/ops/norm.rs rename ml/transformers/src/{attention => }/ops/split.rs (100%) create mode 100644 ml/transformers/src/transform/params.rs diff --git a/core/src/lib.rs b/core/src/lib.rs index ec57f2c4..a55f15ba 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -10,10 +10,12 @@ pub(crate) mod specs; pub(crate) mod utils; pub mod errors; +pub mod states; pub mod step; pub mod prelude { pub use crate::errors::*; + pub use crate::states::*; pub use crate::step::*; pub use crate::primitives::*; diff --git a/core/src/states/mod.rs b/core/src/states/mod.rs new file mode 100644 index 00000000..561bedb4 --- /dev/null +++ b/core/src/states/mod.rs @@ -0,0 +1,11 @@ +/* + Appellation: states + Contrib: FL03 +*/ +pub use self::{state::*, utils::*}; + +pub(crate) mod state; + +pub mod weighted; + +pub(crate) mod utils {} diff --git a/core/src/states/state.rs b/core/src/states/state.rs new file mode 100644 index 00000000..7b7dbda1 --- /dev/null +++ b/core/src/states/state.rs @@ -0,0 +1,63 @@ +/* + Appellation: state + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; + + +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "lowercase")] +pub struct State { + pub kind: String, + pub message: String, + pub ts: u128, +} + +impl State { + pub fn new(kind: impl ToString, message: String) -> Self { + let ts = crate::now(); + Self { kind: kind.to_string(), message, ts } + } + + pub fn kind(&self) -> &str { + &self.kind + } + + pub fn message(&self) -> &str { + &self.message + } + + pub fn ts(&self) -> u128 { + self.ts + } + + pub fn set_kind(&mut self, kind: impl ToString) { + self.kind = kind.to_string(); + self.on_update(); + } + + pub fn set_message(&mut self, message: String) { + self.message = message; + self.on_update(); + } + + pub fn with_kind(mut self, kind: impl ToString) -> Self { + self.kind = kind.to_string(); + self + } + + pub fn with_message(mut self, message: String) -> Self { + self.message = message; + self + } + + fn on_update(&mut self) { + self.ts = crate::now(); + } +} + +impl std::fmt::Display for State { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", serde_json::to_string(self).unwrap()) + } +} diff --git a/core/src/states/weighted.rs b/core/src/states/weighted.rs new file mode 100644 index 00000000..aee6a61c --- /dev/null +++ b/core/src/states/weighted.rs @@ -0,0 +1,31 @@ +/* + Appellation: state + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use smart_default::SmartDefault; +use strum::{Display, EnumIs, EnumIter, EnumVariantNames}; + +#[derive( + Clone, + Debug, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, + SmartDefault, +)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum Weighted { + #[default] + Unweighted, + Weighted, +} diff --git a/data/src/linear/features.rs b/data/src/linear/features.rs new file mode 100644 index 00000000..edb3fd1e --- /dev/null +++ b/data/src/linear/features.rs @@ -0,0 +1,36 @@ +/* + Appellation: features + Contrib: FL03 +*/ +use ndarray::IntoDimension; +use serde::{Deserialize, Serialize}; + +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] +pub struct Features { + pub input: usize, + pub output: usize, +} + +impl Features { + pub fn new(input: usize, output: usize) -> Self { + Self { input, output } + } + + pub fn input(&self) -> usize { + self.input + } + + pub fn output(&self) -> usize { + self.output + } +} + +impl IntoDimension for Features { + type Dim = ndarray::IxDyn; + + fn into_dimension(self) -> Self::Dim { + ndarray::IxDyn(&[self.input, self.output]) + } +} diff --git a/data/src/linear/layer.rs b/data/src/linear/layer.rs index 379b9047..8375b3c0 100644 --- a/data/src/linear/layer.rs +++ b/data/src/linear/layer.rs @@ -2,22 +2,45 @@ Appellation: layer Contrib: FL03 */ +use super::Features; +use ndarray::prelude::{Array1, Array2}; +use num::Float; use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] -#[serde(rename_all = "lowercase")] -pub struct Layer { - data: Vec, +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +pub struct LinearLayer { + bias: Array1, + pub params: Features, + weights: Array2, } -impl Layer { - pub fn new() -> Self { - Self { data: Vec::new() } +impl LinearLayer +where + T: Float, +{ + pub fn new(inputs: usize, outputs: usize) -> Self { + let params = Features::new(inputs, outputs); + let weights = Array2::ones((params.input, params.output)); + let bias = Array1::ones(params.output); + Self { + bias, + params, + weights, + } + } + + pub fn bias(&self) -> &Array1 { + &self.bias + } + + pub fn linear(&self, data: &Array2) -> Array2 + where + T: 'static, + { + data.dot(&self.weights.t()) + &self.bias } -} -impl std::fmt::Display for Layer { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{}", serde_json::to_string(self).unwrap()) + pub fn weights(&self) -> &Array2 { + &self.weights } } diff --git a/data/src/linear/mod.rs b/data/src/linear/mod.rs index 3087b792..bb72fa72 100644 --- a/data/src/linear/mod.rs +++ b/data/src/linear/mod.rs @@ -3,8 +3,21 @@ Contrib: FL03 */ //! # Linear Layers -pub use self::{layer::*, utils::*}; +pub use self::{features::*, layer::*, utils::*}; +pub(crate) mod features; pub(crate) mod layer; -pub(crate) mod utils {} + +pub(crate) mod utils { + use ndarray::prelude::{Array1, Array2}; + use num::Float; + + pub fn linear_transformation( + data: &Array2, + bias: &Array1, + weights: &Array2, + ) -> Array2 { + data.dot(&weights.t()) + bias + } +} diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 51ce7e6c..e229f9a9 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -5,7 +5,7 @@ use super::LayerType; use crate::neurons::activate::Activator; use crate::neurons::Node; -use ndarray::Array2; +use ndarray::prelude::{Array1, Array2}; use std::ops; pub trait L @@ -28,12 +28,14 @@ where #[derive(Clone, Debug, Default, PartialEq)] pub struct Layer { + bias: Array1, layer: LayerType, - nodes: Vec, + nodes: usize, + weights: Array2, } impl Layer { - pub fn new(layer: LayerType, nodes: Vec) -> Self { + pub fn new(layer: LayerType,) -> Self { Self { layer, nodes } } diff --git a/ml/neural/src/primitives.rs b/ml/neural/src/primitives.rs index 92a83797..88844d0e 100644 --- a/ml/neural/src/primitives.rs +++ b/ml/neural/src/primitives.rs @@ -13,4 +13,8 @@ pub(crate) mod statics {} pub(crate) mod types { pub type BoxedFunction = Box T>; + + pub type LayerBias = ndarray::Array1; + + pub type WeightTensor = ndarray::Array2; } diff --git a/ml/neural/src/prop/mod.rs b/ml/neural/src/prop/mod.rs index 12d73f26..6d4b2248 100644 --- a/ml/neural/src/prop/mod.rs +++ b/ml/neural/src/prop/mod.rs @@ -10,4 +10,18 @@ pub use self::{modes::*, propagation::*, utils::*}; pub(crate) mod modes; pub(crate) mod propagation; +// pub mod forward; + +pub trait Backward { + type Output; + + fn backward(&mut self, args: T) -> Self::Output; +} + +pub trait Forward { + type Output; + + fn forward(&mut self, args: T) -> Self::Output; +} + pub(crate) mod utils {} diff --git a/ml/transformers/Cargo.toml b/ml/transformers/Cargo.toml index 701ca784..7bce8b41 100644 --- a/ml/transformers/Cargo.toml +++ b/ml/transformers/Cargo.toml @@ -25,6 +25,7 @@ test = true [dependencies] concision-core.workspace = true +concision-data = { path = "../../data" } concision-neural = { path = "../neural" } anyhow.workspace = true diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index 870b484b..23cdd15c 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -3,156 +3,135 @@ Contrib: FL03 */ use super::params::{HeadShape, QKV}; -use super::{AttentionSpace, HeadSpace}; +use super::{Head, Weight}; use crate::neural::neurons::activate::{Activator, Softmax}; -use ndarray::{Array2, IntoDimension}; +use ndarray::ScalarOperand; +use ndarray::prelude::Array2; +use num::Float; use serde::{Deserialize, Serialize}; use std::ops; #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] #[serde(rename_all = "lowercase")] -pub struct AttentionHead { +pub struct AttentionHead { dim: HeadShape, - mask: Array2, - - score: Array2, - weights: AttentionSpace, + mask: Array2, + weights: Weight, } -impl AttentionHead { +impl AttentionHead { pub fn new(dim: HeadShape) -> Self { Self { dim, mask: Array2::zeros((dim.sequence(), dim.sequence())), - score: Array2::zeros(dim.into_dimension()), - weights: AttentionSpace::new(dim), + weights: Weight::new(dim), } } - pub fn attention(&mut self, data: &Array2) -> Array2{ + pub fn attention(&mut self, data: &Array2) -> Array2 { // multiply the data by the wieghted query, key, and value matrices, respectively let weighted = self.weights.clone() * data; let (q, k, v) = weighted.qkv(); // compute the attention score - let score = { - let inner = (q.dot(&k.t()) + self.mask.clone()) * self.scale(); - Softmax::rho(inner).dot(&v) - }; - self.score = score.clone(); - score + let inner = (q.dot(&k.t()) + self.mask.clone()) * self.scale(); + Softmax::rho(inner).dot(&v) } pub fn dim(&self) -> HeadShape { self.dim } - pub fn mask(&self) -> &Array2 { + pub fn mask(&self) -> &Array2 { &self.mask } - pub fn mask_mut(&mut self) -> &mut Array2 { + pub fn mask_mut(&mut self) -> &mut Array2 { &mut self.mask } - pub fn process(&mut self, data: &Array2) { - // multiply the data by the wieghted query, key, and value matrices, respectively - let weighted = self.weights.clone() * data; - let (q, k, v) = weighted.qkv(); - - // compute the attention score - self.score = { - let inner = (q.dot(&k.t()) + self.mask.clone()) * self.scale(); - Softmax::rho(inner).dot(&v) - }; - } - - pub fn scale(&self) -> f64 { - 1.0 / (self.dim.query_size() as f64).sqrt() - } - - pub fn score(&self) -> &Array2 { - &self.score + pub fn scale(&self) -> T { + T::one() / T::from(self.dim.query_size()).unwrap().sqrt() } - pub fn set_mask(&mut self, mask: Array2) { + pub fn set_mask(&mut self, mask: Array2) { self.mask = mask; } - pub fn with_mask(mut self, mask: Array2) -> Self { + pub fn with_mask(mut self, mask: Array2) -> Self { self.mask = mask; self } } -impl HeadSpace for AttentionHead { - fn query(&self) -> &Array2 { +impl Head for AttentionHead { + fn query(&self) -> &Array2 { &self.weights.query } - fn key(&self) -> &Array2 { + fn key(&self) -> &Array2 { &self.weights.key } - fn value(&self) -> &Array2 { + fn value(&self) -> &Array2 { &self.weights.value } } -impl std::fmt::Display for AttentionHead { +impl std::fmt::Display for AttentionHead { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "{}", serde_json::to_string(self).unwrap()) } } -impl ops::Index for AttentionHead { - type Output = Array2; +impl ops::Index for AttentionHead { + type Output = Array2; fn index(&self, index: QKV) -> &Self::Output { &self.weights[index] } } -impl ops::IndexMut for AttentionHead { +impl ops::IndexMut for AttentionHead { fn index_mut(&mut self, index: QKV) -> &mut Self::Output { &mut self.weights[index] } } -impl ops::Mul> for AttentionHead { - type Output = AttentionHead; +impl ops::Mul> for AttentionHead { + type Output = AttentionHead; - fn mul(self, rhs: Array2) -> Self::Output { + fn mul(self, rhs: Array2) -> Self::Output { let mut head = self.clone(); head.weights = self.weights * rhs; head } } -impl ops::Mul<&Array2> for AttentionHead { - type Output = AttentionHead; +impl ops::Mul<&Array2> for AttentionHead { + type Output = AttentionHead; - fn mul(self, rhs: &Array2) -> Self::Output { + fn mul(self, rhs: &Array2) -> Self::Output { let mut head = self.clone(); head.weights = self.weights * rhs; head } } -impl ops::MulAssign> for AttentionHead { - fn mul_assign(&mut self, rhs: Array2) { +impl ops::MulAssign> for AttentionHead { + fn mul_assign(&mut self, rhs: Array2) { self.weights *= rhs; } } -impl ops::MulAssign<&Array2> for AttentionHead { - fn mul_assign(&mut self, rhs: &Array2) { +impl ops::MulAssign<&Array2> for AttentionHead { + fn mul_assign(&mut self, rhs: &Array2) { self.weights *= rhs; } } -impl ops::MulAssign<&Array2> for &mut AttentionHead { - fn mul_assign(&mut self, rhs: &Array2) { +impl ops::MulAssign<&Array2> for &mut AttentionHead { + fn mul_assign(&mut self, rhs: &Array2) { self.weights *= rhs; } } diff --git a/ml/transformers/src/attention/mod.rs b/ml/transformers/src/attention/mod.rs index 626338b4..d5832e1b 100644 --- a/ml/transformers/src/attention/mod.rs +++ b/ml/transformers/src/attention/mod.rs @@ -8,39 +8,42 @@ //! //! -pub use self::{head::*, reps::*, utils::*}; +pub use self::{head::*, utils::*, weights::*}; pub(crate) mod head; -pub(crate) mod reps; +pub(crate) mod weights; pub mod multi; -pub mod ops; pub mod params; -use ndarray::{Array, Array2}; -use std::ops::MulAssign; +use crate::prelude::BaseDim; +use crate::core::prelude::BoxResult; -pub type BaseDim = ndarray::Dim<[usize; 4]>; +use ndarray::Dimension; +use ndarray::prelude::{Array, Ix2}; +use num::Float; /// (batch, sample, seq, model) pub type InputArray = Array; -pub type AttentionArray = Array2; +pub type AttentionArray = Array; -pub trait Attention { - type Head: HeadSpace; +pub trait Attention { + type Dim: Dimension; + type Score; - fn head(&self) -> &Self::Head; + fn attention(&mut self, data: &Array) -> BoxResult<&Array>; } -pub trait HeadSpace: MulAssign> { - fn query(&self) -> &Array2; - fn key(&self) -> &Array2; - fn value(&self) -> &Array2; +pub trait Head { + + fn query(&self) -> &Array; + fn key(&self) -> &Array; + fn value(&self) -> &Array; } -pub trait Context { - type Dim: ndarray::Dimension; +pub trait Spaces { + type Dim: Dimension; fn query(&self) -> &Array; fn key(&self) -> &Array; @@ -48,11 +51,11 @@ pub trait Context { } pub(crate) mod utils { - use super::ops::Split; + use crate::ops::Split; use crate::neural::prelude::activate::{Activator, Softmax}; - use ndarray::ShapeError; use ndarray::prelude::{Array2, Array3}; - + use ndarray::{ScalarOperand, ShapeError}; + use num::Float; pub fn linear_layer( data: &Array2, @@ -62,15 +65,15 @@ pub(crate) mod utils { data.dot(weights).split(heads) } - pub fn compute_attention( - query: &Array2, - key: &Array2, - value: &Array2, - mask: Option>, - ) -> Array2 { + pub fn compute_attention( + query: &Array2, + key: &Array2, + value: &Array2, + mask: Option>, + ) -> Array2 { let (seq, dk) = query.dim(); - let mask = mask.unwrap_or_else(|| Array2::::zeros((seq, seq))); - let scale = 1.0 / (dk as f64).sqrt(); + let mask = mask.unwrap_or_else(|| Array2::::zeros((seq, seq))); + let scale = T::one() / (T::from(dk).unwrap()).sqrt(); Softmax::rho((query.dot(&key.t()) + mask) * scale).dot(value) } } diff --git a/ml/transformers/src/attention/multi/attention.rs b/ml/transformers/src/attention/multi/attention.rs index 259f5d65..05054e48 100644 --- a/ml/transformers/src/attention/multi/attention.rs +++ b/ml/transformers/src/attention/multi/attention.rs @@ -2,5 +2,18 @@ Appellation: attention Contrib: FL03 */ +use crate::attention::params::AttentionParameters; +use crate::attention::Weight; -pub struct MultiHeadAttention {} +pub struct MultiHeadAttention { + heads: usize, + model: usize, + weights: Weight, +} + +impl MultiHeadAttention { + pub fn new(heads: usize, model: usize) -> Self { + let weights = Weight::new((model, model)); + Self { heads, model, weights } + } +} diff --git a/ml/transformers/src/attention/multi/mod.rs b/ml/transformers/src/attention/multi/mod.rs index 8ec7e454..19cbf068 100644 --- a/ml/transformers/src/attention/multi/mod.rs +++ b/ml/transformers/src/attention/multi/mod.rs @@ -6,42 +6,46 @@ pub use self::{attention::*, utils::*}; pub(crate) mod attention; +use crate::ops::Split; use crate::attention::params::MultiShape; -use crate::attention::AttentionHead; -// use crate::core::concat_iter; -use ndarray::prelude::{Array2, Array4}; -use ndarray::{concatenate, Axis}; +use crate::attention::Weight; +use crate::core::prelude::BoxResult; +use ndarray::prelude::{Array2, Array3}; +use ndarray::ScalarOperand; +use num::Float; -pub trait MultiHead { +pub trait MultiHead +where + T: Float + ScalarOperand, +{ + fn attention(&mut self, data: &Array2) -> BoxResult> { + let weighted = self.weights() * data; + let (q, k, v) = weighted.split(self.dim().heads())?; + let score = utils::multihead(&q, &k, &v, Some(self.mask().clone()))?; + Ok(score) + } + fn dim(&self) -> MultiShape; - fn attention(&self) -> &[AttentionHead]; + fn mask(&self) -> &Array2; - fn heads_mut(&mut self) -> &mut [AttentionHead]; + fn multihead(&self) -> &Array3; - fn process(&mut self, data: &Array2); + fn multihead_mut(&mut self) -> &mut Array3; - fn score(&mut self, data: &Array2) -> Array2 { - let scores = self - .attention() - .iter() - .map(|head| head.score()) - .collect::>(); - let mut score: Array2 = scores[0].clone(); - for i in 1..scores.len() { - score = concatenate!(Axis(0), score, scores[i].clone()); - } - score - } + fn weights(&self) -> &Weight; + + } pub(crate) mod utils { use crate::attention::compute_attention; - use crate::attention::ops::Merge; + use crate::ops::Merge; use ndarray::prelude::{Array2, Array3, Array4}; - use ndarray::{s, ShapeError}; + use ndarray::{s, ScalarOperand, ShapeError}; + use num::Float; - pub fn multihead( + pub fn batched_multihead( query: &Array4, key: &Array4, value: &Array4, @@ -61,6 +65,29 @@ pub(crate) mod utils { } score.merge() } + + pub fn multihead( + query: &Array3, + key: &Array3, + value: &Array3, + mask: Option>, + ) -> Result, ShapeError> + where + T: Float + ScalarOperand, + { + let (heads, seq, _) = query.dim(); + let mask = mask.unwrap_or_else(|| Array2::::zeros((seq, seq))); + let mut score = Array3::::zeros(query.dim()); + for h in 0..heads { + let pos = s![h, .., ..]; + let q = query.slice(pos).to_owned(); + let k = key.slice(pos).to_owned(); + let v = value.slice(pos).to_owned(); + let head = compute_attention(&q, &k, &v, Some(mask.clone())); + score.slice_mut(s![h, .., ..]).assign(&head); + } + score.merge() + } } #[cfg(test)] diff --git a/ml/transformers/src/attention/params/dim.rs b/ml/transformers/src/attention/params/dim.rs index 803c8654..790b8074 100644 --- a/ml/transformers/src/attention/params/dim.rs +++ b/ml/transformers/src/attention/params/dim.rs @@ -10,36 +10,39 @@ //! - `batch`: The batch size //! - `heads`: The number of attention heads //! - `model`: The dimension of the model (embedding size) -use crate::{DEFAULT_ATTENTION_HEADS, DEFAULT_EMBEDDING_SIZE}; +use crate::{HEADS, MODEL_SIZE}; use ndarray::IntoDimension; use serde::{Deserialize, Serialize}; pub trait StructuredDim: IntoDimension {} pub trait BaseDimension: IntoDimension { - fn batch_size(&self) -> usize; fn model_size(&self) -> usize; fn seq_len(&self) -> usize; } +pub trait Batched { + fn batch_size(&self) -> usize; +} + +pub trait ModelSize { + fn model_size(&self) -> usize; +} + pub trait MultiHeadDimension: BaseDimension { fn heads(&self) -> usize; } impl BaseDimension for D where - D: Clone + IntoDimension, + D: Clone + IntoDimension, { - fn batch_size(&self) -> usize { - self.clone().into_dimension()[0] - } - fn model_size(&self) -> usize { - self.clone().into_dimension()[2] + self.clone().into_dimension()[1] } fn seq_len(&self) -> usize { - self.clone().into_dimension()[1] + self.clone().into_dimension()[0] } } @@ -49,6 +52,27 @@ pub enum AttentionDims { MultiHead(MultiShape), // a 4d matrix (batch, heads, seq, query) } +pub enum AttentionShape { + IO { + batch: usize, + seq: usize, + model: usize, + }, + Head { + seq: usize, + query: usize, + }, + Mask { + seq: usize, + }, + MultiHead { + batch: usize, + heads: usize, + seq: usize, + query: usize, + }, +} + #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] @@ -64,11 +88,7 @@ impl BaseShape { } pub fn std(batch: usize, seq: usize) -> Self { - Self::new(batch, seq, DEFAULT_EMBEDDING_SIZE) - } - - pub fn batch_size(&self) -> usize { - self.batch + Self::new(batch, seq, MODEL_SIZE) } pub fn model_size(&self) -> usize { @@ -80,32 +100,36 @@ impl BaseShape { } } +impl Batched for BaseShape { + fn batch_size(&self) -> usize { + self.batch + } +} + +impl IntoDimension for BaseShape { + type Dim = ndarray::Ix3; + + fn into_dimension(self) -> Self::Dim { + ndarray::Ix3(self.batch, self.seq, self.model) + } +} + #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] pub struct MultiShape { - pub batch: usize, pub heads: usize, pub seq: usize, pub query: usize, } impl MultiShape { - pub fn new(batch: usize, heads: usize, seq: usize, query: usize) -> Self { - Self { - batch, - heads, - seq, - query, - } + pub fn new(heads: usize, seq: usize, query: usize) -> Self { + Self { heads, seq, query } } - pub fn std(batch: usize, seq: usize) -> Self { - Self::new(batch, DEFAULT_ATTENTION_HEADS, seq, DEFAULT_EMBEDDING_SIZE) - } - - pub fn batch_size(&self) -> usize { - self.batch + pub fn std(seq: usize) -> Self { + Self::new(HEADS, seq, MODEL_SIZE) } pub fn heads(&self) -> usize { @@ -121,6 +145,55 @@ impl MultiShape { } } +impl From for AttentionShape { + fn from(shape: MultiShape) -> Self { + Self::MultiHead { + batch: 1, + heads: shape.heads, + seq: shape.seq, + query: shape.query, + } + } +} + +impl From for [usize; 3] { + fn from(shape: MultiShape) -> Self { + [shape.heads, shape.seq, shape.query] + } +} + +impl From for (usize, usize, usize) { + fn from(shape: MultiShape) -> Self { + (shape.heads, shape.seq, shape.query) + } +} + +impl IntoDimension for MultiShape { + type Dim = ndarray::Ix3; + + fn into_dimension(self) -> Self::Dim { + ndarray::Ix3(self.heads, self.seq, self.query) + } +} + +pub trait HeadDimension { + fn query_size(&self) -> usize; + fn sequence(&self) -> usize; +} + +impl HeadDimension for T +where + T: Clone + IntoDimension, +{ + fn query_size(&self) -> usize { + self.clone().into_dimension()[1] + } + + fn sequence(&self) -> usize { + self.clone().into_dimension()[0] + } +} + /// #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, @@ -154,15 +227,21 @@ impl From<(usize, usize)> for HeadShape { } } +impl From for (usize, usize) { + fn from(shape: HeadShape) -> Self { + (shape.seq, shape.query) + } +} + impl From<[usize; 2]> for HeadShape { fn from(dim: [usize; 2]) -> Self { Self::new(dim[1], dim[0]) } } -impl From for (usize, usize) { +impl From for [usize; 2] { fn from(shape: HeadShape) -> Self { - (shape.seq, shape.query) + [shape.seq, shape.query] } } diff --git a/ml/transformers/src/attention/params/hyperparams.rs b/ml/transformers/src/attention/params/hyperparams.rs new file mode 100644 index 00000000..658dd0a0 --- /dev/null +++ b/ml/transformers/src/attention/params/hyperparams.rs @@ -0,0 +1,114 @@ +/* + Appellation: hyperparams + Contrib: FL03 +*/ +//! # Hyperparameters +//! +//! Hyperparameters are one which are set before training and are not updated. +//! +//! The hyperparameters for the attention mechanism are: +//! - batch: The number of samples in a batch. +//! - heads: The number of attention heads. +//! - model: The dimension of the model (embedding size). +//! - samples: The number of samples to draw from the attention distribution. +//! +//! + +use super::dim::{BaseShape, HeadShape, MultiShape}; +use crate::{HEADS, MODEL_SIZE, SAMPLES}; +use serde::{Deserialize, Serialize}; + +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] + +pub struct AttentionParameters { + pub batch: usize, + pub heads: usize, + pub model: usize, + pub samples: usize, + pub seq: usize, +} + +impl AttentionParameters { + pub fn new(batch: usize, heads: usize, model: usize, samples: usize, seq: usize) -> Self { + Self { + batch, + heads, + model, + samples, + seq, + } + } + + pub fn std(batch: usize, seq: usize) -> Self { + Self::new(batch, HEADS, MODEL_SIZE, SAMPLES, seq) + } + + pub fn batch_size(&self) -> usize { + self.batch + } + + pub fn heads(&self) -> usize { + self.heads + } + + pub fn model_size(&self) -> usize { + self.model + } + + pub fn samples(&self) -> usize { + self.samples + } + + pub fn seq_len(&self) -> usize { + self.seq + } + + pub fn query_size(&self) -> usize { + self.model / self.heads + } + + pub fn with_batch(mut self, batch: usize) -> Self { + self.batch = batch; + self + } + + pub fn with_heads(mut self, heads: usize) -> Self { + self.heads = heads; + self + } + + pub fn with_model(mut self, model: usize) -> Self { + self.model = model; + self + } + + pub fn with_samples(mut self, samples: usize) -> Self { + self.samples = samples; + self + } + + pub fn with_seq(mut self, seq: usize) -> Self { + self.seq = seq; + self + } +} + +impl From for BaseShape { + fn from(params: AttentionParameters) -> Self { + Self::new(params.batch, params.seq, params.model) + } +} + +impl From for MultiShape { + fn from(params: AttentionParameters) -> Self { + Self::new(params.heads, params.seq, params.model) + } +} + +impl From for HeadShape { + fn from(params: AttentionParameters) -> Self { + Self::new(params.seq, params.query_size()) + } +} diff --git a/ml/transformers/src/attention/params/mod.rs b/ml/transformers/src/attention/params/mod.rs index 9cb6b0d2..6917d2a2 100644 --- a/ml/transformers/src/attention/params/mod.rs +++ b/ml/transformers/src/attention/params/mod.rs @@ -4,89 +4,21 @@ */ //! # Attention Parameters //! -//! ## Hyperparameters -pub use self::{dim::*, qkv::*, utils::*}; +//! ### Hyperparameters +//! +//! Hyperparameters are one which are set before training and are not updated. +//! +//! The hyperparameters for the attention mechanism are: +//! - batch: The number of samples in a batch. +//! - heads: The number of attention heads. +//! - model: The dimension of the model (embedding size). +//! - samples: The number of samples to draw from the attention distribution. +//! +//! +pub use self::{dim::*, hyperparams::*, qkv::*, utils::*}; pub(crate) mod dim; +pub(crate) mod hyperparams; pub(crate) mod qkv; -use crate::{DEFAULT_ATTENTION_HEADS, DEFAULT_EMBEDDING_SIZE, DEFAULT_SAMPLE_SIZE}; -use serde::{Deserialize, Serialize}; - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] - -pub struct Hyperparameters { - pub batch: usize, - pub heads: usize, - pub model: usize, - pub samples: usize, - pub seq: usize, -} - -impl Hyperparameters { - pub fn new(batch: usize, heads: usize, model: usize, samples: usize, seq: usize) -> Self { - Self { - batch, - heads, - model, - samples, - seq, - } - } - - pub fn std(batch: usize, seq: usize) -> Self { - Self::new( - batch, - DEFAULT_ATTENTION_HEADS, - DEFAULT_EMBEDDING_SIZE, - DEFAULT_SAMPLE_SIZE, - seq, - ) - } - - pub fn batch_size(&self) -> usize { - self.batch - } - - pub fn heads(&self) -> usize { - self.heads - } - - pub fn model_size(&self) -> usize { - self.model - } - - pub fn samples(&self) -> usize { - self.samples - } - - pub fn seq_len(&self) -> usize { - self.seq - } - - pub fn query_size(&self) -> usize { - self.model / self.heads - } -} - -impl From for BaseShape { - fn from(hyper: Hyperparameters) -> Self { - Self::new(hyper.batch, hyper.seq, hyper.model) - } -} - -impl From for MultiShape { - fn from(hyper: Hyperparameters) -> Self { - Self::new(hyper.batch, hyper.heads, hyper.seq, hyper.model) - } -} - -impl From for HeadShape { - fn from(hyper: Hyperparameters) -> Self { - Self::new(hyper.seq, hyper.query_size()) - } -} - pub(crate) mod utils {} diff --git a/ml/transformers/src/attention/params/qkv.rs b/ml/transformers/src/attention/params/qkv.rs index 79f8f577..9969424b 100644 --- a/ml/transformers/src/attention/params/qkv.rs +++ b/ml/transformers/src/attention/params/qkv.rs @@ -39,8 +39,6 @@ pub enum QKV { Value, } - - #[cfg(test)] mod tests { use super::*; diff --git a/ml/transformers/src/attention/reps.rs b/ml/transformers/src/attention/reps.rs deleted file mode 100644 index 51dbfab1..00000000 --- a/ml/transformers/src/attention/reps.rs +++ /dev/null @@ -1,178 +0,0 @@ -/* - Appellation: context - Contrib: FL03 -*/ -use super::params::QKV; -use ndarray::{Array2, IntoDimension, Ix2}; -use serde::{Deserialize, Serialize}; -use std::ops; - -pub trait LinearLayer { - fn matmul(&self, data: &Array2) -> Array2; -} - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -pub struct AttentionSpace { - dim: Ix2, - pub key: Array2, - pub query: Array2, - pub value: Array2, -} - -impl AttentionSpace { - pub fn new(dim: D) -> Self - where - D: IntoDimension, - { - let dim = dim.into_dimension(); - Self { - dim, - key: Array2::ones(dim), - query: Array2::ones(dim), - value: Array2::ones(dim), - } - } - - pub fn dim(&self) -> Ix2 { - self.dim - } - - pub fn matmul(&self) -> Array2 { - self.query.dot(&self.key.t()) - } - - pub fn qkv(&self) -> (Array2, Array2, Array2) { - (self.query.clone(), self.key.clone(), self.value.clone()) - } - - pub fn set_weight(&mut self, qkv: QKV, weight: Array2) { - match qkv { - QKV::Key => self.key = weight, - QKV::Query => self.query = weight, - QKV::Value => self.value = weight, - } - } - - pub fn set_weights(&mut self, qkv: impl IntoIterator, weight: Array2) { - for qkv in qkv { - self.set_weight(qkv, weight.clone()); - } - } - - pub fn with_weight(mut self, query: &Array2, key: &Array2, value: &Array2) { - self.key = key.clone(); - self.query = query.clone(); - self.value = value.clone(); - } -} - -impl std::fmt::Display for AttentionSpace { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{}", serde_json::to_string(self).unwrap()) - } -} - -impl From for AttentionSpace -where - D: IntoDimension, -{ - fn from(dim: D) -> Self { - let dim = dim.into_dimension(); - let arr = Array2::ones(dim); - Self { - dim, - key: arr.clone(), - query: arr.clone(), - value: arr, - } - } -} - -impl From for (Array2, Array2, Array2) { - fn from(context: AttentionSpace) -> Self { - (context.key, context.query, context.value) - } -} - -impl ops::Index for AttentionSpace { - type Output = Array2; - - fn index(&self, index: QKV) -> &Self::Output { - use QKV::*; - match index { - Key => &self.key, - Query => &self.query, - Value => &self.value, - } - } -} - -impl ops::IndexMut for AttentionSpace { - fn index_mut(&mut self, index: QKV) -> &mut Self::Output { - match index { - QKV::Key => &mut self.key, - QKV::Query => &mut self.query, - QKV::Value => &mut self.value, - } - } -} - -impl ops::Mul> for AttentionSpace { - type Output = Self; - - fn mul(self, rhs: Array2) -> Self::Output { - let mut ctx = self.clone(); - ctx.key = ctx.key.dot(&rhs); - ctx.query = ctx.query.dot(&rhs); - ctx.value = ctx.value.dot(&rhs); - ctx - } -} - -impl ops::Mul<&Array2> for AttentionSpace { - type Output = Self; - - fn mul(self, rhs: &Array2) -> Self::Output { - let mut ctx = self.clone(); - ctx.key = ctx.key.dot(rhs); - ctx.query = ctx.query.dot(rhs); - ctx.value = ctx.value.dot(rhs); - ctx - } -} - -impl ops::Mul<&Array2> for &AttentionSpace { - type Output = AttentionSpace; - - fn mul(self, rhs: &Array2) -> Self::Output { - let mut ctx = self.clone(); - ctx.key = ctx.key.dot(rhs); - ctx.query = ctx.query.dot(rhs); - ctx.value = ctx.value.dot(rhs); - ctx - } -} - -impl ops::MulAssign> for AttentionSpace { - fn mul_assign(&mut self, rhs: Array2) { - self.key = self.key.dot(&rhs); - self.query = self.query.dot(&rhs); - self.value = self.value.dot(&rhs); - } -} - -impl ops::MulAssign<&Array2> for AttentionSpace { - fn mul_assign(&mut self, rhs: &Array2) { - self.key = self.key.dot(rhs); - self.query = self.query.dot(rhs); - self.value = self.value.dot(rhs); - } -} - -impl ops::MulAssign<&Array2> for &mut AttentionSpace { - fn mul_assign(&mut self, rhs: &Array2) { - self.key = self.key.dot(rhs); - self.query = self.query.dot(rhs); - self.value = self.value.dot(rhs); - } -} diff --git a/ml/transformers/src/attention/weights.rs b/ml/transformers/src/attention/weights.rs new file mode 100644 index 00000000..644d98c9 --- /dev/null +++ b/ml/transformers/src/attention/weights.rs @@ -0,0 +1,170 @@ +/* + Appellation: weights + Contrib: FL03 +*/ +use crate::ops::Split; +use super::params::QKV; + +use ndarray::prelude::{Array, Array2, Array3}; +use ndarray::{IntoDimension, Ix2}; +use num::Float; +use serde::{Deserialize, Serialize}; +use strum::IntoEnumIterator; +use std::ops; + + +pub type WeightTensor = Array; // (seq, model) + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +pub struct Weight where T: Float { + dim: Ix2, + pub key: Array2, + pub query: Array2, + pub value: Array2, +} + +impl Weight { + pub fn new(dim: D) -> Self + where + D: IntoDimension, + { + let dim = dim.into_dimension(); + let arr = Array2::ones(dim); + Self { + dim, + key: arr.clone(), + query: arr.clone(), + value: arr, + } + } + + pub fn dim(&self) -> Ix2 { + self.dim + } + + pub fn qkv(&self) -> (Array2, Array2, Array2) { + self.clone().into() + } +} + +impl std::fmt::Display for Weight { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", serde_json::to_string(self).unwrap()) + } +} + +impl From for Weight +where + D: IntoDimension, + T: Float +{ + fn from(dim: D) -> Self { + let dim = dim.into_dimension(); + let arr = Array2::ones(dim); + Self { + dim, + key: arr.clone(), + query: arr.clone(), + value: arr, + } + } +} + +impl From> for (Array2, Array2, Array2) { + fn from(context: Weight) -> Self { + (context.key, context.query, context.value) + } +} + +impl Split<(Array3, Array3, Array3)> for Weight { + type Error = ndarray::ShapeError; + + fn split(&self, heads: usize) -> Result<(Array3, Array3, Array3), Self::Error> { + let (key, query, value) = self.qkv(); + Ok((key.split(heads)?, query.split(heads)?, value.split(heads)?)) + } +} + +impl ops::Index for Weight { + type Output = Array2; + + fn index(&self, index: QKV) -> &Self::Output { + use QKV::*; + match index { + Key => &self.key, + Query => &self.query, + Value => &self.value, + } + } +} + +impl ops::IndexMut for Weight { + fn index_mut(&mut self, index: QKV) -> &mut Self::Output { + use QKV::*; + match index { + Key => &mut self.key, + Query => &mut self.query, + Value => &mut self.value, + } + } +} + +impl ops::Mul> for Weight { + type Output = Self; + + fn mul(self, rhs: Array2) -> Self::Output { + let mut ctx = self.clone(); + ctx.key = ctx.key.dot(&rhs); + ctx.query = ctx.query.dot(&rhs); + ctx.value = ctx.value.dot(&rhs); + ctx + } +} + +impl ops::Mul<&Array2> for Weight { + type Output = Self; + + fn mul(self, rhs: &Array2) -> Self::Output { + let mut ctx = self.clone(); + ctx.key = ctx.key.dot(rhs); + ctx.query = ctx.query.dot(rhs); + ctx.value = ctx.value.dot(rhs); + ctx + } +} + +impl ops::Mul<&Array2> for &Weight { + type Output = Weight; + + fn mul(self, rhs: &Array2) -> Self::Output { + let mut ctx = self.clone(); + ctx.key = ctx.key.dot(rhs); + ctx.query = ctx.query.dot(rhs); + ctx.value = ctx.value.dot(rhs); + ctx + } +} + +impl ops::MulAssign> for Weight { + fn mul_assign(&mut self, rhs: Array2) { + for qkv in QKV::iter() { + self[qkv] = self[qkv].dot(&rhs); + } + } +} + +impl ops::MulAssign<&Array2> for Weight { + fn mul_assign(&mut self, rhs: &Array2) { + for qkv in QKV::iter() { + self[qkv] = self[qkv].dot(rhs); + } + } +} + +impl ops::MulAssign<&Array2> for &mut Weight { + fn mul_assign(&mut self, rhs: &Array2) { + for qkv in QKV::iter() { + self[qkv] = self[qkv].dot(rhs); + } + } +} diff --git a/ml/transformers/src/codec/decode/mod.rs b/ml/transformers/src/codec/decode/mod.rs index fcc6d4bb..f9114f9b 100644 --- a/ml/transformers/src/codec/decode/mod.rs +++ b/ml/transformers/src/codec/decode/mod.rs @@ -3,9 +3,10 @@ Contrib: FL03 */ //! # Decode -pub use self::{decoder::*, utils::*}; +pub use self::{decoder::*, params::*, utils::*}; pub(crate) mod decoder; +pub(crate) mod params; pub trait Decode {} diff --git a/ml/transformers/src/codec/decode/params.rs b/ml/transformers/src/codec/decode/params.rs new file mode 100644 index 00000000..fe8fd57a --- /dev/null +++ b/ml/transformers/src/codec/decode/params.rs @@ -0,0 +1,4 @@ +/* + Appellation: params + Contrib: FL03 +*/ \ No newline at end of file diff --git a/ml/transformers/src/codec/encode/encoder.rs b/ml/transformers/src/codec/encode/encoder.rs index 1270bf6b..fd7f8b1b 100644 --- a/ml/transformers/src/codec/encode/encoder.rs +++ b/ml/transformers/src/codec/encode/encoder.rs @@ -2,5 +2,21 @@ Appellation: encoder Contrib: FL03 */ +use super::EncoderParams; +use crate::attention::multi::MultiHeadAttention; +use crate::ffn::FFN; -pub struct Encoder {} + +pub struct Encoder { + attention: MultiHeadAttention, + network: FFN, + params: EncoderParams, +} + +impl Encoder { + pub fn new(params: EncoderParams) -> Self { + let attention = MultiHeadAttention::new(params.heads, params.model); + let network = FFN::new(params.model, None); + Self { attention, network, params, } + } +} diff --git a/ml/transformers/src/codec/encode/mod.rs b/ml/transformers/src/codec/encode/mod.rs index 03cd8a0a..206c9640 100644 --- a/ml/transformers/src/codec/encode/mod.rs +++ b/ml/transformers/src/codec/encode/mod.rs @@ -3,9 +3,11 @@ Contrib: FL03 */ //! # Encode -pub use self::{encoder::*, utils::*}; +pub use self::{encoder::*, params::*, stack::*, utils::*}; pub(crate) mod encoder; +pub(crate) mod params; +pub(crate) mod stack; pub trait Encode {} diff --git a/ml/transformers/src/codec/encode/params.rs b/ml/transformers/src/codec/encode/params.rs new file mode 100644 index 00000000..c244a1db --- /dev/null +++ b/ml/transformers/src/codec/encode/params.rs @@ -0,0 +1,11 @@ +/* + Appellation: params + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct EncoderParams { + pub heads: usize, + pub model: usize, +} \ No newline at end of file diff --git a/ml/transformers/src/codec/encode/stack.rs b/ml/transformers/src/codec/encode/stack.rs new file mode 100644 index 00000000..11a7a203 --- /dev/null +++ b/ml/transformers/src/codec/encode/stack.rs @@ -0,0 +1,25 @@ +/* + Appellation: stack + Contrib: FL03 +*/ +use super::{Encoder, EncoderParams}; + +pub struct EncoderStack { + layers: usize, + params: EncoderParams, + stack: Vec, +} + +impl EncoderStack { + pub fn new(layers: usize, params: EncoderParams) -> Self { + let stack = Vec::with_capacity(layers); + + Self { layers, params, stack } + } + + pub fn setup(&mut self) { + for _ in 0..self.layers { + self.stack.push(Encoder::new(self.params)); + } + } +} diff --git a/ml/transformers/src/ffn/mod.rs b/ml/transformers/src/ffn/mod.rs index 2c45033a..17c68075 100644 --- a/ml/transformers/src/ffn/mod.rs +++ b/ml/transformers/src/ffn/mod.rs @@ -3,9 +3,10 @@ Contrib: FL03 */ //! # Decode -pub use self::{network::*, utils::*}; +pub use self::{network::*, params::*, utils::*}; pub(crate) mod network; +pub(crate) mod params; pub(crate) mod utils {} diff --git a/ml/transformers/src/ffn/network.rs b/ml/transformers/src/ffn/network.rs index 743b6a04..b83c6db1 100644 --- a/ml/transformers/src/ffn/network.rs +++ b/ml/transformers/src/ffn/network.rs @@ -2,26 +2,31 @@ Appellation: network Contrib: FL03 */ +use super::FFNParams; +use crate::data::linear::LinearLayer; use crate::neural::neurons::activate::{Activator, ReLU}; -use ndarray::{Array1, Array2}; - -/// All vectors have a dimension of (nodes, elem) -pub fn ffn(data: Array2, bias: Array2, weights: Array2) -> Array2 { - let a = data.dot(&weights) + bias.clone(); - ReLU::rho(a).dot(&weights) + bias -} +use ndarray::prelude::Array2; +use serde::{Deserialize, Serialize}; +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] pub struct FFN { - bias: Array2, - weights: Array2, + input: LinearLayer, + output: LinearLayer, + pub params: FFNParams, } impl FFN { - pub fn new(bias: Array2, weights: Array2) -> Self { - Self { bias, weights } + pub fn new(model: usize, network: Option) -> Self { + let params = FFNParams::new(model, network.unwrap_or(crate::NETWORK_SIZE)); + let layer = LinearLayer::new(params.model, params.network); + Self { + input: layer.clone(), + output: layer, + params, + } } - pub fn forward(&self, data: Array2) -> Array2 { - ffn(data, self.bias.clone(), self.weights.clone()) + pub fn forward(&self, data: &Array2) -> Array2 { + self.output.linear(&ReLU::rho(self.input.linear(data))) } } diff --git a/ml/transformers/src/ffn/params.rs b/ml/transformers/src/ffn/params.rs new file mode 100644 index 00000000..1474c9aa --- /dev/null +++ b/ml/transformers/src/ffn/params.rs @@ -0,0 +1,34 @@ +/* + Appellation: params + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct FFNParams { + pub model: usize, + pub network: usize, +} + +impl FFNParams { + pub fn new(model: usize, network: usize) -> Self { + Self { model, network } + } + + pub fn model_size(&self) -> usize { + self.model + } + + pub fn network_size(&self) -> usize { + self.network + } +} + +impl Default for FFNParams { + fn default() -> Self { + Self { + model: crate::MODEL_SIZE, + network: crate::NETWORK_SIZE, + } + } +} \ No newline at end of file diff --git a/ml/transformers/src/lib.rs b/ml/transformers/src/lib.rs index b934c0ff..cf4c8ff0 100644 --- a/ml/transformers/src/lib.rs +++ b/ml/transformers/src/lib.rs @@ -12,14 +12,18 @@ pub(crate) mod utils; pub mod attention; pub mod codec; pub mod ffn; +pub mod ops; pub mod transform; pub(crate) use concision_core as core; +pub(crate) use concision_data as data; pub(crate) use concision_neural as neural; pub mod prelude { - pub use crate::attention::*; + pub use crate::attention::params::*; pub use crate::codec::*; + pub use crate::ffn::*; + pub use crate::ops::*; pub use crate::transform::*; pub use crate::primitives::*; diff --git a/ml/transformers/src/attention/ops/merge.rs b/ml/transformers/src/ops/merge.rs similarity index 100% rename from ml/transformers/src/attention/ops/merge.rs rename to ml/transformers/src/ops/merge.rs diff --git a/ml/transformers/src/attention/ops/mod.rs b/ml/transformers/src/ops/mod.rs similarity index 91% rename from ml/transformers/src/attention/ops/mod.rs rename to ml/transformers/src/ops/mod.rs index 4d523d2a..d0418912 100644 --- a/ml/transformers/src/attention/ops/mod.rs +++ b/ml/transformers/src/ops/mod.rs @@ -2,9 +2,10 @@ Appellation: ops Contrib: FL03 */ -pub use self::{merge::*, split::*, utils::*}; +pub use self::{merge::*, norm::*, split::*, utils::*}; pub(crate) mod merge; +pub(crate) mod norm; pub(crate) mod split; pub(crate) mod utils { @@ -42,12 +43,7 @@ pub(crate) mod utils { where T: Clone, { - let (batch, n, seq, query) = ( - heads.shape()[0], - heads.shape()[1], - heads.shape()[2], - heads.shape()[3], - ); + let (batch, n, seq, query) = heads.dim(); let mut tmp = heads.clone(); // swap the head and sequence axes tmp.swap_axes(1, 2); @@ -84,7 +80,9 @@ mod tests { let a = split_batch(&data, 2).unwrap(); assert_eq!(a.shape(), &dim_split); + assert_eq!(&a, &data.split(2).unwrap()); let b = merge_batch(&a).unwrap(); assert_eq!(b.shape(), &dim_input); + assert_eq!(&b, &data); } } diff --git a/ml/transformers/src/ops/norm.rs b/ml/transformers/src/ops/norm.rs new file mode 100644 index 00000000..e69de29b diff --git a/ml/transformers/src/attention/ops/split.rs b/ml/transformers/src/ops/split.rs similarity index 100% rename from ml/transformers/src/attention/ops/split.rs rename to ml/transformers/src/ops/split.rs diff --git a/ml/transformers/src/primitives.rs b/ml/transformers/src/primitives.rs index 48a29aaa..c787b326 100644 --- a/ml/transformers/src/primitives.rs +++ b/ml/transformers/src/primitives.rs @@ -4,19 +4,30 @@ */ pub use self::{constants::*, statics::*, types::*}; -/// Collection of constants used throughout the system pub(crate) mod constants { - pub const DEFAULT_EMBEDDING_SIZE: usize = 512; - - pub const DEFAULT_ATTENTION_HEADS: usize = 8; + /// The default dropout rate + pub const DROPOUT: f64 = 0.1; + /// The default number of heads in the multi-head attention layer + pub const HEADS: usize = 8; + /// The default dimension of the model (embedding size) + pub const MODEL_SIZE: usize = 512; + /// The default dimension of the feed-forward network + pub const NETWORK_SIZE: usize = 2048; + /// The default number of samples to draw from the attention distribution + pub const SAMPLES: usize = 10000; +} - pub const DEFAULT_ATTENTION_HEAD_SIZE: usize = 64; +pub(crate) mod statics { + use super::constants::*; + use lazy_static::lazy_static; - pub const DEFAULT_SAMPLE_SIZE: usize = 10000; + lazy_static!{ + /// The default dimensions of the query, key, and value tensors w/r/2 a single head + pub static ref QUERY_SIZE: usize = MODEL_SIZE / HEADS; + } } -/// Collection of static references used throughout -pub(crate) mod statics {} - -/// Collection of types used throughout the system -pub(crate) mod types {} +pub(crate) mod types { + /// The dimension of all inputs and outputs for each layer of the model (batch, seq, model) + pub type BaseDim = ndarray::Ix3; +} diff --git a/ml/transformers/src/transform/mod.rs b/ml/transformers/src/transform/mod.rs index 44c67180..9c65329c 100644 --- a/ml/transformers/src/transform/mod.rs +++ b/ml/transformers/src/transform/mod.rs @@ -3,8 +3,9 @@ Contrib: FL03 */ //! # Transform -pub use self::{transformer::*, utils::*}; +pub use self::{params::*, transformer::*, utils::*}; +pub(crate) mod params; pub(crate) mod transformer; pub trait Transform {} diff --git a/ml/transformers/src/transform/params.rs b/ml/transformers/src/transform/params.rs new file mode 100644 index 00000000..c6720ad2 --- /dev/null +++ b/ml/transformers/src/transform/params.rs @@ -0,0 +1,14 @@ +/* + Appellation: params + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct TransformerParams { + pub batch: usize, + pub heads: usize, + pub layers: usize, + pub model: usize, + pub samples: usize, +} \ No newline at end of file From 8006f922e22aa2c8fa70bdb339820f03a4f8465a Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 29 Oct 2023 12:44:13 -0500 Subject: [PATCH 034/118] update Signed-off-by: FL03 --- Cargo.toml | 3 + data/src/linear/layer.rs | 2 + ml/neural/Cargo.toml | 3 + ml/neural/src/layers/features.rs | 66 ++++++++++++++++++++++ ml/neural/src/layers/layer.rs | 95 ++++++++++---------------------- ml/neural/src/layers/mod.rs | 3 +- ml/neural/src/specs.rs | 19 +++++++ ml/transformers/src/ops/merge.rs | 2 +- ml/transformers/src/ops/norm.rs | 61 ++++++++++++++++++++ 9 files changed, 186 insertions(+), 68 deletions(-) create mode 100644 ml/neural/src/layers/features.rs diff --git a/Cargo.toml b/Cargo.toml index 32ad33ab..42861539 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,11 @@ computare = { features = ["full"], branch = "v0.1.0", git = "https://github.com/ anyhow = "1" lazy_static = "1" ndarray = { features = ["serde-1"], version = "0.15" } +ndarray-rand = { features = [], version = "0.14" } +ndarray-stats = { features = [], version = "0.5.1" } num = { features = ["serde"], version = "0.4" } petgraph = { features = ["serde-1"], version = "0.6" } +rand = "0.8" serde = { features = ["derive"], version = "1" } serde_json = "1" smart-default = "0.7" diff --git a/data/src/linear/layer.rs b/data/src/linear/layer.rs index 8375b3c0..2613ebe2 100644 --- a/data/src/linear/layer.rs +++ b/data/src/linear/layer.rs @@ -7,6 +7,8 @@ use ndarray::prelude::{Array1, Array2}; use num::Float; use serde::{Deserialize, Serialize}; + + #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] pub struct LinearLayer { bias: Array1, diff --git a/ml/neural/Cargo.toml b/ml/neural/Cargo.toml index 30ab801c..203ece4d 100644 --- a/ml/neural/Cargo.toml +++ b/ml/neural/Cargo.toml @@ -28,8 +28,11 @@ concision-core.workspace = true anyhow.workspace = true ndarray.workspace = true +ndarray-rand.workspace = true +ndarray-stats.workspace = true num.workspace = true petgraph.workspace = true +rand.workspace = true serde.workspace = true serde_json.workspace = true smart-default.workspace = true diff --git a/ml/neural/src/layers/features.rs b/ml/neural/src/layers/features.rs new file mode 100644 index 00000000..bd6ddd95 --- /dev/null +++ b/ml/neural/src/layers/features.rs @@ -0,0 +1,66 @@ +/* + Appellation: features + Contrib: FL03 +*/ +use ndarray::IntoDimension; +use serde::{Deserialize, Serialize}; + +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] +pub struct Features { + pub inputs: usize, + pub outputs: usize, +} + +impl Features { + pub fn new(inputs: usize, outputs: usize) -> Self { + Self { inputs, outputs } + } + + pub fn inputs(&self) -> usize { + self.inputs + } + + pub fn outputs(&self) -> usize { + self.outputs + } +} + +impl std::fmt::Display for Features { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "({}, {})", self.inputs, self.outputs) + } +} + +impl IntoDimension for Features { + type Dim = ndarray::IxDyn; + + fn into_dimension(self) -> Self::Dim { + ndarray::IxDyn(&[self.inputs, self.outputs]) + } +} + +impl From for ndarray::Ix2 { + fn from(features: Features) -> Self { + ndarray::Ix2(features.inputs, features.outputs) + } +} + +impl From for ndarray::IxDyn { + fn from(features: Features) -> Self { + ndarray::IxDyn(&[features.inputs, features.outputs]) + } +} + +impl From for [usize; 2] { + fn from(features: Features) -> Self { + [features.inputs, features.outputs] + } +} + +impl From for (usize, usize) { + fn from(features: Features) -> Self { + (features.inputs, features.outputs) + } +} diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index e229f9a9..2e444ecf 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -2,49 +2,58 @@ Appellation: layer Contrib: FL03 */ -use super::LayerType; +use super::{Features, LayerType}; use crate::neurons::activate::Activator; -use crate::neurons::Node; use ndarray::prelude::{Array1, Array2}; -use std::ops; +use num::Float; -pub trait L -where - T: num::Float + 'static, -{ - type Activator: Activator; +pub trait L { // - fn activate(&self, args: &Array2) -> Array2 { - let z = args.dot(self.weights()) - self.bias(); - z.mapv(|x| self.activator().activate(x)) + fn process(&self, args: &Array2, rho: impl Activator) -> Array2 where T: 'static { + let z = args.dot(self.weights()) + self.bias(); + z.mapv(|x| rho.activate(x)) } - fn activator(&self) -> &Self::Activator; - - fn bias(&self) -> &Array2; + fn bias(&self) -> &Array1; fn weights(&self) -> &Array2; } +pub trait Linear { + fn linear(&self, data: &Array2) -> Array2 where T: 'static; +} + #[derive(Clone, Debug, Default, PartialEq)] pub struct Layer { - bias: Array1, + bias: Option>, + features: Features, layer: LayerType, - nodes: usize, weights: Array2, } impl Layer { - pub fn new(layer: LayerType,) -> Self { - Self { layer, nodes } + pub fn new(inputs: usize, outputs: usize, bias: bool, layer: LayerType) -> Self { + let features = Features::new(inputs, outputs); + let bias = if bias { + Some(Array1::ones(outputs)) + } else { + None + }; + let weights = Array2::ones((features.inputs(), features.outputs())); + + Self { bias, features, layer, weights } + } + + pub fn bias(&self) -> &Option> { + &self.bias } pub fn layer(&self) -> &LayerType { &self.layer } - pub fn nodes(&self) -> &[Node] { - &self.nodes + pub fn features(&self) -> Features { + self.features } pub fn set_layer(&mut self, layer: LayerType) { @@ -52,51 +61,5 @@ impl Layer { } } -impl AsRef<[Node]> for Layer { - fn as_ref(&self) -> &[Node] { - &self.nodes - } -} - -impl AsMut<[Node]> for Layer { - fn as_mut(&mut self) -> &mut [Node] { - &mut self.nodes - } -} - -impl IntoIterator for Layer { - type Item = Node; - type IntoIter = std::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.nodes.into_iter() - } -} - -impl ops::Index for Layer { - type Output = Node; - fn index(&self, index: usize) -> &Self::Output { - &self.nodes[index] - } -} - -impl ops::IndexMut for Layer { - fn index_mut(&mut self, index: usize) -> &mut Self::Output { - &mut self.nodes[index] - } -} -impl ops::Index> for Layer { - type Output = [Node]; - - fn index(&self, index: ops::Range) -> &Self::Output { - &self.nodes[index] - } -} - -impl ops::IndexMut> for Layer { - fn index_mut(&mut self, index: ops::Range) -> &mut Self::Output { - &mut self.nodes[index] - } -} diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index 8b6405cd..0c9d9962 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -3,8 +3,9 @@ Contrib: FL03 */ //! # Layers -pub use self::{kinds::*, layer::*, utils::*}; +pub use self::{features::*, kinds::*, layer::*, utils::*}; +pub(crate) mod features; pub(crate) mod kinds; pub(crate) mod layer; diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index dcb9462a..d83579af 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -2,6 +2,25 @@ Appellation: specs Contrib: FL03 */ +use ndarray::prelude::Array1; +use ndarray_rand::RandomExt; +use ndarray_rand::rand_distr::Uniform; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use num::Float; + + + +pub trait Bias { + fn init_uniform(features: usize) -> Array1 { + let k = (T::from(features).unwrap()).sqrt(); + let uniform = Uniform::new(-k, k); + Array1::random(features, uniform) + } + + fn bias(&self) -> &Array1; + fn bias_mut(&mut self) -> &mut Array1; + +} pub trait Trainable { fn train(&mut self, args: &[f64]) -> f64; diff --git a/ml/transformers/src/ops/merge.rs b/ml/transformers/src/ops/merge.rs index e8c246db..bb16f626 100644 --- a/ml/transformers/src/ops/merge.rs +++ b/ml/transformers/src/ops/merge.rs @@ -1,5 +1,5 @@ /* - Appellation: split + Appellation: merge Contrib: FL03 */ use ndarray::prelude::{Array2, Array3, Array4}; diff --git a/ml/transformers/src/ops/norm.rs b/ml/transformers/src/ops/norm.rs index e69de29b..5f9c6360 100644 --- a/ml/transformers/src/ops/norm.rs +++ b/ml/transformers/src/ops/norm.rs @@ -0,0 +1,61 @@ +/* + Appellation: norm + Contrib: FL03 +*/ +use ndarray::{Axis, ScalarOperand}; +use ndarray::prelude::{Array1, Array2}; +use num::{Float, FromPrimitive}; +use serde::{Deserialize, Serialize}; + +pub fn layer_normalize(x: &Array2) -> Array2 +where + T: Float + FromPrimitive + ScalarOperand, +{ + // Calculate the mean and variance of the activations along the feature axis. + let mean = x.mean_axis(Axis(1)).unwrap(); + let var = x.var_axis(Axis(1), T::one()); + + // Normalize the activations. + let epsilon = T::from(1e-6).unwrap(); + let scale = (var + epsilon).mapv(|i| T::one() / i.sqrt()); + let normalized_x = (x - &mean) * &scale; + + // Scale and shift the normalized activations with learnable parameters gamma and beta. + let gamma = Array1::ones(mean.dim()); + let beta = Array1::zeros(mean.dim()); + let scaled_shifted_x = &normalized_x * &gamma + β + + scaled_shifted_x +} + + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +pub struct Normalize { + alpha: Array1, + beta: Array1, +} + +impl Normalize { + pub fn new(features: usize) -> Self { + Self { + alpha: Array1::ones(features), + beta: Array1::zeros(features), + } + } + + pub fn forward(&self, x: &Array2) -> Array2 where T: FromPrimitive + ScalarOperand { + // Calculate the mean and variance of the activations along the feature axis. + let mean = x.mean_axis(Axis(1)).unwrap(); + let var = x.var_axis(Axis(1), T::one()); + + // Normalize the activations. + let epsilon = T::from(1e-6).unwrap(); + let scale = (var + epsilon).mapv(|i| T::one() / i.sqrt()); + let normalized_x = (x - &mean) * &scale; + + // Scale and shift the normalized activations with learnable parameters gamma and beta. + let scaled_shifted_x = &normalized_x * &self.alpha + &self.beta; + + scaled_shifted_x + } +} \ No newline at end of file From 5a1a3a87346bf6b16941a93615b0c1fa7876570d Mon Sep 17 00:00:00 2001 From: FL03 Date: Tue, 31 Oct 2023 15:30:25 -0500 Subject: [PATCH 035/118] update Signed-off-by: FL03 --- core/src/specs.rs | 35 +++++++++++ core/src/states/state.rs | 7 ++- data/src/linear/features.rs | 30 ++++----- data/src/linear/layer.rs | 2 - data/src/linear/mod.rs | 19 +++--- ml/neural/src/bias.rs | 36 +++++++++++ ml/neural/src/layers/features.rs | 60 +++++++++--------- ml/neural/src/layers/layer.rs | 19 ++++-- ml/neural/src/lib.rs | 4 ++ ml/neural/src/models/mod.rs | 13 ++++ ml/neural/src/models/model.rs | 4 ++ ml/neural/src/ops/dropout.rs | 22 +++++++ ml/neural/src/ops/mod.rs | 30 +++++++++ ml/neural/src/ops/norm.rs | 38 ++++++++++++ ml/neural/src/prop/mod.rs | 4 +- ml/neural/src/specs.rs | 7 +-- ml/transformers/src/attention/head.rs | 2 +- ml/transformers/src/attention/mod.rs | 7 +-- .../src/attention/multi/attention.rs | 53 +++++++++++++--- ml/transformers/src/attention/multi/mod.rs | 20 +++--- ml/transformers/src/attention/multi/params.rs | 48 +++++++++++++++ ml/transformers/src/attention/weights.rs | 12 ++-- ml/transformers/src/codec/decode/params.rs | 2 +- ml/transformers/src/codec/encode/encoder.rs | 51 +++++++++++++--- ml/transformers/src/codec/encode/params.rs | 29 ++++++++- ml/transformers/src/codec/encode/stack.rs | 28 +++++---- ml/transformers/src/ffn/network.rs | 7 ++- ml/transformers/src/ffn/params.rs | 2 +- ml/transformers/src/ops/mod.rs | 3 +- ml/transformers/src/ops/norm.rs | 61 ------------------- ml/transformers/src/ops/split.rs | 12 +++- ml/transformers/src/primitives.rs | 2 +- ml/transformers/src/transform/params.rs | 6 +- 33 files changed, 480 insertions(+), 195 deletions(-) create mode 100644 ml/neural/src/bias.rs create mode 100644 ml/neural/src/models/mod.rs create mode 100644 ml/neural/src/models/model.rs create mode 100644 ml/neural/src/ops/dropout.rs create mode 100644 ml/neural/src/ops/mod.rs create mode 100644 ml/neural/src/ops/norm.rs create mode 100644 ml/transformers/src/attention/multi/params.rs delete mode 100644 ml/transformers/src/ops/norm.rs diff --git a/core/src/specs.rs b/core/src/specs.rs index 0ad8fffa..d654aa70 100644 --- a/core/src/specs.rs +++ b/core/src/specs.rs @@ -2,6 +2,8 @@ Appellation: specs Contrib: FL03 */ +use num::{Num, One}; +use std::ops::MulAssign; pub trait Pair { fn pair(&self) -> (A, B); @@ -15,3 +17,36 @@ where self.clone().into() } } + +pub trait Product { + type Item: Num; + + fn product(&self) -> Self::Item; +} + +impl Product for I +where + I: Clone + IntoIterator, + T: One + Num + MulAssign, +{ + type Item = T; + + fn product(&self) -> Self::Item { + let mut res = T::one(); + for i in self.clone().into_iter() { + res *= i; + } + res + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_product() { + let args = vec![2, 4, 6]; + assert_eq!(args.product(), 48); + } +} diff --git a/core/src/states/state.rs b/core/src/states/state.rs index 7b7dbda1..1641712f 100644 --- a/core/src/states/state.rs +++ b/core/src/states/state.rs @@ -4,7 +4,6 @@ */ use serde::{Deserialize, Serialize}; - #[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] #[serde(rename_all = "lowercase")] pub struct State { @@ -16,7 +15,11 @@ pub struct State { impl State { pub fn new(kind: impl ToString, message: String) -> Self { let ts = crate::now(); - Self { kind: kind.to_string(), message, ts } + Self { + kind: kind.to_string(), + message, + ts, + } } pub fn kind(&self) -> &str { diff --git a/data/src/linear/features.rs b/data/src/linear/features.rs index edb3fd1e..2854ef26 100644 --- a/data/src/linear/features.rs +++ b/data/src/linear/features.rs @@ -9,28 +9,28 @@ use serde::{Deserialize, Serialize}; Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] pub struct Features { - pub input: usize, - pub output: usize, + pub input: usize, + pub output: usize, } impl Features { - pub fn new(input: usize, output: usize) -> Self { - Self { input, output } - } + pub fn new(input: usize, output: usize) -> Self { + Self { input, output } + } - pub fn input(&self) -> usize { - self.input - } + pub fn input(&self) -> usize { + self.input + } - pub fn output(&self) -> usize { - self.output - } + pub fn output(&self) -> usize { + self.output + } } impl IntoDimension for Features { - type Dim = ndarray::IxDyn; + type Dim = ndarray::IxDyn; - fn into_dimension(self) -> Self::Dim { - ndarray::IxDyn(&[self.input, self.output]) - } + fn into_dimension(self) -> Self::Dim { + ndarray::IxDyn(&[self.input, self.output]) + } } diff --git a/data/src/linear/layer.rs b/data/src/linear/layer.rs index 2613ebe2..8375b3c0 100644 --- a/data/src/linear/layer.rs +++ b/data/src/linear/layer.rs @@ -7,8 +7,6 @@ use ndarray::prelude::{Array1, Array2}; use num::Float; use serde::{Deserialize, Serialize}; - - #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] pub struct LinearLayer { bias: Array1, diff --git a/data/src/linear/mod.rs b/data/src/linear/mod.rs index bb72fa72..8c426ed3 100644 --- a/data/src/linear/mod.rs +++ b/data/src/linear/mod.rs @@ -8,16 +8,15 @@ pub use self::{features::*, layer::*, utils::*}; pub(crate) mod features; pub(crate) mod layer; - pub(crate) mod utils { - use ndarray::prelude::{Array1, Array2}; - use num::Float; + use ndarray::prelude::{Array1, Array2}; + use num::Float; - pub fn linear_transformation( - data: &Array2, - bias: &Array1, - weights: &Array2, - ) -> Array2 { - data.dot(&weights.t()) + bias - } + pub fn linear_transformation( + data: &Array2, + bias: &Array1, + weights: &Array2, + ) -> Array2 { + data.dot(&weights.t()) + bias + } } diff --git a/ml/neural/src/bias.rs b/ml/neural/src/bias.rs new file mode 100644 index 00000000..cdc22c3a --- /dev/null +++ b/ml/neural/src/bias.rs @@ -0,0 +1,36 @@ +/* + Appellation: bias + Contrib: FL03 +*/ +use ndarray::prelude::Array1; +use ndarray_rand::rand_distr::{uniform::SampleUniform, Uniform}; +use ndarray_rand::RandomExt; +use num::Float; + +fn _generate_bias(size: usize) -> Array1 { + let ds = (T::from(size).unwrap()).sqrt(); + let dist = Uniform::new(-ds, ds); + Array1::::random(size, dist) +} + +pub enum Bias { + Biased(Array1), + Unbiased, +} + +impl Bias { + pub fn biased(size: usize) -> Self + where + T: SampleUniform, + { + let bias = _generate_bias(size); + Self::Biased(bias) + } + + pub fn forward(&self, data: &Array1) -> Array1 { + match self { + Self::Biased(bias) => data + bias, + Self::Unbiased => data.clone(), + } + } +} diff --git a/ml/neural/src/layers/features.rs b/ml/neural/src/layers/features.rs index bd6ddd95..fd263495 100644 --- a/ml/neural/src/layers/features.rs +++ b/ml/neural/src/layers/features.rs @@ -9,58 +9,58 @@ use serde::{Deserialize, Serialize}; Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] pub struct Features { - pub inputs: usize, - pub outputs: usize, + pub inputs: usize, + pub outputs: usize, } impl Features { - pub fn new(inputs: usize, outputs: usize) -> Self { - Self { inputs, outputs } - } + pub fn new(inputs: usize, outputs: usize) -> Self { + Self { inputs, outputs } + } - pub fn inputs(&self) -> usize { - self.inputs - } + pub fn inputs(&self) -> usize { + self.inputs + } - pub fn outputs(&self) -> usize { - self.outputs - } + pub fn outputs(&self) -> usize { + self.outputs + } } impl std::fmt::Display for Features { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "({}, {})", self.inputs, self.outputs) - } + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "({}, {})", self.inputs, self.outputs) + } } impl IntoDimension for Features { - type Dim = ndarray::IxDyn; + type Dim = ndarray::IxDyn; - fn into_dimension(self) -> Self::Dim { - ndarray::IxDyn(&[self.inputs, self.outputs]) - } + fn into_dimension(self) -> Self::Dim { + ndarray::IxDyn(&[self.inputs, self.outputs]) + } } impl From for ndarray::Ix2 { - fn from(features: Features) -> Self { - ndarray::Ix2(features.inputs, features.outputs) - } + fn from(features: Features) -> Self { + ndarray::Ix2(features.inputs, features.outputs) + } } impl From for ndarray::IxDyn { - fn from(features: Features) -> Self { - ndarray::IxDyn(&[features.inputs, features.outputs]) - } + fn from(features: Features) -> Self { + ndarray::IxDyn(&[features.inputs, features.outputs]) + } } impl From for [usize; 2] { - fn from(features: Features) -> Self { - [features.inputs, features.outputs] - } + fn from(features: Features) -> Self { + [features.inputs, features.outputs] + } } impl From for (usize, usize) { - fn from(features: Features) -> Self { - (features.inputs, features.outputs) - } + fn from(features: Features) -> Self { + (features.inputs, features.outputs) + } } diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 2e444ecf..5176e85f 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -9,7 +9,10 @@ use num::Float; pub trait L { // - fn process(&self, args: &Array2, rho: impl Activator) -> Array2 where T: 'static { + fn process(&self, args: &Array2, rho: impl Activator) -> Array2 + where + T: 'static, + { let z = args.dot(self.weights()) + self.bias(); z.mapv(|x| rho.activate(x)) } @@ -20,7 +23,9 @@ pub trait L { } pub trait Linear { - fn linear(&self, data: &Array2) -> Array2 where T: 'static; + fn linear(&self, data: &Array2) -> Array2 + where + T: 'static; } #[derive(Clone, Debug, Default, PartialEq)] @@ -41,7 +46,12 @@ impl Layer { }; let weights = Array2::ones((features.inputs(), features.outputs())); - Self { bias, features, layer, weights } + Self { + bias, + features, + layer, + weights, + } } pub fn bias(&self) -> &Option> { @@ -60,6 +70,3 @@ impl Layer { self.layer = layer; } } - - - diff --git a/ml/neural/src/lib.rs b/ml/neural/src/lib.rs index ed779bea..9ded1e8d 100644 --- a/ml/neural/src/lib.rs +++ b/ml/neural/src/lib.rs @@ -14,9 +14,12 @@ pub(crate) mod specs; pub(crate) mod utils; pub mod arch; +pub mod bias; pub mod layers; +pub mod models; pub mod neurons; pub mod nn; +pub mod ops; pub mod prop; // pub(crate) use concision_core as core; @@ -25,6 +28,7 @@ pub mod prelude { pub use crate::layers::*; pub use crate::neurons::*; pub use crate::nn::*; + pub use crate::ops::*; pub use crate::prop::*; pub use crate::primitives::*; diff --git a/ml/neural/src/models/mod.rs b/ml/neural/src/models/mod.rs new file mode 100644 index 00000000..3dde2e9a --- /dev/null +++ b/ml/neural/src/models/mod.rs @@ -0,0 +1,13 @@ +/* + Appellation: models + Contrib: FL03 +*/ +//! # Model +//! +pub use self::{model::*, utils::*}; + +pub(crate) mod model; + +pub trait Model {} + +pub(crate) mod utils {} diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs new file mode 100644 index 00000000..e2a0d764 --- /dev/null +++ b/ml/neural/src/models/model.rs @@ -0,0 +1,4 @@ +/* + Appellation: model + Contrib: FL03 +*/ diff --git a/ml/neural/src/ops/dropout.rs b/ml/neural/src/ops/dropout.rs new file mode 100644 index 00000000..80e41f82 --- /dev/null +++ b/ml/neural/src/ops/dropout.rs @@ -0,0 +1,22 @@ +/* + Appellation: dropout + Contrib: FL03 +*/ +use ndarray::prelude::{Array, Ix1}; +use ndarray_rand::rand_distr::Bernoulli; +use ndarray_rand::RandomExt; + +pub fn dropout(array: &Array, p: f64) -> Array +where + T: num::Float, +{ + // Create a Bernoulli distribution for dropout + let distribution = Bernoulli::new(p).unwrap(); + + // Create a mask of the same shape as the input array + let mask: Array = Array::::random(array.dim(), distribution); + let mask = mask.mapv(|x| if x { T::zero() } else { T::one() }); + + // Element-wise multiplication to apply dropout + array * mask +} diff --git a/ml/neural/src/ops/mod.rs b/ml/neural/src/ops/mod.rs new file mode 100644 index 00000000..b7a90f4d --- /dev/null +++ b/ml/neural/src/ops/mod.rs @@ -0,0 +1,30 @@ +/* + Appellation: ops + Contrib: FL03 +*/ +pub use self::{dropout::*, norm::*, utils::*}; + +pub(crate) mod dropout; +pub(crate) mod norm; + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests { + use super::*; + use computare::prelude::RoundTo; + use ndarray::prelude::{array, Array, Ix2}; + + #[test] + fn test_layer_norm() { + let features = 4; + let data: Array = Array::linspace(1., 4., 4) + .into_shape((1, features)) + .unwrap(); + let norm = LayerNorm::::new(features); + let normed = norm.forward(&data); + let rounded = normed.map(|x| x.round_to(4)); + let exp = array![[-1.1619, -0.3873, 0.3873, 1.1619]]; + assert_eq!(rounded, exp); + } +} diff --git a/ml/neural/src/ops/norm.rs b/ml/neural/src/ops/norm.rs new file mode 100644 index 00000000..815044f8 --- /dev/null +++ b/ml/neural/src/ops/norm.rs @@ -0,0 +1,38 @@ +/* + Appellation: norm + Contrib: FL03 +*/ +use ndarray::prelude::{Array1, Array2}; +use ndarray::{Axis, ScalarOperand}; +use num::{Float, FromPrimitive}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +pub struct LayerNorm { + alpha: Array1, + beta: Array1, +} + +impl LayerNorm { + pub fn new(features: usize) -> Self { + Self { + alpha: Array1::ones(features), + beta: Array1::zeros(features), + } + } + + pub fn forward(&self, x: &Array2) -> Array2 + where + T: FromPrimitive + ScalarOperand, + { + let epsilon = T::from(1e-6).unwrap(); + // Calculate the mean and standard deviation of the activations along the feature axis. + let mean = x.mean_axis(Axis(1)).unwrap(); + let std = x.std_axis(Axis(1), T::one()); + // Normalize the activations. + let norm = (x - &mean) / (&std + epsilon); + + // Scale and shift the normalized activations with learnable parameters alpha and beta. + &norm * &self.alpha + &self.beta + } +} diff --git a/ml/neural/src/prop/mod.rs b/ml/neural/src/prop/mod.rs index 6d4b2248..14826613 100644 --- a/ml/neural/src/prop/mod.rs +++ b/ml/neural/src/prop/mod.rs @@ -15,13 +15,13 @@ pub(crate) mod propagation; pub trait Backward { type Output; - fn backward(&mut self, args: T) -> Self::Output; + fn backward(&mut self, args: &T) -> Self::Output; } pub trait Forward { type Output; - fn forward(&mut self, args: T) -> Self::Output; + fn forward(&self, args: &T) -> Self::Output; } pub(crate) mod utils {} diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index d83579af..51cdb5e4 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -3,13 +3,11 @@ Contrib: FL03 */ use ndarray::prelude::Array1; -use ndarray_rand::RandomExt; -use ndarray_rand::rand_distr::Uniform; use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::Uniform; +use ndarray_rand::RandomExt; use num::Float; - - pub trait Bias { fn init_uniform(features: usize) -> Array1 { let k = (T::from(features).unwrap()).sqrt(); @@ -19,7 +17,6 @@ pub trait Bias { fn bias(&self) -> &Array1; fn bias_mut(&mut self) -> &mut Array1; - } pub trait Trainable { diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index 23cdd15c..0365093b 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -5,8 +5,8 @@ use super::params::{HeadShape, QKV}; use super::{Head, Weight}; use crate::neural::neurons::activate::{Activator, Softmax}; -use ndarray::ScalarOperand; use ndarray::prelude::Array2; +use ndarray::ScalarOperand; use num::Float; use serde::{Deserialize, Serialize}; use std::ops; diff --git a/ml/transformers/src/attention/mod.rs b/ml/transformers/src/attention/mod.rs index d5832e1b..c299ad1f 100644 --- a/ml/transformers/src/attention/mod.rs +++ b/ml/transformers/src/attention/mod.rs @@ -16,11 +16,11 @@ pub(crate) mod weights; pub mod multi; pub mod params; -use crate::prelude::BaseDim; use crate::core::prelude::BoxResult; +use crate::prelude::BaseDim; -use ndarray::Dimension; use ndarray::prelude::{Array, Ix2}; +use ndarray::Dimension; use num::Float; /// (batch, sample, seq, model) @@ -36,7 +36,6 @@ pub trait Attention { } pub trait Head { - fn query(&self) -> &Array; fn key(&self) -> &Array; fn value(&self) -> &Array; @@ -51,8 +50,8 @@ pub trait Spaces { } pub(crate) mod utils { - use crate::ops::Split; use crate::neural::prelude::activate::{Activator, Softmax}; + use crate::ops::Split; use ndarray::prelude::{Array2, Array3}; use ndarray::{ScalarOperand, ShapeError}; use num::Float; diff --git a/ml/transformers/src/attention/multi/attention.rs b/ml/transformers/src/attention/multi/attention.rs index 05054e48..38278009 100644 --- a/ml/transformers/src/attention/multi/attention.rs +++ b/ml/transformers/src/attention/multi/attention.rs @@ -2,18 +2,55 @@ Appellation: attention Contrib: FL03 */ -use crate::attention::params::AttentionParameters; +use super::utils::multihead; +use super::MultiHeadParams; use crate::attention::Weight; +use crate::neural::prelude::Forward; +use crate::ops::Split; +use ndarray::prelude::Array2; pub struct MultiHeadAttention { - heads: usize, - model: usize, - weights: Weight, + mask: Array2, + params: MultiHeadParams, + weights: Weight, } impl MultiHeadAttention { - pub fn new(heads: usize, model: usize) -> Self { - let weights = Weight::new((model, model)); - Self { heads, model, weights } - } + pub fn new(heads: usize, model: usize) -> Self { + let params = MultiHeadParams::new(heads, model); + let mask = Array2::::zeros((params.model, params.model)); + let weights = Weight::new((params.model, params.model)); + Self { + mask, + params, + weights, + } + } + + pub fn attention(&self, data: &Array2) -> Array2 { + let weighted = self.weights() * data; + let (q, k, v) = weighted.split(self.params().heads()).unwrap(); + let score = multihead(&q, &k, &v, Some(self.mask().clone())).unwrap(); + score + } + + pub fn mask(&self) -> &Array2 { + &self.mask + } + + pub fn params(&self) -> MultiHeadParams { + self.params + } + + pub fn weights(&self) -> &Weight { + &self.weights + } +} + +impl Forward> for MultiHeadAttention { + type Output = Array2; + + fn forward(&self, data: &Array2) -> Self::Output { + self.attention(data) + } } diff --git a/ml/transformers/src/attention/multi/mod.rs b/ml/transformers/src/attention/multi/mod.rs index 19cbf068..bf02cfdf 100644 --- a/ml/transformers/src/attention/multi/mod.rs +++ b/ml/transformers/src/attention/multi/mod.rs @@ -2,15 +2,15 @@ Appellation: multi Contrib: FL03 */ -pub use self::{attention::*, utils::*}; +pub use self::{attention::*, params::*, utils::*}; pub(crate) mod attention; +pub(crate) mod params; -use crate::ops::Split; -use crate::attention::params::MultiShape; use crate::attention::Weight; use crate::core::prelude::BoxResult; -use ndarray::prelude::{Array2, Array3}; +use crate::ops::Split; +use ndarray::prelude::Array2; use ndarray::ScalarOperand; use num::Float; @@ -20,22 +20,16 @@ where { fn attention(&mut self, data: &Array2) -> BoxResult> { let weighted = self.weights() * data; - let (q, k, v) = weighted.split(self.dim().heads())?; + let (q, k, v) = weighted.split(self.params().heads())?; let score = utils::multihead(&q, &k, &v, Some(self.mask().clone()))?; Ok(score) } - - fn dim(&self) -> MultiShape; - - fn mask(&self) -> &Array2; - fn multihead(&self) -> &Array3; + fn params(&self) -> MultiHeadParams; - fn multihead_mut(&mut self) -> &mut Array3; + fn mask(&self) -> &Array2; fn weights(&self) -> &Weight; - - } pub(crate) mod utils { diff --git a/ml/transformers/src/attention/multi/params.rs b/ml/transformers/src/attention/multi/params.rs new file mode 100644 index 00000000..ed508a7c --- /dev/null +++ b/ml/transformers/src/attention/multi/params.rs @@ -0,0 +1,48 @@ +/* + Appellation: params + Contrib: FL03 +*/ +use crate::{HEADS, MODEL_SIZE}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct MultiHeadParams { + pub heads: usize, + pub model: usize, +} + +impl MultiHeadParams { + pub fn new(heads: usize, model: usize) -> Self { + Self { heads, model } + } + + pub fn heads(&self) -> usize { + self.heads + } + + pub fn model_size(&self) -> usize { + self.model + } + + pub fn query_size(&self) -> usize { + self.model / self.heads + } +} + +impl Default for MultiHeadParams { + fn default() -> Self { + Self::new(HEADS, MODEL_SIZE) + } +} + +impl From for (usize, usize) { + fn from(params: MultiHeadParams) -> Self { + (params.heads, params.model) + } +} + +impl From for [usize; 2] { + fn from(params: MultiHeadParams) -> Self { + [params.heads, params.model] + } +} diff --git a/ml/transformers/src/attention/weights.rs b/ml/transformers/src/attention/weights.rs index 644d98c9..a5720fb5 100644 --- a/ml/transformers/src/attention/weights.rs +++ b/ml/transformers/src/attention/weights.rs @@ -2,21 +2,23 @@ Appellation: weights Contrib: FL03 */ -use crate::ops::Split; use super::params::QKV; +use crate::ops::Split; use ndarray::prelude::{Array, Array2, Array3}; use ndarray::{IntoDimension, Ix2}; use num::Float; use serde::{Deserialize, Serialize}; -use strum::IntoEnumIterator; use std::ops; - +use strum::IntoEnumIterator; pub type WeightTensor = Array; // (seq, model) #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -pub struct Weight where T: Float { +pub struct Weight +where + T: Float, +{ dim: Ix2, pub key: Array2, pub query: Array2, @@ -56,7 +58,7 @@ impl std::fmt::Display for Weight { impl From for Weight where D: IntoDimension, - T: Float + T: Float, { fn from(dim: D) -> Self { let dim = dim.into_dimension(); diff --git a/ml/transformers/src/codec/decode/params.rs b/ml/transformers/src/codec/decode/params.rs index fe8fd57a..e1913f35 100644 --- a/ml/transformers/src/codec/decode/params.rs +++ b/ml/transformers/src/codec/decode/params.rs @@ -1,4 +1,4 @@ /* Appellation: params Contrib: FL03 -*/ \ No newline at end of file +*/ diff --git a/ml/transformers/src/codec/encode/encoder.rs b/ml/transformers/src/codec/encode/encoder.rs index fd7f8b1b..fd433cd9 100644 --- a/ml/transformers/src/codec/encode/encoder.rs +++ b/ml/transformers/src/codec/encode/encoder.rs @@ -5,18 +5,53 @@ use super::EncoderParams; use crate::attention::multi::MultiHeadAttention; use crate::ffn::FFN; +use crate::neural::prelude::{Forward, LayerNorm}; +use ndarray::prelude::Array2; +pub struct Sublayer { + layer: T, + norm: LayerNorm, +} pub struct Encoder { - attention: MultiHeadAttention, - network: FFN, - params: EncoderParams, + attention: MultiHeadAttention, + network: FFN, + norm_attention: LayerNorm, + norm_network: LayerNorm, + params: EncoderParams, } impl Encoder { - pub fn new(params: EncoderParams) -> Self { - let attention = MultiHeadAttention::new(params.heads, params.model); - let network = FFN::new(params.model, None); - Self { attention, network, params, } - } + pub fn new(params: EncoderParams) -> Self { + let attention = MultiHeadAttention::new(params.heads, params.model); + let network = FFN::new(params.model, None); + let norm = LayerNorm::new(params.model); + Self { + attention, + network, + norm_attention: norm.clone(), + norm_network: norm, + params, + } + } + + pub fn forward(&mut self, data: &Array2) -> Array2 { + let attention = data + self.attention.attention(data); + let norm = self.norm_attention.forward(&attention); + let network = data + self.network.forward(&norm); + let norm = self.norm_network.forward(&network); + norm + } + + pub fn _forward(&mut self, data: &Array2) -> Array2 { + let norm = self.norm_attention.forward(data); + let attention = data + self.attention.attention(&norm); + let norm = self.norm_network.forward(&attention); + let network = data + self.network.forward(&norm); + network + } + + pub fn params(&self) -> EncoderParams { + self.params + } } diff --git a/ml/transformers/src/codec/encode/params.rs b/ml/transformers/src/codec/encode/params.rs index c244a1db..b1cb416d 100644 --- a/ml/transformers/src/codec/encode/params.rs +++ b/ml/transformers/src/codec/encode/params.rs @@ -2,10 +2,35 @@ Appellation: params Contrib: FL03 */ +use crate::{HEADS, MODEL_SIZE}; use serde::{Deserialize, Serialize}; -#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] pub struct EncoderParams { pub heads: usize, pub model: usize, -} \ No newline at end of file +} + +impl EncoderParams { + pub fn new(heads: usize, model: usize) -> Self { + Self { heads, model } + } + + pub fn heads(&self) -> usize { + self.heads + } + + pub fn model_size(&self) -> usize { + self.model + } + + pub fn query_size(&self) -> usize { + self.model / self.heads + } +} + +impl Default for EncoderParams { + fn default() -> Self { + Self::new(HEADS, MODEL_SIZE) + } +} diff --git a/ml/transformers/src/codec/encode/stack.rs b/ml/transformers/src/codec/encode/stack.rs index 11a7a203..5b4c321e 100644 --- a/ml/transformers/src/codec/encode/stack.rs +++ b/ml/transformers/src/codec/encode/stack.rs @@ -5,21 +5,25 @@ use super::{Encoder, EncoderParams}; pub struct EncoderStack { - layers: usize, - params: EncoderParams, - stack: Vec, + layers: usize, + params: EncoderParams, + stack: Vec, } impl EncoderStack { - pub fn new(layers: usize, params: EncoderParams) -> Self { - let stack = Vec::with_capacity(layers); + pub fn new(layers: usize, params: EncoderParams) -> Self { + let stack = Vec::with_capacity(layers); - Self { layers, params, stack } - } + Self { + layers, + params, + stack, + } + } - pub fn setup(&mut self) { - for _ in 0..self.layers { - self.stack.push(Encoder::new(self.params)); - } - } + pub fn setup(&mut self) { + for _ in 0..self.layers { + self.stack.push(Encoder::new(self.params)); + } + } } diff --git a/ml/transformers/src/ffn/network.rs b/ml/transformers/src/ffn/network.rs index b83c6db1..67d98dda 100644 --- a/ml/transformers/src/ffn/network.rs +++ b/ml/transformers/src/ffn/network.rs @@ -5,6 +5,7 @@ use super::FFNParams; use crate::data::linear::LinearLayer; use crate::neural::neurons::activate::{Activator, ReLU}; +use crate::neural::prelude::Forward; use ndarray::prelude::Array2; use serde::{Deserialize, Serialize}; @@ -25,8 +26,12 @@ impl FFN { params, } } +} + +impl Forward> for FFN { + type Output = Array2; - pub fn forward(&self, data: &Array2) -> Array2 { + fn forward(&self, data: &Array2) -> Self::Output { self.output.linear(&ReLU::rho(self.input.linear(data))) } } diff --git a/ml/transformers/src/ffn/params.rs b/ml/transformers/src/ffn/params.rs index 1474c9aa..83f388a3 100644 --- a/ml/transformers/src/ffn/params.rs +++ b/ml/transformers/src/ffn/params.rs @@ -31,4 +31,4 @@ impl Default for FFNParams { network: crate::NETWORK_SIZE, } } -} \ No newline at end of file +} diff --git a/ml/transformers/src/ops/mod.rs b/ml/transformers/src/ops/mod.rs index d0418912..65867743 100644 --- a/ml/transformers/src/ops/mod.rs +++ b/ml/transformers/src/ops/mod.rs @@ -2,10 +2,9 @@ Appellation: ops Contrib: FL03 */ -pub use self::{merge::*, norm::*, split::*, utils::*}; +pub use self::{merge::*, split::*, utils::*}; pub(crate) mod merge; -pub(crate) mod norm; pub(crate) mod split; pub(crate) mod utils { diff --git a/ml/transformers/src/ops/norm.rs b/ml/transformers/src/ops/norm.rs deleted file mode 100644 index 5f9c6360..00000000 --- a/ml/transformers/src/ops/norm.rs +++ /dev/null @@ -1,61 +0,0 @@ -/* - Appellation: norm - Contrib: FL03 -*/ -use ndarray::{Axis, ScalarOperand}; -use ndarray::prelude::{Array1, Array2}; -use num::{Float, FromPrimitive}; -use serde::{Deserialize, Serialize}; - -pub fn layer_normalize(x: &Array2) -> Array2 -where - T: Float + FromPrimitive + ScalarOperand, -{ - // Calculate the mean and variance of the activations along the feature axis. - let mean = x.mean_axis(Axis(1)).unwrap(); - let var = x.var_axis(Axis(1), T::one()); - - // Normalize the activations. - let epsilon = T::from(1e-6).unwrap(); - let scale = (var + epsilon).mapv(|i| T::one() / i.sqrt()); - let normalized_x = (x - &mean) * &scale; - - // Scale and shift the normalized activations with learnable parameters gamma and beta. - let gamma = Array1::ones(mean.dim()); - let beta = Array1::zeros(mean.dim()); - let scaled_shifted_x = &normalized_x * &gamma + β - - scaled_shifted_x -} - - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -pub struct Normalize { - alpha: Array1, - beta: Array1, -} - -impl Normalize { - pub fn new(features: usize) -> Self { - Self { - alpha: Array1::ones(features), - beta: Array1::zeros(features), - } - } - - pub fn forward(&self, x: &Array2) -> Array2 where T: FromPrimitive + ScalarOperand { - // Calculate the mean and variance of the activations along the feature axis. - let mean = x.mean_axis(Axis(1)).unwrap(); - let var = x.var_axis(Axis(1), T::one()); - - // Normalize the activations. - let epsilon = T::from(1e-6).unwrap(); - let scale = (var + epsilon).mapv(|i| T::one() / i.sqrt()); - let normalized_x = (x - &mean) * &scale; - - // Scale and shift the normalized activations with learnable parameters gamma and beta. - let scaled_shifted_x = &normalized_x * &self.alpha + &self.beta; - - scaled_shifted_x - } -} \ No newline at end of file diff --git a/ml/transformers/src/ops/split.rs b/ml/transformers/src/ops/split.rs index a52154c5..484435cb 100644 --- a/ml/transformers/src/ops/split.rs +++ b/ml/transformers/src/ops/split.rs @@ -3,7 +3,17 @@ Contrib: FL03 */ use ndarray::prelude::{Array2, Array3, Array4}; -use ndarray::ShapeError; +use ndarray::{Dimension, ShapeError}; + +// pub fn split(param: &Array, heads: usize) -> Result, ShapeError> { +// let mut dim = param.dim() +// let query = param.shape().last().unwrap() / heads; +// // reshape the qkv matrix into a 3d array +// let mut res = param.clone().into_shape((param.shape()[0], heads, query))?; +// // swap the sequence and head axes +// res.swap_axes(0, 1); +// Ok(res) +// } pub trait Split { type Error; diff --git a/ml/transformers/src/primitives.rs b/ml/transformers/src/primitives.rs index c787b326..851c0fd0 100644 --- a/ml/transformers/src/primitives.rs +++ b/ml/transformers/src/primitives.rs @@ -21,7 +21,7 @@ pub(crate) mod statics { use super::constants::*; use lazy_static::lazy_static; - lazy_static!{ + lazy_static! { /// The default dimensions of the query, key, and value tensors w/r/2 a single head pub static ref QUERY_SIZE: usize = MODEL_SIZE / HEADS; } diff --git a/ml/transformers/src/transform/params.rs b/ml/transformers/src/transform/params.rs index c6720ad2..1b66b482 100644 --- a/ml/transformers/src/transform/params.rs +++ b/ml/transformers/src/transform/params.rs @@ -4,11 +4,13 @@ */ use serde::{Deserialize, Serialize}; -#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] pub struct TransformerParams { pub batch: usize, pub heads: usize, pub layers: usize, pub model: usize, pub samples: usize, -} \ No newline at end of file +} From 3cd84e454d2c437ab9d40348cf044425a7b41bcf Mon Sep 17 00:00:00 2001 From: FL03 Date: Thu, 2 Nov 2023 15:26:25 -0500 Subject: [PATCH 036/118] update Signed-off-by: FL03 --- ml/neural/src/bias.rs | 56 +++++++++++++++++++- ml/neural/src/layers/layer.rs | 32 +++++++++--- ml/neural/src/layers/mod.rs | 3 +- ml/neural/src/layers/sublayer.rs | 28 ++++++++++ ml/neural/src/nn/mod.rs | 5 +- ml/neural/src/nn/network.rs | 5 +- ml/neural/src/ops/dropout.rs | 28 +++++++++- ml/neural/src/specs.rs | 2 +- ml/nlp/src/encode/mod.rs | 23 +++++++++ ml/nlp/src/encode/positional.rs | 57 +++++++++++++++++++++ ml/nlp/src/lib.rs | 2 + ml/transformers/src/attention/mod.rs | 13 +---- ml/transformers/src/codec/encode/encoder.rs | 13 ++--- ml/transformers/src/codec/encode/mod.rs | 30 +++-------- ml/transformers/src/ops/split.rs | 2 +- 15 files changed, 240 insertions(+), 59 deletions(-) create mode 100644 ml/neural/src/layers/sublayer.rs create mode 100644 ml/nlp/src/encode/mod.rs create mode 100644 ml/nlp/src/encode/positional.rs diff --git a/ml/neural/src/bias.rs b/ml/neural/src/bias.rs index cdc22c3a..28e83fa1 100644 --- a/ml/neural/src/bias.rs +++ b/ml/neural/src/bias.rs @@ -2,10 +2,14 @@ Appellation: bias Contrib: FL03 */ -use ndarray::prelude::Array1; +use ndarray::Dimension; +use ndarray::prelude::{Array, Array1}; use ndarray_rand::rand_distr::{uniform::SampleUniform, Uniform}; use ndarray_rand::RandomExt; use num::Float; +use serde::{Deserialize, Serialize}; +use smart_default::SmartDefault; +use std::ops; fn _generate_bias(size: usize) -> Array1 { let ds = (T::from(size).unwrap()).sqrt(); @@ -13,8 +17,10 @@ fn _generate_bias(size: usize) -> Array1 { Array1::::random(size, dist) } +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, SmartDefault)] pub enum Bias { Biased(Array1), + #[default] Unbiased, } @@ -34,3 +40,51 @@ impl Bias { } } } + +impl ops::Add> for Bias where Array: ops::Add, Output = Array> { + type Output = Array; + + fn add(self, rhs: Array) -> Self::Output { + match self { + Self::Biased(bias) => rhs + bias, + Self::Unbiased => rhs, + } + } +} + +impl ops::Add<&Array> for Bias where Array: ops::Add, Output = Array> { + type Output = Array; + + fn add(self, rhs: &Array) -> Self::Output { + match self { + Self::Biased(bias) => rhs.clone() + bias, + Self::Unbiased => rhs.clone(), + } + } +} + +impl ops::Add> for Array where Array: ops::Add, Output = Array> { + type Output = Array; + + fn add(self, bias: Bias) -> Self::Output { + match bias.clone() { + Bias::Biased(bias) => self.clone() + bias, + Bias::Unbiased => self.clone(), + } + } +} + +impl ops::Add<&Bias> for Array where Array: ops::Add, Output = Array> { + type Output = Array; + + fn add(self, bias: &Bias) -> Self::Output { + match bias.clone() { + Bias::Biased(bias) => self.clone() + bias, + Bias::Unbiased => self.clone(), + } + } +} + + + + diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 5176e85f..7f78600a 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -3,8 +3,12 @@ Contrib: FL03 */ use super::{Features, LayerType}; +use crate::bias::Bias; use crate::neurons::activate::Activator; +use crate::prop::Forward; + use ndarray::prelude::{Array1, Array2}; +use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; pub trait L { @@ -29,20 +33,20 @@ pub trait Linear { } #[derive(Clone, Debug, Default, PartialEq)] -pub struct Layer { - bias: Option>, +pub struct Layer { + bias: Bias, features: Features, layer: LayerType, - weights: Array2, + weights: Array2, } -impl Layer { - pub fn new(inputs: usize, outputs: usize, bias: bool, layer: LayerType) -> Self { +impl Layer { + pub fn new(inputs: usize, outputs: usize, bias: bool, layer: LayerType) -> Self where T: SampleUniform { let features = Features::new(inputs, outputs); let bias = if bias { - Some(Array1::ones(outputs)) + Bias::biased(outputs) } else { - None + Bias::default() }; let weights = Array2::ones((features.inputs(), features.outputs())); @@ -54,7 +58,7 @@ impl Layer { } } - pub fn bias(&self) -> &Option> { + pub fn bias(&self) -> &Bias { &self.bias } @@ -69,4 +73,16 @@ impl Layer { pub fn set_layer(&mut self, layer: LayerType) { self.layer = layer; } + + pub fn weights(&self) -> &Array2 { + &self.weights + } } + +impl Forward> for Layer { + type Output = Array2; + + fn forward(&self, data: &Array2) -> Self::Output { + data.dot(&self.weights().t()) + self.bias() + } +} \ No newline at end of file diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index 0c9d9962..f84b984f 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -3,11 +3,12 @@ Contrib: FL03 */ //! # Layers -pub use self::{features::*, kinds::*, layer::*, utils::*}; +pub use self::{features::*, kinds::*, layer::*, sublayer::*, utils::*}; pub(crate) mod features; pub(crate) mod kinds; pub(crate) mod layer; +pub(crate) mod sublayer; pub(crate) mod utils {} diff --git a/ml/neural/src/layers/sublayer.rs b/ml/neural/src/layers/sublayer.rs new file mode 100644 index 00000000..fc6a82e2 --- /dev/null +++ b/ml/neural/src/layers/sublayer.rs @@ -0,0 +1,28 @@ +/* + Appellation: sublayers + Contrib: FL03 +*/ +use super::Layer; +use crate::ops::LayerNorm; +use crate::prop::Forward; + +use ndarray::ScalarOperand; +use ndarray::prelude::Array2; +use num::{Float, FromPrimitive}; + +pub struct Sublayer { + layer: Layer, + norm: LayerNorm, +} + +impl Sublayer { + pub fn new(layer: Layer, norm: LayerNorm) -> Self { + Self { layer, norm } + } + + pub fn forward(&self, data: &Array2) -> Array2 where T: FromPrimitive + ScalarOperand { + let norm = self.norm.forward(data); + let layer = data + self.layer.forward(&norm); + layer + } +} \ No newline at end of file diff --git a/ml/neural/src/nn/mod.rs b/ml/neural/src/nn/mod.rs index e4c4d54d..ac22f117 100644 --- a/ml/neural/src/nn/mod.rs +++ b/ml/neural/src/nn/mod.rs @@ -11,13 +11,14 @@ pub(crate) mod network; use crate::layers::Layer; use crate::Trainable; +use num::Float; -pub trait NeuralNet: Trainable { +pub trait NeuralNet: Trainable { fn depth(&self) -> usize { self.layers().len() } - fn layers(&self) -> &[Layer]; + fn layers(&self) -> &[Layer]; } pub(crate) mod utils {} diff --git a/ml/neural/src/nn/network.rs b/ml/neural/src/nn/network.rs index 29c1c417..16df0f3a 100644 --- a/ml/neural/src/nn/network.rs +++ b/ml/neural/src/nn/network.rs @@ -3,7 +3,8 @@ Contrib: FL03 */ use crate::layers::Layer; +use num::Float; -pub struct NeuralNetwork { - pub layers: Vec, +pub struct NeuralNetwork { + pub layers: Vec>, } diff --git a/ml/neural/src/ops/dropout.rs b/ml/neural/src/ops/dropout.rs index 80e41f82..f44602de 100644 --- a/ml/neural/src/ops/dropout.rs +++ b/ml/neural/src/ops/dropout.rs @@ -5,10 +5,12 @@ use ndarray::prelude::{Array, Ix1}; use ndarray_rand::rand_distr::Bernoulli; use ndarray_rand::RandomExt; +use num::Float; +use serde::{Deserialize, Serialize}; pub fn dropout(array: &Array, p: f64) -> Array where - T: num::Float, + T: Float, { // Create a Bernoulli distribution for dropout let distribution = Bernoulli::new(p).unwrap(); @@ -20,3 +22,27 @@ where // Element-wise multiplication to apply dropout array * mask } + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, PartialOrd, Serialize)] +pub struct Dropout { + p: f64, +} + +impl Dropout { + pub fn new(p: f64) -> Self { + Self { p } + } + + pub fn apply(&self, array: &Array) -> Array + where + T: Float, + { + dropout(array, self.p) + } +} + +impl Default for Dropout { + fn default() -> Self { + Self::new(0.5) + } +} diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index 51cdb5e4..cea270a6 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -8,7 +8,7 @@ use ndarray_rand::rand_distr::Uniform; use ndarray_rand::RandomExt; use num::Float; -pub trait Bias { +pub trait Biased { fn init_uniform(features: usize) -> Array1 { let k = (T::from(features).unwrap()).sqrt(); let uniform = Uniform::new(-k, k); diff --git a/ml/nlp/src/encode/mod.rs b/ml/nlp/src/encode/mod.rs new file mode 100644 index 00000000..f23c8b91 --- /dev/null +++ b/ml/nlp/src/encode/mod.rs @@ -0,0 +1,23 @@ +/* + Appellation: encode + Contrib: FL03 +*/ +pub use self::utils::*; + +pub mod positional; + +use ndarray::Dimension; +use ndarray::prelude::{Array, Array2}; + +pub trait Encode { + type Output; + + fn encode(&self, data: &T) -> Self::Output; +} + +pub trait EncodeArr { + type Dim: Dimension; + + fn encode(&self, data: &Array) -> Array2; +} +pub(crate) mod utils {} diff --git a/ml/nlp/src/encode/positional.rs b/ml/nlp/src/encode/positional.rs new file mode 100644 index 00000000..730dbfff --- /dev/null +++ b/ml/nlp/src/encode/positional.rs @@ -0,0 +1,57 @@ +/* + Appellation: positional + Contrib: FL03 +*/ + +use ndarray::Array2; + +pub fn get_position_encoding(seq_len: usize, d: usize, n: f64) -> Array2 { + let denom = | i: usize | f64::powf(n, 2.0 * (i as f64) / d as f64); + let mut p = Array2::zeros((seq_len, d)); + for k in 0..seq_len { + for i in 0..d / 2 { + p[[k, 2 * i]] = (k as f64 / denom(i)).sin(); + p[[k, 2 * i + 1]] = (k as f64 / denom(i)).cos(); + } + } + p +} + +pub struct PositionalEncoder { + model: usize, + sequence: usize, + samples: usize, +} + +impl PositionalEncoder { + pub fn new(model: usize, sequence: usize, samples: usize) -> Self { + Self { + model, + sequence, + samples, + } + } + + pub fn encode(&self, data: &Array2) -> Array2 { + let x = data * (self.model as f64).sqrt(); + x + self.positional() + + } + + pub fn positional(&self) -> Array2 { + get_position_encoding(self.sequence, self.model, self.samples as f64) + } +} + + +#[cfg(test)] +mod tests { + use super::*; + use ndarray::array; + + #[test] + fn test_positional_encoding() { + let p = get_position_encoding(4, 4, 10000.); + assert_eq!(p.row(0), array![0.0, 1.0, 0.0, 1.0]); + } +} diff --git a/ml/nlp/src/lib.rs b/ml/nlp/src/lib.rs index 3a81de3f..9c75e27c 100644 --- a/ml/nlp/src/lib.rs +++ b/ml/nlp/src/lib.rs @@ -10,9 +10,11 @@ pub(crate) mod specs; pub(crate) mod utils; pub mod embed; +pub mod encode; pub mod prelude { pub use crate::embed::*; + pub use crate::encode::*; pub use crate::primitives::*; pub use crate::specs::*; diff --git a/ml/transformers/src/attention/mod.rs b/ml/transformers/src/attention/mod.rs index c299ad1f..bfd7b54e 100644 --- a/ml/transformers/src/attention/mod.rs +++ b/ml/transformers/src/attention/mod.rs @@ -51,19 +51,10 @@ pub trait Spaces { pub(crate) mod utils { use crate::neural::prelude::activate::{Activator, Softmax}; - use crate::ops::Split; - use ndarray::prelude::{Array2, Array3}; - use ndarray::{ScalarOperand, ShapeError}; + use ndarray::prelude::Array2; + use ndarray::ScalarOperand; use num::Float; - pub fn linear_layer( - data: &Array2, - weights: &Array2, - heads: usize, - ) -> Result, ShapeError> { - data.dot(weights).split(heads) - } - pub fn compute_attention( query: &Array2, key: &Array2, diff --git a/ml/transformers/src/codec/encode/encoder.rs b/ml/transformers/src/codec/encode/encoder.rs index fd433cd9..9803bf5c 100644 --- a/ml/transformers/src/codec/encode/encoder.rs +++ b/ml/transformers/src/codec/encode/encoder.rs @@ -8,10 +8,6 @@ use crate::ffn::FFN; use crate::neural::prelude::{Forward, LayerNorm}; use ndarray::prelude::Array2; -pub struct Sublayer { - layer: T, - norm: LayerNorm, -} pub struct Encoder { attention: MultiHeadAttention, @@ -25,17 +21,16 @@ impl Encoder { pub fn new(params: EncoderParams) -> Self { let attention = MultiHeadAttention::new(params.heads, params.model); let network = FFN::new(params.model, None); - let norm = LayerNorm::new(params.model); Self { attention, network, - norm_attention: norm.clone(), - norm_network: norm, + norm_attention: LayerNorm::new(params.model), + norm_network: LayerNorm::new(params.model), params, } } - pub fn forward(&mut self, data: &Array2) -> Array2 { + fn _forward(&self, data: &Array2) -> Array2 { let attention = data + self.attention.attention(data); let norm = self.norm_attention.forward(&attention); let network = data + self.network.forward(&norm); @@ -43,7 +38,7 @@ impl Encoder { norm } - pub fn _forward(&mut self, data: &Array2) -> Array2 { + pub fn forward(&mut self, data: &Array2) -> Array2 { let norm = self.norm_attention.forward(data); let attention = data + self.attention.attention(&norm); let norm = self.norm_network.forward(&attention); diff --git a/ml/transformers/src/codec/encode/mod.rs b/ml/transformers/src/codec/encode/mod.rs index 206c9640..9298e844 100644 --- a/ml/transformers/src/codec/encode/mod.rs +++ b/ml/transformers/src/codec/encode/mod.rs @@ -11,33 +11,19 @@ pub(crate) mod stack; pub trait Encode {} -pub(crate) mod utils { - use ndarray::Array2; - - pub fn get_position_encoding(seq_len: usize, d: usize, n: f64) -> Array2 { - let mut p = Array2::zeros((seq_len, d)); - for k in 0..seq_len { - for i in 0..d / 2 { - let denominator = f64::powf(n, 2.0 * i as f64 / d as f64); - p[[k, 2 * i]] = (k as f64 / denominator).sin(); - p[[k, 2 * i + 1]] = (k as f64 / denominator).cos(); - } - } - p - } -} +pub(crate) mod utils {} #[cfg(test)] mod tests { use super::*; - use ndarray::array; - - #[test] - fn test_encoder() {} + use ndarray::Array2; #[test] - fn test_positional_encoding() { - let p = get_position_encoding(4, 4, 1000.); - assert_eq!(p.row(0), array![0.0, 1.0, 0.0, 1.0]); + fn test_encoder() { + let (heads, model) = (8, 512); + let _data = Array2::::zeros((512, 512)); + let params = EncoderParams::new(heads, model); + let encoder = Encoder::new(params); + assert_eq!(encoder.params().heads(), heads); } } diff --git a/ml/transformers/src/ops/split.rs b/ml/transformers/src/ops/split.rs index 484435cb..13c367e4 100644 --- a/ml/transformers/src/ops/split.rs +++ b/ml/transformers/src/ops/split.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use ndarray::prelude::{Array2, Array3, Array4}; -use ndarray::{Dimension, ShapeError}; +use ndarray::ShapeError; // pub fn split(param: &Array, heads: usize) -> Result, ShapeError> { // let mut dim = param.dim() From bb0ae07b15a8a3f24cf6b7edfec4f7a9ee93cc81 Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 3 Nov 2023 14:08:07 -0500 Subject: [PATCH 037/118] update Signed-off-by: FL03 --- Cargo.toml | 1 + core/src/errors/error.rs | 13 +- core/src/specs.rs | 59 +++++++- ml/neural/Cargo.toml | 1 + ml/neural/src/{bias.rs => bias/biases.rs} | 45 +++--- ml/neural/src/bias/mask.rs | 142 ++++++++++++++++++ ml/neural/src/bias/mod.rs | 23 +++ ml/neural/src/layers/layer.rs | 58 ++++--- ml/neural/src/layers/linear/layer.rs | 59 ++++++++ ml/neural/src/layers/linear/mod.rs | 55 +++++++ ml/neural/src/layers/mod.rs | 27 ++++ ml/neural/src/layers/sublayer.rs | 11 +- ml/neural/src/lib.rs | 6 +- ml/neural/src/neurons/activate/activator.rs | 42 ++++++ ml/neural/src/neurons/activate/binary.rs | 18 ++- ml/neural/src/neurons/activate/mod.rs | 31 ++-- ml/neural/src/neurons/activate/nonlinear.rs | 77 +++++++--- ml/neural/src/neurons/mod.rs | 20 +-- ml/neural/src/nn/mod.rs | 2 +- ml/neural/src/specs.rs | 27 ++-- ml/neural/src/weights/mod.rs | 35 +++++ ml/neural/src/weights/weight.rs | 32 ++++ ml/nlp/src/encode/mod.rs | 2 +- ml/nlp/src/encode/positional.rs | 4 +- ml/transformers/Cargo.toml | 3 + ml/transformers/src/attention/head.rs | 44 +++--- ml/transformers/src/attention/mod.rs | 59 ++++++-- .../src/attention/multi/attention.rs | 70 +++++---- ml/transformers/src/attention/multi/mod.rs | 51 +++++-- ml/transformers/src/attention/params/dim.rs | 64 ++++---- ml/transformers/src/attention/weights.rs | 104 ++++++++++--- ml/transformers/src/codec/encode/encoder.rs | 15 +- ml/transformers/src/ffn/network.rs | 4 +- ml/transformers/src/ops/merge.rs | 19 ++- 34 files changed, 947 insertions(+), 276 deletions(-) rename ml/neural/src/{bias.rs => bias/biases.rs} (72%) create mode 100644 ml/neural/src/bias/mask.rs create mode 100644 ml/neural/src/bias/mod.rs create mode 100644 ml/neural/src/layers/linear/layer.rs create mode 100644 ml/neural/src/layers/linear/mod.rs create mode 100644 ml/neural/src/neurons/activate/activator.rs create mode 100644 ml/neural/src/weights/mod.rs create mode 100644 ml/neural/src/weights/weight.rs diff --git a/Cargo.toml b/Cargo.toml index 42861539..3e638501 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ computare = { features = ["full"], branch = "v0.1.0", git = "https://github.com/ anyhow = "1" lazy_static = "1" ndarray = { features = ["serde-1"], version = "0.15" } +# ndarray-linalg = { features = [], version = "0.16" } ndarray-rand = { features = [], version = "0.14" } ndarray-stats = { features = [], version = "0.5.1" } num = { features = ["serde"], version = "0.4" } diff --git a/core/src/errors/error.rs b/core/src/errors/error.rs index 1815d3e9..5bcff20b 100644 --- a/core/src/errors/error.rs +++ b/core/src/errors/error.rs @@ -26,6 +26,7 @@ use strum::{Display, EnumIs, EnumIter, EnumVariantNames}; #[strum(serialize_all = "lowercase")] pub enum Errors { Async, + Codec, Connection, Data, Dimension, @@ -34,6 +35,7 @@ pub enum Errors { Execution, IO, Null, + Parse, Process, Runtime, Syntax, @@ -125,13 +127,13 @@ impl From for Error { impl From for Error { fn from(err: std::num::ParseFloatError) -> Self { - Self::new(Errors::Syntax, err.to_string()) + Self::new(Errors::Parse, err.to_string()) } } impl From for Error { fn from(err: std::num::ParseIntError) -> Self { - Self::new(Errors::Syntax, err.to_string()) + Self::new(Errors::Parse, err.to_string()) } } @@ -146,3 +148,10 @@ impl From for Error { Self::new(Errors::Dimension, err.to_string()) } } + +impl From for Error { + fn from(err: serde_json::Error) -> Self { + Self::new(Errors::Syntax, err.to_string()) + } +} + diff --git a/core/src/specs.rs b/core/src/specs.rs index d654aa70..d23453e5 100644 --- a/core/src/specs.rs +++ b/core/src/specs.rs @@ -2,9 +2,13 @@ Appellation: specs Contrib: FL03 */ -use num::{Num, One}; +use ndarray::{Dimension, ShapeError}; +use ndarray::prelude::{Array}; +use num::{Num, One, Zero}; use std::ops::MulAssign; +pub trait BinaryNum: One + Zero {} + pub trait Pair { fn pair(&self) -> (A, B); } @@ -40,6 +44,59 @@ where } } + +trait Matmul +where + T: Num, + D: Dimension, +{ + fn matmul(&self, other: &Array) -> Result, ShapeError>; + + fn shape(&self) -> D; +} + +// impl Matmul for Array +// where +// T: Num + std::ops::Mul + std::ops::Add + Clone, +// D: Dimension, +// { +// fn matmul(&self, other: &Array) -> Result, ShapeError> { +// let self_shape = self.shape(); +// let other_shape = other.shape(); + +// if self_shape[self.ndim() - 1] != other_shape[self.ndim() - 2] { +// return Err(ShapeError::from_kind(ndarray::ErrorKind::IncompatibleShape)); +// } + +// let mut result = Array::zeros(self_shape); + +// let mut self_shape = self_shape.to_vec(); +// let self_last = self_shape.pop().unwrap(); +// let other_shape = other_shape.to_vec(); + +// let mut iter_self = self.iter(); +// let mut iter_other = other.iter(); + + + +// for mut row_result in result.outer_iter_mut() { +// for mut col_other in other.inner_iter() { +// let row_self = iter_self.clone(); +// let mut col_other = col_other.clone(); +// let dot = dot_product(&mut row_self, &mut col_other, self_last, &other_shape); +// row_result.assign(&dot); +// } +// iter_self.step_by(self_shape.last().unwrap().index()); +// } + +// Ok(result) +// } + +// fn shape(&self) -> D { +// self.raw_dim() +// } +// } + #[cfg(test)] mod tests { use super::*; diff --git a/ml/neural/Cargo.toml b/ml/neural/Cargo.toml index 203ece4d..011259f5 100644 --- a/ml/neural/Cargo.toml +++ b/ml/neural/Cargo.toml @@ -28,6 +28,7 @@ concision-core.workspace = true anyhow.workspace = true ndarray.workspace = true +# ndarray-linalg.workspace = true ndarray-rand.workspace = true ndarray-stats.workspace = true num.workspace = true diff --git a/ml/neural/src/bias.rs b/ml/neural/src/bias/biases.rs similarity index 72% rename from ml/neural/src/bias.rs rename to ml/neural/src/bias/biases.rs index 28e83fa1..4a11a5ce 100644 --- a/ml/neural/src/bias.rs +++ b/ml/neural/src/bias/biases.rs @@ -2,13 +2,14 @@ Appellation: bias Contrib: FL03 */ -use ndarray::Dimension; use ndarray::prelude::{Array, Array1}; +use ndarray::Dimension; use ndarray_rand::rand_distr::{uniform::SampleUniform, Uniform}; use ndarray_rand::RandomExt; use num::Float; use serde::{Deserialize, Serialize}; use smart_default::SmartDefault; +use strum::EnumIs; use std::ops; fn _generate_bias(size: usize) -> Array1 { @@ -17,7 +18,7 @@ fn _generate_bias(size: usize) -> Array1 { Array1::::random(size, dist) } -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, SmartDefault)] +#[derive(Clone, Debug, Deserialize, EnumIs, PartialEq, Serialize, SmartDefault)] pub enum Bias { Biased(Array1), #[default] @@ -25,14 +26,6 @@ pub enum Bias { } impl Bias { - pub fn biased(size: usize) -> Self - where - T: SampleUniform, - { - let bias = _generate_bias(size); - Self::Biased(bias) - } - pub fn forward(&self, data: &Array1) -> Array1 { match self { Self::Biased(bias) => data + bias, @@ -41,7 +34,18 @@ impl Bias { } } -impl ops::Add> for Bias where Array: ops::Add, Output = Array> { +impl Bias where T: Float + SampleUniform { + pub fn biased(size: usize) -> Self + { + let bias = _generate_bias(size); + Self::Biased(bias) + } +} + +impl ops::Add> for Bias +where + Array: ops::Add, Output = Array>, +{ type Output = Array; fn add(self, rhs: Array) -> Self::Output { @@ -52,7 +56,10 @@ impl ops::Add> for Bias where Array } } -impl ops::Add<&Array> for Bias where Array: ops::Add, Output = Array> { +impl ops::Add<&Array> for Bias +where + Array: ops::Add, Output = Array>, +{ type Output = Array; fn add(self, rhs: &Array) -> Self::Output { @@ -63,7 +70,10 @@ impl ops::Add<&Array> for Bias where Array ops::Add> for Array where Array: ops::Add, Output = Array> { +impl ops::Add> for Array +where + Array: ops::Add, Output = Array>, +{ type Output = Array; fn add(self, bias: Bias) -> Self::Output { @@ -74,7 +84,10 @@ impl ops::Add> for Array where Array } } -impl ops::Add<&Bias> for Array where Array: ops::Add, Output = Array> { +impl ops::Add<&Bias> for Array +where + Array: ops::Add, Output = Array>, +{ type Output = Array; fn add(self, bias: &Bias) -> Self::Output { @@ -84,7 +97,3 @@ impl ops::Add<&Bias> for Array where Array + Contrib: FL03 +*/ +use ndarray::prelude::{Array, Array2}; +use ndarray::Dimension; +use ndarray_rand::rand_distr::{uniform::SampleUniform, Uniform}; +use ndarray_rand::RandomExt; +use num::Float; +use serde::{Deserialize, Serialize}; +use smart_default::SmartDefault; +use strum::EnumIs; +use std::ops; + +#[derive(Clone, Debug, Deserialize, EnumIs, PartialEq, Serialize, SmartDefault)] +pub enum Mask { + Masked(Array2), + #[default] + Unmasked, +} + +impl Mask { + pub fn forward(&self, data: &Array2) -> Array2 { + match self { + Self::Masked(bias) => data + bias, + Self::Unmasked => data.clone(), + } + } +} + +impl Mask where T: Float + SampleUniform { + pub fn masked(size: usize) -> Self + { + let ds = (T::from(size).unwrap()).sqrt(); + let dist = Uniform::new(-ds, ds); + let mask = Array2::::random((size, size), dist); + Self::Masked(mask) + } +} + +impl From for Mask where T: Float + SampleUniform { + fn from(size: usize) -> Self + { + let ds = (T::from(size).unwrap()).sqrt(); + let dist = Uniform::new(-ds, ds); + let mask = Array2::::random((size, size), dist); + Self::Masked(mask) + } +} + +impl From> for Mask +where + T: Float, +{ + fn from(bias: Array2) -> Self { + Self::Masked(bias) + } +} + +impl From>> for Mask +where + T: Float, +{ + fn from(bias: Option>) -> Self { + match bias { + Some(bias) => Self::Masked(bias), + None => Self::Unmasked, + } + } +} + +impl From> for Option> +where + T: Float, +{ + fn from(bias: Mask) -> Self { + match bias { + Mask::Masked(bias) => Some(bias), + Mask::Unmasked => None, + } + } +} + +impl ops::Add> for Mask +where + Array: ops::Add, Output = Array>, +{ + type Output = Array; + + fn add(self, rhs: Array) -> Self::Output { + use Mask::*; + if let Masked(bias) = self { + return rhs.clone() + bias; + } + rhs.clone() + } +} + +impl ops::Add<&Array> for Mask +where + Array: ops::Add, Output = Array>, +{ + type Output = Array; + + fn add(self, rhs: &Array) -> Self::Output { + use Mask::*; + if let Masked(bias) = self { + return rhs.clone() + bias; + } + rhs.clone() + } +} + +impl ops::Add> for Array +where + Array: ops::Add, Output = Array>, +{ + type Output = Array; + + fn add(self, bias: Mask) -> Self::Output { + use Mask::*; + if let Masked(bias) = bias { + return self.clone() + bias; + } + self.clone() + } +} + +impl ops::Add<&Mask> for Array +where + Array: ops::Add, Output = Array>, +{ + type Output = Array; + + fn add(self, bias: &Mask) -> Self::Output { + use Mask::*; + if let Masked(m) = bias.clone() { + return self.clone() + m; + } + self.clone() + } +} diff --git a/ml/neural/src/bias/mod.rs b/ml/neural/src/bias/mod.rs new file mode 100644 index 00000000..ab2c98d8 --- /dev/null +++ b/ml/neural/src/bias/mod.rs @@ -0,0 +1,23 @@ +/* + Appellation: bias + Contrib: FL03 +*/ +//! # Bias +pub use self::{biases::*, mask::*, utils::*}; + +pub(crate) mod biases; +pub(crate) mod mask; + +use num::Float; + +pub trait Biased { + + fn bias(&self) -> &Bias; + fn bias_mut(&mut self) -> &mut Bias; +} + +pub(crate) mod utils { +} + +#[cfg(test)] +mod tests {} diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 7f78600a..6c5ddcaf 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -4,35 +4,14 @@ */ use super::{Features, LayerType}; use crate::bias::Bias; -use crate::neurons::activate::Activator; use crate::prop::Forward; -use ndarray::prelude::{Array1, Array2}; +use ndarray::prelude::Array2; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; +use serde::{Deserialize, Serialize}; -pub trait L { - // - fn process(&self, args: &Array2, rho: impl Activator) -> Array2 - where - T: 'static, - { - let z = args.dot(self.weights()) + self.bias(); - z.mapv(|x| rho.activate(x)) - } - - fn bias(&self) -> &Array1; - - fn weights(&self) -> &Array2; -} - -pub trait Linear { - fn linear(&self, data: &Array2) -> Array2 - where - T: 'static; -} - -#[derive(Clone, Debug, Default, PartialEq)] +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] pub struct Layer { bias: Bias, features: Features, @@ -41,17 +20,15 @@ pub struct Layer { } impl Layer { - pub fn new(inputs: usize, outputs: usize, bias: bool, layer: LayerType) -> Self where T: SampleUniform { + pub fn new(inputs: usize, outputs: usize, layer: LayerType) -> Self + where + T: SampleUniform, + { let features = Features::new(inputs, outputs); - let bias = if bias { - Bias::biased(outputs) - } else { - Bias::default() - }; let weights = Array2::ones((features.inputs(), features.outputs())); Self { - bias, + bias: Bias::default(), features, layer, weights, @@ -79,10 +56,27 @@ impl Layer { } } +impl Layer where T: Float + SampleUniform { + pub fn biased(inputs: usize, outputs: usize, layer: LayerType) -> Self + where + T: SampleUniform, + { + let features = Features::new(inputs, outputs); + let weights = Array2::ones((features.inputs(), features.outputs())); + + Self { + bias: Bias::biased(outputs), + features, + layer, + weights, + } + } +} + impl Forward> for Layer { type Output = Array2; fn forward(&self, data: &Array2) -> Self::Output { data.dot(&self.weights().t()) + self.bias() } -} \ No newline at end of file +} diff --git a/ml/neural/src/layers/linear/layer.rs b/ml/neural/src/layers/linear/layer.rs new file mode 100644 index 00000000..a53fedc9 --- /dev/null +++ b/ml/neural/src/layers/linear/layer.rs @@ -0,0 +1,59 @@ +/* + Appellation: layer + Contrib: FL03 +*/ +use crate::layers::Features; +use crate::prelude::{Bias, Forward}; +use ndarray::prelude::Array2; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use num::Float; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +pub struct LinearLayer { + bias: Bias, + pub params: Features, + weights: Array2, +} + +impl LinearLayer +where + T: Float, +{ + + pub fn bias(&self) -> &Bias { + &self.bias + } + + pub fn linear(&self, data: &Array2) -> Array2 + where + T: 'static, + { + data.dot(&self.weights.t()) + &self.bias + } + + pub fn weights(&self) -> &Array2 { + &self.weights + } +} + +impl LinearLayer where T: Float + SampleUniform { + pub fn new(inputs: usize, outputs: usize) -> Self { + let params = Features::new(inputs, outputs); + let weights = Array2::ones((inputs, outputs)); + let bias = Bias::biased(outputs); + Self { + bias, + params, + weights, + } + } +} + +impl Forward> for LinearLayer { + type Output = Array2; + + fn forward(&self, data: &Array2) -> Self::Output { + data.dot(&self.weights().t()) + self.bias() + } +} \ No newline at end of file diff --git a/ml/neural/src/layers/linear/mod.rs b/ml/neural/src/layers/linear/mod.rs new file mode 100644 index 00000000..3dcb48d7 --- /dev/null +++ b/ml/neural/src/layers/linear/mod.rs @@ -0,0 +1,55 @@ +/* + Appellation: linear + Contrib: FL03 +*/ +//! # Linear Layer +pub use self::{layer::*, utils::*}; + +pub(crate) mod layer; + +use crate::bias::Biased; +use crate::weights::Weighted; +use ndarray::prelude::Array2; +use num::Float; + +pub trait LinearTransformation where T: Float { + fn linear(&self, data: &Array2) -> Array2; +} + +impl LinearTransformation for S +where + S: Biased + Weighted, + T: Float + 'static, +{ + fn linear(&self, data: &Array2) -> Array2 { + data.dot(&self.weights().t()) + self.bias() + } +} + +pub(crate) mod utils { + use ndarray::prelude::{Array1, Array2}; + use num::Float; + + pub fn linear_transformation( + data: &Array2, + bias: &Array1, + weights: &Array2, + ) -> Array2 { + data.dot(&weights.t()) + bias + } +} + + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_linear_layer() { + let (inputs, outputs) = (2, 2); + let data = Array2::::ones((inputs, outputs)); + let layer = LinearLayer::new(inputs, outputs); + let linear = layer.linear(&data); + + } +} \ No newline at end of file diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index f84b984f..c73c9e0a 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -10,6 +10,33 @@ pub(crate) mod kinds; pub(crate) mod layer; pub(crate) mod sublayer; +pub mod linear; + +use crate::neurons::activate::Activate; +use ndarray::prelude::{Array1, Array2}; +use num::Float; + +pub trait L { + // + fn process(&self, args: &Array2, rho: impl Activate) -> Array2 + where + T: 'static, + { + let z = args.dot(self.weights()) + self.bias(); + z.mapv(|x| rho.activate(x)) + } + + fn bias(&self) -> &Array1; + + fn weights(&self) -> &Array2; +} + +pub trait Linear { + fn linear(&self, data: &Array2) -> Array2 + where + T: 'static; +} + pub(crate) mod utils {} #[cfg(test)] diff --git a/ml/neural/src/layers/sublayer.rs b/ml/neural/src/layers/sublayer.rs index fc6a82e2..bd88240c 100644 --- a/ml/neural/src/layers/sublayer.rs +++ b/ml/neural/src/layers/sublayer.rs @@ -6,10 +6,12 @@ use super::Layer; use crate::ops::LayerNorm; use crate::prop::Forward; -use ndarray::ScalarOperand; use ndarray::prelude::Array2; +use ndarray::ScalarOperand; use num::{Float, FromPrimitive}; +use serde::{Deserialize, Serialize}; +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Sublayer { layer: Layer, norm: LayerNorm, @@ -20,9 +22,12 @@ impl Sublayer { Self { layer, norm } } - pub fn forward(&self, data: &Array2) -> Array2 where T: FromPrimitive + ScalarOperand { + pub fn forward(&self, data: &Array2) -> Array2 + where + T: FromPrimitive + ScalarOperand, + { let norm = self.norm.forward(data); let layer = data + self.layer.forward(&norm); layer } -} \ No newline at end of file +} diff --git a/ml/neural/src/lib.rs b/ml/neural/src/lib.rs index 9ded1e8d..705b54ce 100644 --- a/ml/neural/src/lib.rs +++ b/ml/neural/src/lib.rs @@ -21,12 +21,16 @@ pub mod neurons; pub mod nn; pub mod ops; pub mod prop; +pub mod weights; // pub(crate) use concision_core as core; pub mod prelude { + pub use crate::arch::*; + pub use crate::bias::*; pub use crate::layers::*; - pub use crate::neurons::*; + pub use crate::neurons::activate::*; + pub use crate::neurons::{Neuron, Node, Weight}; pub use crate::nn::*; pub use crate::ops::*; pub use crate::prop::*; diff --git a/ml/neural/src/neurons/activate/activator.rs b/ml/neural/src/neurons/activate/activator.rs new file mode 100644 index 00000000..867f1b37 --- /dev/null +++ b/ml/neural/src/neurons/activate/activator.rs @@ -0,0 +1,42 @@ +/* + Appellation: activator + Contrib: FL03 +*/ +use super::{Activate, ActivationMethod, ActivationFn}; +use std::marker::PhantomData; + +pub trait ActivationParams {} + +pub trait LinearActivation: ActivationMethod { + fn rho() -> ActivationFn { + |x| x + } + + fn linear(&self, x: T) -> T { + x + } +} + +pub struct Activator where A: Activate { + method: A, + _args: PhantomData +} + +impl Activator where A: Activate { + pub fn new(method: A) -> Self { + Activator { + method, + _args: PhantomData + } + } +} + +impl Activate for Activator where A: Activate { + fn activate(&self, x: T) -> T { + self.method.activate(x) + } +} + + + + diff --git a/ml/neural/src/neurons/activate/binary.rs b/ml/neural/src/neurons/activate/binary.rs index a852091e..453ff1d0 100644 --- a/ml/neural/src/neurons/activate/binary.rs +++ b/ml/neural/src/neurons/activate/binary.rs @@ -2,14 +2,18 @@ Appellation: binary Contrib: FL03 */ -use crate::neurons::activate::Activator; -use ndarray::Array; +use crate::neurons::activate::Activate; +use ndarray::prelude::Array; +use ndarray::Dimension; use num::{One, Zero}; pub struct Heavyside; impl Heavyside { - pub fn heavyside(x: T) -> T { + pub fn heavyside(x: T) -> T + where + T: One + PartialOrd + Zero, + { if x > T::zero() { T::one() } else { @@ -18,12 +22,12 @@ impl Heavyside { } } -impl Activator> for Heavyside +impl Activate> for Heavyside where - D: ndarray::Dimension, - T: Clone + PartialOrd + One + Zero, + D: Dimension, + T: Clone + One + PartialOrd + Zero, { - fn rho(x: Array) -> Array { + fn activate(&self, x: Array) -> Array { x.mapv(|x| Self::heavyside(x)) } } diff --git a/ml/neural/src/neurons/activate/mod.rs b/ml/neural/src/neurons/activate/mod.rs index eea0a127..3ec13e5c 100644 --- a/ml/neural/src/neurons/activate/mod.rs +++ b/ml/neural/src/neurons/activate/mod.rs @@ -5,41 +5,34 @@ //! # activate //! //! This module contains the activation functions for the neurons. -pub use self::{binary::*, nonlinear::*, utils::*}; +pub use self::{activator::*, binary::*, nonlinear::*, utils::*}; +pub(crate) mod activator; pub(crate) mod binary; pub(crate) mod nonlinear; pub type ActivationFn = fn(T) -> T; -pub struct Linear; +pub struct LinearActivation; -impl Linear { - pub fn act() -> ActivationFn { +impl LinearActivation { + pub fn method() -> ActivationFn { |x| x } } -impl Activator for Linear { - fn rho(x: T) -> T { - x +impl Activate for LinearActivation { + fn activate(&self, x: T) -> T { + Self::method()(x) } } -pub trait Activate { - fn activate(&self, x: T) -> T; -} - -pub trait ActivateMethod { - fn method() -> fn(T) -> T; +pub trait ActivationMethod { + fn method_name(&self) -> &str; } -pub trait Activator { - fn activate(&self, x: T) -> T { - Self::rho(x) - } - - fn rho(x: T) -> T; +pub trait Activate { + fn activate(&self, x: T) -> T; } impl Activate for F diff --git a/ml/neural/src/neurons/activate/nonlinear.rs b/ml/neural/src/neurons/activate/nonlinear.rs index 89da75eb..29edac7f 100644 --- a/ml/neural/src/neurons/activate/nonlinear.rs +++ b/ml/neural/src/neurons/activate/nonlinear.rs @@ -2,12 +2,15 @@ Appellation: nonlinear Contrib: FL03 */ -use super::Activator; -use ndarray::prelude::{Array, Array1}; +use super::Activate; +use ndarray::{Dimension, RemoveAxis, ScalarOperand}; +use ndarray::prelude::{Array, Array1, Axis}; +use num::{Float, Zero}; +use serde::{Deserialize, Serialize}; pub fn softmax(args: Array1) -> Array1 where - T: num::Float, + T: Float, { let denom = args.mapv(|x| x.exp()).sum(); args.mapv(|x| x.exp() / denom) @@ -16,7 +19,7 @@ where pub struct ReLU; impl ReLU { - pub fn compute(x: T) -> T { + pub fn compute(x: T) -> T { if x > T::zero() { x } else { @@ -25,12 +28,12 @@ impl ReLU { } } -impl Activator> for ReLU +impl Activate> for ReLU where - D: ndarray::Dimension, - T: num::Float, + D: Dimension, + T: Float, { - fn rho(x: Array) -> Array { + fn activate(&self, x: Array) -> Array { x.mapv(|x| Self::compute(x)) } } @@ -38,36 +41,63 @@ where pub struct Sigmoid; impl Sigmoid { - pub fn compute(x: T) -> T { + pub fn compute(x: T) -> T { T::one() / (T::one() + (-x).exp()) } } -impl Activator> for Sigmoid +impl Activate> for Sigmoid where - D: ndarray::Dimension, - T: num::Float, + D: Dimension, + T: Float, { - fn rho(x: Array) -> Array { + fn activate(&self, x: Array) -> Array { x.mapv(|x| Self::compute(x)) } } -pub struct Softmax; + +pub fn softmax_axis(args: Array, axis: Option) -> Array +where + T: Float + ScalarOperand, + D: Dimension + RemoveAxis, +{ + let exp = args.mapv(|x| x.exp()); + if let Some(axis) = axis { + let denom = exp.sum_axis(Axis(axis)); + exp / denom + } else { + let denom = exp.sum(); + exp / denom + } +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct Softmax { + axis: Option, +} impl Softmax { - pub fn new() -> Self { - Self + pub fn new(axis: Option) -> Self { + Self { + axis + } } } -impl Activator> for Softmax +impl Activate> for Softmax where - D: ndarray::Dimension, - T: num::Float, + D: Dimension + RemoveAxis, + T: Float + ScalarOperand, { - fn rho(x: Array) -> Array { - let denom = x.mapv(|x| x.exp()).sum(); - x.mapv(|x| x.exp() / denom) + fn activate(&self, x: Array) -> Array { + let exp = x.mapv(|x| x.exp()); + if let Some(axis) = self.axis { + let denom = exp.sum_axis(Axis(axis)); + exp / denom + } else { + let denom = exp.sum(); + exp / denom + } } } @@ -81,7 +111,8 @@ mod tests { fn test_softmax() { let exp = array![0.09003057, 0.24472847, 0.66524096]; let args = array![1.0, 2.0, 3.0]; - let res = Softmax::rho(args).mapv(|i| i.round_to(8)); + + let res = Activate::activate(&Softmax::new(None), args).mapv(|i| i.round_to(8)); assert_eq!(res, exp); } } diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index fe89bbee..686cb0d2 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -16,14 +16,14 @@ pub(crate) mod utils {} #[cfg(test)] mod tests { - use super::activate::{Activator, Softmax}; + use super::activate::{Activate, Softmax, softmax}; use super::*; use ndarray::{array, Array1}; fn _artificial( args: &Array1, bias: Option>, - rho: impl Activator>, + rho: impl Activate>, weights: &Array1, ) -> Array1 { rho.activate( @@ -37,17 +37,17 @@ mod tests { let a_data = array![10.0, 10.0, 6.0, 1.0, 8.0]; let a_weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; - let a = Neuron::new(Softmax::rho, bias.clone(), a_weights.clone()); + let a = Neuron::new(softmax, bias.clone(), a_weights.clone()); - let exp = _artificial(&a_data, Some(bias.clone()), Softmax, &a_weights); + let exp = _artificial(&a_data, Some(bias.clone()), Softmax::default(), &a_weights); assert_eq!(a.compute(&a_data), exp); let b_data = array![0.0, 9.0, 3.0, 5.0, 3.0]; let b_weights = array![2.0, 8.0, 8.0, 0.0, 3.0]; - let b = Neuron::new(Softmax::rho, bias.clone(), b_weights.clone()); + let b = Neuron::new(softmax, bias.clone(), b_weights.clone()); - let exp = _artificial(&b_data, Some(bias), Softmax, &b_weights); + let exp = _artificial(&b_data, Some(bias), Softmax::default(), &b_weights); assert_eq!(b.compute(&b_data), exp); // assert_eq!(a.dot() + b.dot(), 252.0); @@ -59,18 +59,18 @@ mod tests { let a_data = array![10.0, 10.0, 6.0, 1.0, 8.0]; let a_weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; - let a = Neuron::new(Softmax::rho, bias.clone(), a_weights.clone()); + let a = Neuron::new(softmax, bias.clone(), a_weights.clone()); let node_a = Node::new(a.clone()).with_data(a_data.clone()); - let exp = _artificial(&a_data, Some(bias.clone()), Softmax, &a_weights); + let exp = _artificial(&a_data, Some(bias.clone()), Softmax::default(), &a_weights); assert_eq!(node_a.process(), exp); let b_data = array![0.0, 9.0, 3.0, 5.0, 3.0]; let b_weights = array![2.0, 8.0, 8.0, 0.0, 3.0]; - let b = Neuron::new(Softmax::rho, bias.clone(), b_weights.clone()); + let b = Neuron::new(softmax, bias.clone(), b_weights.clone()); let node_b = Node::new(b.clone()).with_data(b_data.clone()); - let exp = _artificial(&b_data, Some(bias), Softmax, &b_weights); + let exp = _artificial(&b_data, Some(bias), Softmax::default(), &b_weights); assert_eq!(node_b.process(), exp); assert_eq!(node_a.dot() + node_b.dot(), 252.0); diff --git a/ml/neural/src/nn/mod.rs b/ml/neural/src/nn/mod.rs index ac22f117..5c0285bf 100644 --- a/ml/neural/src/nn/mod.rs +++ b/ml/neural/src/nn/mod.rs @@ -13,7 +13,7 @@ use crate::layers::Layer; use crate::Trainable; use num::Float; -pub trait NeuralNet: Trainable { +pub trait NeuralNet: Trainable { fn depth(&self) -> usize { self.layers().len() } diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index cea270a6..0d53686f 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -2,23 +2,26 @@ Appellation: specs Contrib: FL03 */ -use ndarray::prelude::Array1; +use ndarray::{Dimension, IntoDimension}; +use ndarray::prelude::{Array, Array1, Array2}; use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::Uniform; +use ndarray_rand::rand_distr as dist; use ndarray_rand::RandomExt; use num::Float; -pub trait Biased { - fn init_uniform(features: usize) -> Array1 { - let k = (T::from(features).unwrap()).sqrt(); - let uniform = Uniform::new(-k, k); - Array1::random(features, uniform) - } +pub trait InitUniform where T: Float + SampleUniform { + type Dim: Dimension; - fn bias(&self) -> &Array1; - fn bias_mut(&mut self) -> &mut Array1; + fn uniform(axis: usize, dim: impl IntoDimension) -> Array { + let dim = dim.into_dimension(); + let k = (T::from(dim[axis]).unwrap()).sqrt(); + let uniform = dist::Uniform::new(-k, k); + Array::random(dim, uniform) + } } -pub trait Trainable { - fn train(&mut self, args: &[f64]) -> f64; + + +pub trait Trainable { + fn train(&mut self, args: &Array2) -> Array2; } diff --git a/ml/neural/src/weights/mod.rs b/ml/neural/src/weights/mod.rs new file mode 100644 index 00000000..7551ccc0 --- /dev/null +++ b/ml/neural/src/weights/mod.rs @@ -0,0 +1,35 @@ +/* + Appellation: weights + Contrib: FL03 +*/ +//! # Weights +pub use self::{weight::*, utils::*}; + +pub(crate) mod weight; + + +use ndarray::prelude::Array2; +use num::Float; + +pub trait Weighted where T: Float { + fn weights(&self) -> &Array2; + + fn weights_mut(&mut self) -> &mut Array2; +} + +impl Weighted for S where S: AsMut> + AsRef>, T: Float { + fn weights(&self) -> &Array2 { + self.as_ref() + } + + fn weights_mut(&mut self) -> &mut Array2 { + self.as_mut() + } +} + + +pub(crate) mod utils { +} + +#[cfg(test)] +mod tests {} diff --git a/ml/neural/src/weights/weight.rs b/ml/neural/src/weights/weight.rs new file mode 100644 index 00000000..ea94776f --- /dev/null +++ b/ml/neural/src/weights/weight.rs @@ -0,0 +1,32 @@ +/* + Appellation: weight + Contrib: FL03 +*/ +use ndarray::prelude::Array2; +use ndarray_rand::RandomExt; +use ndarray_rand::rand_distr::Uniform; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use num::Float; +use serde::{Deserialize, Serialize}; + + + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct Weight { + weights: Array2 +} + +impl Weight where T: Default { + pub fn new(m: usize, n: usize) -> Self { + let weights = Array2::default((m, n)); + Self { weights } + } +} + +impl Weight where T: Float + SampleUniform { + pub fn uniform(m: usize, n: usize) -> Array2 { + let dk = (T::from(m).unwrap()).sqrt(); + let dist = Uniform::new(-dk, dk); + Array2::random((m, n), dist) + } +} \ No newline at end of file diff --git a/ml/nlp/src/encode/mod.rs b/ml/nlp/src/encode/mod.rs index f23c8b91..dc114db2 100644 --- a/ml/nlp/src/encode/mod.rs +++ b/ml/nlp/src/encode/mod.rs @@ -6,8 +6,8 @@ pub use self::utils::*; pub mod positional; -use ndarray::Dimension; use ndarray::prelude::{Array, Array2}; +use ndarray::Dimension; pub trait Encode { type Output; diff --git a/ml/nlp/src/encode/positional.rs b/ml/nlp/src/encode/positional.rs index 730dbfff..4d573bf0 100644 --- a/ml/nlp/src/encode/positional.rs +++ b/ml/nlp/src/encode/positional.rs @@ -6,7 +6,7 @@ use ndarray::Array2; pub fn get_position_encoding(seq_len: usize, d: usize, n: f64) -> Array2 { - let denom = | i: usize | f64::powf(n, 2.0 * (i as f64) / d as f64); + let denom = |i: usize| f64::powf(n, 2.0 * (i as f64) / d as f64); let mut p = Array2::zeros((seq_len, d)); for k in 0..seq_len { for i in 0..d / 2 { @@ -35,7 +35,6 @@ impl PositionalEncoder { pub fn encode(&self, data: &Array2) -> Array2 { let x = data * (self.model as f64).sqrt(); x + self.positional() - } pub fn positional(&self) -> Array2 { @@ -43,7 +42,6 @@ impl PositionalEncoder { } } - #[cfg(test)] mod tests { use super::*; diff --git a/ml/transformers/Cargo.toml b/ml/transformers/Cargo.toml index 7bce8b41..82793ee8 100644 --- a/ml/transformers/Cargo.toml +++ b/ml/transformers/Cargo.toml @@ -31,6 +31,9 @@ concision-neural = { path = "../neural" } anyhow.workspace = true lazy_static.workspace = true ndarray.workspace = true +# ndarray-linalg.workspace = true +ndarray-rand.workspace = true +ndarray-stats.workspace = true num.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index 0365093b..ca4cea49 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -4,7 +4,7 @@ */ use super::params::{HeadShape, QKV}; use super::{Head, Weight}; -use crate::neural::neurons::activate::{Activator, Softmax}; +use crate::neural::neurons::activate::{Activate, Softmax}; use ndarray::prelude::Array2; use ndarray::ScalarOperand; use num::Float; @@ -19,7 +19,7 @@ pub struct AttentionHead { weights: Weight, } -impl AttentionHead { +impl AttentionHead { pub fn new(dim: HeadShape) -> Self { Self { dim, @@ -28,23 +28,11 @@ impl AttentionHead { } } - pub fn attention(&mut self, data: &Array2) -> Array2 { - // multiply the data by the wieghted query, key, and value matrices, respectively - let weighted = self.weights.clone() * data; - let (q, k, v) = weighted.qkv(); - - // compute the attention score - let inner = (q.dot(&k.t()) + self.mask.clone()) * self.scale(); - Softmax::rho(inner).dot(&v) - } - pub fn dim(&self) -> HeadShape { self.dim } - pub fn mask(&self) -> &Array2 { - &self.mask - } + pub fn mask_mut(&mut self) -> &mut Array2 { &mut self.mask @@ -54,6 +42,10 @@ impl AttentionHead { T::one() / T::from(self.dim.query_size()).unwrap().sqrt() } + pub fn weights(&self) -> &Weight { + &self.weights + } + pub fn set_mask(&mut self, mask: Array2) { self.mask = mask; } @@ -64,15 +56,31 @@ impl AttentionHead { } } -impl Head for AttentionHead { - fn query(&self) -> &Array2 { - &self.weights.query +impl AttentionHead { + pub fn attention(&mut self, data: &Array2) -> Array2 { + // multiply the data by the wieghted query, key, and value matrices, respectively + let weighted = data * self.weights(); + let (q, k, v) = weighted.qkv(); + + // compute the attention score + let inner = (q.dot(&k.t()) + self.mask.clone()) * self.scale(); + Activate::activate(&Softmax::default(), inner).dot(&v) } +} +impl Head for AttentionHead { fn key(&self) -> &Array2 { &self.weights.key } + fn mask(&self) -> &Array2 { + &self.mask + } + + fn query(&self) -> &Array2 { + &self.weights.query + } + fn value(&self) -> &Array2 { &self.weights.value } diff --git a/ml/transformers/src/attention/mod.rs b/ml/transformers/src/attention/mod.rs index bfd7b54e..0e6289a1 100644 --- a/ml/transformers/src/attention/mod.rs +++ b/ml/transformers/src/attention/mod.rs @@ -19,9 +19,10 @@ pub mod params; use crate::core::prelude::BoxResult; use crate::prelude::BaseDim; -use ndarray::prelude::{Array, Ix2}; -use ndarray::Dimension; +use ndarray::prelude::{Array, Array2, Ix2}; +use ndarray::{Dimension, ScalarOperand}; use num::Float; +use std::ops::Mul; /// (batch, sample, seq, model) pub type InputArray = Array; @@ -29,16 +30,37 @@ pub type InputArray = Array; pub type AttentionArray = Array; pub trait Attention { - type Dim: Dimension; - type Score; + fn attention(&self, data: &Array2) -> BoxResult> + where + T: ScalarOperand, + { + // let (seq, model) = data.dim(); + + let q = self.query().dot(data); + let k = self.key().dot(data); + let v = self.value().dot(data); + + let score = attention(&q, &k, &v, Some(self.mask().clone())); + Ok(score) + } + + fn key(&self) -> &Array2; + + fn mask(&self) -> &Array2; - fn attention(&mut self, data: &Array) -> BoxResult<&Array>; + fn query(&self) -> &Array2; + + fn value(&self) -> &Array2; } pub trait Head { - fn query(&self) -> &Array; - fn key(&self) -> &Array; - fn value(&self) -> &Array; + fn key(&self) -> &Array2; + + fn mask(&self) -> &Array2; + + fn query(&self) -> &Array2; + + fn value(&self) -> &Array2; } pub trait Spaces { @@ -49,13 +71,25 @@ pub trait Spaces { fn value(&self) -> &Array; } +pub trait Weights: Mul, Output = Self> { + fn key(&self) -> &Array2; + + fn query(&self) -> &Array2; + + fn value(&self) -> &Array2; + + fn qkv(&self) -> (&Array2, &Array2, &Array2) { + (self.query(), self.key(), self.value()) + } +} + pub(crate) mod utils { - use crate::neural::prelude::activate::{Activator, Softmax}; + use crate::neural::prelude::{Activate, Softmax}; use ndarray::prelude::Array2; use ndarray::ScalarOperand; use num::Float; - pub fn compute_attention( + pub fn attention( query: &Array2, key: &Array2, value: &Array2, @@ -64,7 +98,10 @@ pub(crate) mod utils { let (seq, dk) = query.dim(); let mask = mask.unwrap_or_else(|| Array2::::zeros((seq, seq))); let scale = T::one() / (T::from(dk).unwrap()).sqrt(); - Softmax::rho((query.dot(&key.t()) + mask) * scale).dot(value) + let softmax = Softmax::new(Some(1)); + softmax + .activate((query.dot(&key.t()) + mask) * scale) + .dot(value) } } diff --git a/ml/transformers/src/attention/multi/attention.rs b/ml/transformers/src/attention/multi/attention.rs index 38278009..e4e08a6a 100644 --- a/ml/transformers/src/attention/multi/attention.rs +++ b/ml/transformers/src/attention/multi/attention.rs @@ -2,55 +2,71 @@ Appellation: attention Contrib: FL03 */ -use super::utils::multihead; -use super::MultiHeadParams; +use super::{multihead, MultiHeadParams}; use crate::attention::Weight; -use crate::neural::prelude::Forward; +use crate::neural::prelude::{ Mask}; use crate::ops::Split; +use ndarray::{ScalarOperand, ShapeError}; use ndarray::prelude::Array2; +use num::Float; +use serde::{Deserialize, Serialize}; -pub struct MultiHeadAttention { - mask: Array2, +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct MultiHeadAttention { params: MultiHeadParams, - weights: Weight, + weights: Weight, } -impl MultiHeadAttention { +impl MultiHeadAttention { pub fn new(heads: usize, model: usize) -> Self { let params = MultiHeadParams::new(heads, model); - let mask = Array2::::zeros((params.model, params.model)); - let weights = Weight::new((params.model, params.model)); + let weights = Weight::new((model, model)); Self { - mask, params, weights, } } - pub fn attention(&self, data: &Array2) -> Array2 { - let weighted = self.weights() * data; - let (q, k, v) = weighted.split(self.params().heads()).unwrap(); - let score = multihead(&q, &k, &v, Some(self.mask().clone())).unwrap(); - score - } - - pub fn mask(&self) -> &Array2 { - &self.mask - } - pub fn params(&self) -> MultiHeadParams { self.params } - pub fn weights(&self) -> &Weight { + pub fn weights(&self) -> &Weight { &self.weights } } -impl Forward> for MultiHeadAttention { - type Output = Array2; - - fn forward(&self, data: &Array2) -> Self::Output { - self.attention(data) +impl MultiHeadAttention { + pub fn attention(&self, data: &Array2, mask: &Mask) -> Result, ShapeError> { + let weighted = data * self.weights(); + let (q, k, v) = weighted.split(self.params().heads())?; + let score = multihead(&q, &k, &v, mask)?; + Ok(score) } } + +// impl Attention for MultiHeadAttention { +// fn key(&self) -> &Array2 { +// self.weights.key() +// } + +// fn mask(&self) -> &Array2 { +// &self.mask +// } + +// fn query(&self) -> &Array2 { +// &self.weights.query() +// } + +// fn value(&self) -> &Array2 { +// &self.weights.value() +// } +// } + +// impl Forward> for MultiHeadAttention { +// type Output = Result, ShapeError>; + +// fn forward(&self, data: &Array2) -> Self::Output { +// self.attention(data) +// } +// } diff --git a/ml/transformers/src/attention/multi/mod.rs b/ml/transformers/src/attention/multi/mod.rs index bf02cfdf..f828fba4 100644 --- a/ml/transformers/src/attention/multi/mod.rs +++ b/ml/transformers/src/attention/multi/mod.rs @@ -9,6 +9,7 @@ pub(crate) mod params; use crate::attention::Weight; use crate::core::prelude::BoxResult; +use crate::neural::prelude::Mask; use crate::ops::Split; use ndarray::prelude::Array2; use ndarray::ScalarOperand; @@ -18,25 +19,24 @@ pub trait MultiHead where T: Float + ScalarOperand, { - fn attention(&mut self, data: &Array2) -> BoxResult> { - let weighted = self.weights() * data; + fn attention(&mut self, data: &Array2, mask: &Mask) -> BoxResult> { + let weighted = data * self.weights(); let (q, k, v) = weighted.split(self.params().heads())?; - let score = utils::multihead(&q, &k, &v, Some(self.mask().clone()))?; + let score = utils::multihead(&q, &k, &v, mask)?; Ok(score) } fn params(&self) -> MultiHeadParams; - fn mask(&self) -> &Array2; - fn weights(&self) -> &Weight; } pub(crate) mod utils { - use crate::attention::compute_attention; + use crate::attention::attention; + use crate::neural::prelude::Mask; use crate::ops::Merge; - use ndarray::prelude::{Array2, Array3, Array4}; - use ndarray::{s, ScalarOperand, ShapeError}; + use ndarray::prelude::{Array2, Array3, Array4, Axis, azip, s}; + use ndarray::{ScalarOperand, ShapeError}; use num::Float; pub fn batched_multihead( @@ -53,7 +53,7 @@ pub(crate) mod utils { let q = query.slice(s![i, h, .., ..]).to_owned(); let k = key.slice(s![i, h, .., ..]).to_owned(); let v = value.slice(s![i, h, .., ..]).to_owned(); - let head = compute_attention(&q, &k, &v, Some(mask.clone())); + let head = attention(&q, &k, &v, Some(mask.clone())); score.slice_mut(s![i, h, .., ..]).assign(&head); } } @@ -64,25 +64,46 @@ pub(crate) mod utils { query: &Array3, key: &Array3, value: &Array3, - mask: Option>, + mask: &Mask, ) -> Result, ShapeError> where T: Float + ScalarOperand, { - let (heads, seq, _) = query.dim(); - let mask = mask.unwrap_or_else(|| Array2::::zeros((seq, seq))); + let (heads, seq, dk) = query.dim(); + println!("heads: {}, seq: {}, query: {}", heads, seq, dk); let mut score = Array3::::zeros(query.dim()); for h in 0..heads { let pos = s![h, .., ..]; let q = query.slice(pos).to_owned(); let k = key.slice(pos).to_owned(); let v = value.slice(pos).to_owned(); - let head = compute_attention(&q, &k, &v, Some(mask.clone())); + let head = attention(&q, &k, &v, mask.clone().into()); + println!("head dim: {:?}", &head.shape()); score.slice_mut(s![h, .., ..]).assign(&head); } - score.merge() + + let score = score.merge()?; + println!("score: {:?}", &score.shape()); + Ok(score) } } #[cfg(test)] -mod tests {} +mod tests { + use super::*; + use crate::neural::prelude::Mask; + + + #[test] + fn test_multihead() { + let (heads, seq, model) = (8, 10, 512); + let data = Array2::::zeros((seq, model)); + let _weights = Weight::::new((model, model)); + let _params = MultiHeadParams::new(heads, model); + + let mask: Mask = Array2::::zeros((seq, seq)).into(); + let attention = MultiHeadAttention::new(heads, model); + let score = attention.attention(&data, &mask).expect("Failed to compute attention"); + assert_eq!(score, Array2::::zeros((10, model))); + } +} diff --git a/ml/transformers/src/attention/params/dim.rs b/ml/transformers/src/attention/params/dim.rs index 790b8074..ff665dda 100644 --- a/ml/transformers/src/attention/params/dim.rs +++ b/ml/transformers/src/attention/params/dim.rs @@ -10,41 +10,28 @@ //! - `batch`: The batch size //! - `heads`: The number of attention heads //! - `model`: The dimension of the model (embedding size) -use crate::{HEADS, MODEL_SIZE}; +use crate::{HEADS, MODEL_SIZE, QUERY_SIZE}; use ndarray::IntoDimension; +use ndarray::prelude::{Ix3, Ix4}; use serde::{Deserialize, Serialize}; -pub trait StructuredDim: IntoDimension {} - -pub trait BaseDimension: IntoDimension { - fn model_size(&self) -> usize; - fn seq_len(&self) -> usize; -} - pub trait Batched { - fn batch_size(&self) -> usize; + fn batch(&self) -> usize; } -pub trait ModelSize { - fn model_size(&self) -> usize; +impl Batched for Ix3 { + fn batch(&self) -> usize { + self[0] + } } -pub trait MultiHeadDimension: BaseDimension { - fn heads(&self) -> usize; +impl Batched for Ix4 { + fn batch(&self) -> usize { + self[0] + } } -impl BaseDimension for D -where - D: Clone + IntoDimension, -{ - fn model_size(&self) -> usize { - self.clone().into_dimension()[1] - } - fn seq_len(&self) -> usize { - self.clone().into_dimension()[0] - } -} pub enum AttentionDims { Base(BaseShape), // a 3d matrix (batch, seq, model) @@ -52,8 +39,8 @@ pub enum AttentionDims { MultiHead(MultiShape), // a 4d matrix (batch, heads, seq, query) } -pub enum AttentionShape { - IO { +pub enum Shapes { + Data { batch: usize, seq: usize, model: usize, @@ -68,8 +55,8 @@ pub enum AttentionShape { MultiHead { batch: usize, heads: usize, + model: usize, seq: usize, - query: usize, }, } @@ -91,6 +78,10 @@ impl BaseShape { Self::new(batch, seq, MODEL_SIZE) } + pub fn batch(&self) -> usize { + self.batch + } + pub fn model_size(&self) -> usize { self.model } @@ -100,11 +91,6 @@ impl BaseShape { } } -impl Batched for BaseShape { - fn batch_size(&self) -> usize { - self.batch - } -} impl IntoDimension for BaseShape { type Dim = ndarray::Ix3; @@ -129,7 +115,7 @@ impl MultiShape { } pub fn std(seq: usize) -> Self { - Self::new(HEADS, seq, MODEL_SIZE) + Self::new(HEADS, seq, *QUERY_SIZE) } pub fn heads(&self) -> usize { @@ -137,6 +123,10 @@ impl MultiShape { } pub fn model_size(&self) -> usize { + self.heads() * self.query_size() + } + + pub fn query_size(&self) -> usize { self.query } @@ -145,13 +135,13 @@ impl MultiShape { } } -impl From for AttentionShape { +impl From for Shapes { fn from(shape: MultiShape) -> Self { Self::MultiHead { batch: 1, - heads: shape.heads, - seq: shape.seq, - query: shape.query, + heads: shape.heads(), + model: shape.model_size(), + seq: shape.seq_len(), } } } diff --git a/ml/transformers/src/attention/weights.rs b/ml/transformers/src/attention/weights.rs index a5720fb5..7d7fe587 100644 --- a/ml/transformers/src/attention/weights.rs +++ b/ml/transformers/src/attention/weights.rs @@ -2,6 +2,7 @@ Appellation: weights Contrib: FL03 */ +use super::Weights; use super::params::QKV; use crate::ops::Split; @@ -14,6 +15,8 @@ use strum::IntoEnumIterator; pub type WeightTensor = Array; // (seq, model) + + #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] pub struct Weight where @@ -26,10 +29,7 @@ where } impl Weight { - pub fn new(dim: D) -> Self - where - D: IntoDimension, - { + pub fn new(dim: impl IntoDimension) -> Self { let dim = dim.into_dimension(); let arr = Array2::ones(dim); Self { @@ -55,6 +55,29 @@ impl std::fmt::Display for Weight { } } +impl Split<(Array3, Array3, Array3)> for Weight { + type Error = ndarray::ShapeError; + + fn split(&self, heads: usize) -> Result<(Array3, Array3, Array3), Self::Error> { + let (key, query, value) = self.qkv(); + Ok((key.split(heads)?, query.split(heads)?, value.split(heads)?)) + } +} + +impl Weights for Weight { + fn key(&self) -> &Array2 { + &self.key + } + + fn query(&self) -> &Array2 { + &self.query + } + + fn value(&self) -> &Array2 { + &self.value + } +} + impl From for Weight where D: IntoDimension, @@ -78,14 +101,7 @@ impl From> for (Array2, Array2, Array2) { } } -impl Split<(Array3, Array3, Array3)> for Weight { - type Error = ndarray::ShapeError; - fn split(&self, heads: usize) -> Result<(Array3, Array3, Array3), Self::Error> { - let (key, query, value) = self.qkv(); - Ok((key.split(heads)?, query.split(heads)?, value.split(heads)?)) - } -} impl ops::Index for Weight { type Output = Array2; @@ -111,14 +127,62 @@ impl ops::IndexMut for Weight { } } +impl ndarray::linalg::Dot> for Weight { + type Output = Self; + + fn dot(&self, rhs: &Array2) -> Self::Output { + let mut ctx = self.clone(); + for qkv in QKV::iter() { + ctx[qkv] = ctx[qkv].dot(rhs); + } + ctx + } +} + +impl ops::Mul> for Array2 { + type Output = Weight; + + fn mul(self, rhs: Weight) -> Self::Output { + let mut ctx = rhs.clone(); + for qkv in QKV::iter() { + ctx[qkv] = self.dot(&ctx[qkv]); + } + ctx + } +} + +impl ops::Mul> for &Array2 { + type Output = Weight; + + fn mul(self, rhs: Weight) -> Self::Output { + let mut ctx = rhs.clone(); + for qkv in QKV::iter() { + ctx[qkv] = self.dot(&ctx[qkv]); + } + ctx + } +} + +impl ops::Mul<&Weight> for &Array2 { + type Output = Weight; + + fn mul(self, rhs: &Weight) -> Self::Output { + let mut ctx = rhs.clone(); + for qkv in QKV::iter() { + ctx[qkv] = self.dot(&ctx[qkv]); + } + ctx + } +} + impl ops::Mul> for Weight { type Output = Self; fn mul(self, rhs: Array2) -> Self::Output { let mut ctx = self.clone(); - ctx.key = ctx.key.dot(&rhs); - ctx.query = ctx.query.dot(&rhs); - ctx.value = ctx.value.dot(&rhs); + for qkv in QKV::iter() { + ctx[qkv] = ctx[qkv].dot(&rhs); + } ctx } } @@ -128,9 +192,9 @@ impl ops::Mul<&Array2> for Weight { fn mul(self, rhs: &Array2) -> Self::Output { let mut ctx = self.clone(); - ctx.key = ctx.key.dot(rhs); - ctx.query = ctx.query.dot(rhs); - ctx.value = ctx.value.dot(rhs); + for qkv in QKV::iter() { + ctx[qkv] = ctx[qkv].dot(rhs); + } ctx } } @@ -140,9 +204,9 @@ impl ops::Mul<&Array2> for &Weight { fn mul(self, rhs: &Array2) -> Self::Output { let mut ctx = self.clone(); - ctx.key = ctx.key.dot(rhs); - ctx.query = ctx.query.dot(rhs); - ctx.value = ctx.value.dot(rhs); + for qkv in QKV::iter() { + ctx[qkv] = ctx[qkv].dot(rhs); + } ctx } } diff --git a/ml/transformers/src/codec/encode/encoder.rs b/ml/transformers/src/codec/encode/encoder.rs index 9803bf5c..9ef05913 100644 --- a/ml/transformers/src/codec/encode/encoder.rs +++ b/ml/transformers/src/codec/encode/encoder.rs @@ -5,10 +5,9 @@ use super::EncoderParams; use crate::attention::multi::MultiHeadAttention; use crate::ffn::FFN; -use crate::neural::prelude::{Forward, LayerNorm}; +use crate::neural::prelude::{Forward, LayerNorm, Mask}; use ndarray::prelude::Array2; - pub struct Encoder { attention: MultiHeadAttention, network: FFN, @@ -30,20 +29,20 @@ impl Encoder { } } - fn _forward(&self, data: &Array2) -> Array2 { - let attention = data + self.attention.attention(data); + fn _forward(&self, data: &Array2, mask: &Mask) -> anyhow::Result> { + let attention = data + self.attention.attention(data, mask)?; let norm = self.norm_attention.forward(&attention); let network = data + self.network.forward(&norm); let norm = self.norm_network.forward(&network); - norm + Ok(norm) } - pub fn forward(&mut self, data: &Array2) -> Array2 { + pub fn forward(&mut self, data: &Array2, mask: &Mask) -> anyhow::Result> { let norm = self.norm_attention.forward(data); - let attention = data + self.attention.attention(&norm); + let attention = data + self.attention.attention(&norm, mask)?; let norm = self.norm_network.forward(&attention); let network = data + self.network.forward(&norm); - network + Ok(network) } pub fn params(&self) -> EncoderParams { diff --git a/ml/transformers/src/ffn/network.rs b/ml/transformers/src/ffn/network.rs index 67d98dda..0cc8ad9e 100644 --- a/ml/transformers/src/ffn/network.rs +++ b/ml/transformers/src/ffn/network.rs @@ -4,7 +4,7 @@ */ use super::FFNParams; use crate::data::linear::LinearLayer; -use crate::neural::neurons::activate::{Activator, ReLU}; +use crate::neural::neurons::activate::{Activate, ReLU}; use crate::neural::prelude::Forward; use ndarray::prelude::Array2; use serde::{Deserialize, Serialize}; @@ -32,6 +32,6 @@ impl Forward> for FFN { type Output = Array2; fn forward(&self, data: &Array2) -> Self::Output { - self.output.linear(&ReLU::rho(self.input.linear(data))) + self.output.linear(&Activate::activate(&ReLU, self.input.linear(data))) } } diff --git a/ml/transformers/src/ops/merge.rs b/ml/transformers/src/ops/merge.rs index bb16f626..4f87a01f 100644 --- a/ml/transformers/src/ops/merge.rs +++ b/ml/transformers/src/ops/merge.rs @@ -2,7 +2,7 @@ Appellation: merge Contrib: FL03 */ -use ndarray::prelude::{Array2, Array3, Array4}; +use ndarray::prelude::{Array2, Array3, Array4, Axis}; use ndarray::ShapeError; pub trait Merge { @@ -11,16 +11,25 @@ pub trait Merge { fn merge(&self) -> Result; } -impl Merge> for Array3 { +impl Merge> for Array3 where T: Clone { type Error = ShapeError; fn merge(&self) -> Result, Self::Error> { - let (heads, seq, query) = self.dim(); + if self.ndim() < 3 { + return Err(ShapeError::from_kind(ndarray::ErrorKind::IncompatibleShape)); + } + let axes = (self.ndim() - 3, self.ndim() - 2, self.ndim() - 1); + let mut tmp = self.clone(); // swap the head and sequence axes - tmp.swap_axes(0, 1); + tmp.swap_axes(axes.0, axes.1); // reshape the qkv matrix into a 2d array - tmp.into_shape((seq, heads * query)) + if tmp.merge_axes(Axis(axes.1), Axis(axes.2)) { + let res = tmp.remove_axis(Axis(axes.1)); + Ok(res) + } else { + Err(ShapeError::from_kind(ndarray::ErrorKind::IncompatibleShape)) + } } } From 707fb03017aa304d29b72bc1a53d8109873d46ee Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 3 Nov 2023 15:13:37 -0500 Subject: [PATCH 038/118] update Signed-off-by: FL03 --- core/src/errors/error.rs | 1 - core/src/specs.rs | 5 +- data/src/lib.rs | 1 - data/src/linear/features.rs | 36 --------------- data/src/linear/layer.rs | 46 ------------------- data/src/linear/mod.rs | 22 --------- ml/neural/src/bias/biases.rs | 10 ++-- ml/neural/src/bias/mask.rs | 18 +++++--- ml/neural/src/bias/mod.rs | 4 +- ml/neural/src/layers/layer.rs | 7 ++- ml/neural/src/layers/linear/layer.rs | 8 ++-- ml/neural/src/layers/linear/mod.rs | 9 ++-- ml/neural/src/neurons/activate/activator.rs | 25 ++++++---- ml/neural/src/neurons/activate/nonlinear.rs | 10 ++-- ml/neural/src/neurons/mod.rs | 2 +- ml/neural/src/specs.rs | 11 +++-- ml/neural/src/weights/mod.rs | 18 +++++--- ml/neural/src/weights/weight.rs | 20 ++++---- ml/transformers/src/attention/head.rs | 2 - .../src/attention/multi/attention.rs | 42 +++++++++++------ ml/transformers/src/attention/multi/mod.rs | 24 ++++------ ml/transformers/src/attention/params/dim.rs | 5 +- ml/transformers/src/attention/weights.rs | 6 +-- ml/transformers/src/codec/encode/mod.rs | 7 ++- ml/transformers/src/ffn/network.rs | 5 +- ml/transformers/src/lib.rs | 1 - ml/transformers/src/ops/merge.rs | 28 +++++------ 27 files changed, 144 insertions(+), 229 deletions(-) delete mode 100644 data/src/linear/features.rs delete mode 100644 data/src/linear/layer.rs delete mode 100644 data/src/linear/mod.rs diff --git a/core/src/errors/error.rs b/core/src/errors/error.rs index 5bcff20b..95861212 100644 --- a/core/src/errors/error.rs +++ b/core/src/errors/error.rs @@ -154,4 +154,3 @@ impl From for Error { Self::new(Errors::Syntax, err.to_string()) } } - diff --git a/core/src/specs.rs b/core/src/specs.rs index d23453e5..54b1c7d7 100644 --- a/core/src/specs.rs +++ b/core/src/specs.rs @@ -2,8 +2,8 @@ Appellation: specs Contrib: FL03 */ +use ndarray::prelude::Array; use ndarray::{Dimension, ShapeError}; -use ndarray::prelude::{Array}; use num::{Num, One, Zero}; use std::ops::MulAssign; @@ -44,7 +44,6 @@ where } } - trait Matmul where T: Num, @@ -77,8 +76,6 @@ where // let mut iter_self = self.iter(); // let mut iter_other = other.iter(); - - // for mut row_result in result.outer_iter_mut() { // for mut col_other in other.inner_iter() { // let row_self = iter_self.clone(); diff --git a/data/src/lib.rs b/data/src/lib.rs index f02f2fcd..f736b907 100644 --- a/data/src/lib.rs +++ b/data/src/lib.rs @@ -11,7 +11,6 @@ pub(crate) mod utils; pub mod df; pub mod flows; -pub mod linear; pub mod tensors; pub mod prelude { diff --git a/data/src/linear/features.rs b/data/src/linear/features.rs deleted file mode 100644 index 2854ef26..00000000 --- a/data/src/linear/features.rs +++ /dev/null @@ -1,36 +0,0 @@ -/* - Appellation: features - Contrib: FL03 -*/ -use ndarray::IntoDimension; -use serde::{Deserialize, Serialize}; - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct Features { - pub input: usize, - pub output: usize, -} - -impl Features { - pub fn new(input: usize, output: usize) -> Self { - Self { input, output } - } - - pub fn input(&self) -> usize { - self.input - } - - pub fn output(&self) -> usize { - self.output - } -} - -impl IntoDimension for Features { - type Dim = ndarray::IxDyn; - - fn into_dimension(self) -> Self::Dim { - ndarray::IxDyn(&[self.input, self.output]) - } -} diff --git a/data/src/linear/layer.rs b/data/src/linear/layer.rs deleted file mode 100644 index 8375b3c0..00000000 --- a/data/src/linear/layer.rs +++ /dev/null @@ -1,46 +0,0 @@ -/* - Appellation: layer - Contrib: FL03 -*/ -use super::Features; -use ndarray::prelude::{Array1, Array2}; -use num::Float; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -pub struct LinearLayer { - bias: Array1, - pub params: Features, - weights: Array2, -} - -impl LinearLayer -where - T: Float, -{ - pub fn new(inputs: usize, outputs: usize) -> Self { - let params = Features::new(inputs, outputs); - let weights = Array2::ones((params.input, params.output)); - let bias = Array1::ones(params.output); - Self { - bias, - params, - weights, - } - } - - pub fn bias(&self) -> &Array1 { - &self.bias - } - - pub fn linear(&self, data: &Array2) -> Array2 - where - T: 'static, - { - data.dot(&self.weights.t()) + &self.bias - } - - pub fn weights(&self) -> &Array2 { - &self.weights - } -} diff --git a/data/src/linear/mod.rs b/data/src/linear/mod.rs deleted file mode 100644 index 8c426ed3..00000000 --- a/data/src/linear/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -/* - Appellation: linear - Contrib: FL03 -*/ -//! # Linear Layers -pub use self::{features::*, layer::*, utils::*}; - -pub(crate) mod features; -pub(crate) mod layer; - -pub(crate) mod utils { - use ndarray::prelude::{Array1, Array2}; - use num::Float; - - pub fn linear_transformation( - data: &Array2, - bias: &Array1, - weights: &Array2, - ) -> Array2 { - data.dot(&weights.t()) + bias - } -} diff --git a/ml/neural/src/bias/biases.rs b/ml/neural/src/bias/biases.rs index 4a11a5ce..f634e7e5 100644 --- a/ml/neural/src/bias/biases.rs +++ b/ml/neural/src/bias/biases.rs @@ -9,8 +9,8 @@ use ndarray_rand::RandomExt; use num::Float; use serde::{Deserialize, Serialize}; use smart_default::SmartDefault; -use strum::EnumIs; use std::ops; +use strum::EnumIs; fn _generate_bias(size: usize) -> Array1 { let ds = (T::from(size).unwrap()).sqrt(); @@ -34,9 +34,11 @@ impl Bias { } } -impl Bias where T: Float + SampleUniform { - pub fn biased(size: usize) -> Self - { +impl Bias +where + T: Float + SampleUniform, +{ + pub fn biased(size: usize) -> Self { let bias = _generate_bias(size); Self::Biased(bias) } diff --git a/ml/neural/src/bias/mask.rs b/ml/neural/src/bias/mask.rs index b794622f..00fc7db7 100644 --- a/ml/neural/src/bias/mask.rs +++ b/ml/neural/src/bias/mask.rs @@ -9,8 +9,8 @@ use ndarray_rand::RandomExt; use num::Float; use serde::{Deserialize, Serialize}; use smart_default::SmartDefault; -use strum::EnumIs; use std::ops; +use strum::EnumIs; #[derive(Clone, Debug, Deserialize, EnumIs, PartialEq, Serialize, SmartDefault)] pub enum Mask { @@ -28,9 +28,11 @@ impl Mask { } } -impl Mask where T: Float + SampleUniform { - pub fn masked(size: usize) -> Self - { +impl Mask +where + T: Float + SampleUniform, +{ + pub fn masked(size: usize) -> Self { let ds = (T::from(size).unwrap()).sqrt(); let dist = Uniform::new(-ds, ds); let mask = Array2::::random((size, size), dist); @@ -38,9 +40,11 @@ impl Mask where T: Float + SampleUniform { } } -impl From for Mask where T: Float + SampleUniform { - fn from(size: usize) -> Self - { +impl From for Mask +where + T: Float + SampleUniform, +{ + fn from(size: usize) -> Self { let ds = (T::from(size).unwrap()).sqrt(); let dist = Uniform::new(-ds, ds); let mask = Array2::::random((size, size), dist); diff --git a/ml/neural/src/bias/mod.rs b/ml/neural/src/bias/mod.rs index ab2c98d8..192adc34 100644 --- a/ml/neural/src/bias/mod.rs +++ b/ml/neural/src/bias/mod.rs @@ -11,13 +11,11 @@ pub(crate) mod mask; use num::Float; pub trait Biased { - fn bias(&self) -> &Bias; fn bias_mut(&mut self) -> &mut Bias; } -pub(crate) mod utils { -} +pub(crate) mod utils {} #[cfg(test)] mod tests {} diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 6c5ddcaf..44611d25 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -56,8 +56,11 @@ impl Layer { } } -impl Layer where T: Float + SampleUniform { - pub fn biased(inputs: usize, outputs: usize, layer: LayerType) -> Self +impl Layer +where + T: Float + SampleUniform, +{ + pub fn biased(inputs: usize, outputs: usize, layer: LayerType) -> Self where T: SampleUniform, { diff --git a/ml/neural/src/layers/linear/layer.rs b/ml/neural/src/layers/linear/layer.rs index a53fedc9..ad0b7918 100644 --- a/ml/neural/src/layers/linear/layer.rs +++ b/ml/neural/src/layers/linear/layer.rs @@ -20,7 +20,6 @@ impl LinearLayer where T: Float, { - pub fn bias(&self) -> &Bias { &self.bias } @@ -37,7 +36,10 @@ where } } -impl LinearLayer where T: Float + SampleUniform { +impl LinearLayer +where + T: Float + SampleUniform, +{ pub fn new(inputs: usize, outputs: usize) -> Self { let params = Features::new(inputs, outputs); let weights = Array2::ones((inputs, outputs)); @@ -56,4 +58,4 @@ impl Forward> for LinearLayer { fn forward(&self, data: &Array2) -> Self::Output { data.dot(&self.weights().t()) + self.bias() } -} \ No newline at end of file +} diff --git a/ml/neural/src/layers/linear/mod.rs b/ml/neural/src/layers/linear/mod.rs index 3dcb48d7..85745615 100644 --- a/ml/neural/src/layers/linear/mod.rs +++ b/ml/neural/src/layers/linear/mod.rs @@ -12,7 +12,10 @@ use crate::weights::Weighted; use ndarray::prelude::Array2; use num::Float; -pub trait LinearTransformation where T: Float { +pub trait LinearTransformation +where + T: Float, +{ fn linear(&self, data: &Array2) -> Array2; } @@ -39,7 +42,6 @@ pub(crate) mod utils { } } - #[cfg(test)] mod tests { use super::*; @@ -50,6 +52,5 @@ mod tests { let data = Array2::::ones((inputs, outputs)); let layer = LinearLayer::new(inputs, outputs); let linear = layer.linear(&data); - } -} \ No newline at end of file +} diff --git a/ml/neural/src/neurons/activate/activator.rs b/ml/neural/src/neurons/activate/activator.rs index 867f1b37..6a36d526 100644 --- a/ml/neural/src/neurons/activate/activator.rs +++ b/ml/neural/src/neurons/activate/activator.rs @@ -2,7 +2,7 @@ Appellation: activator Contrib: FL03 */ -use super::{Activate, ActivationMethod, ActivationFn}; +use super::{Activate, ActivationFn, ActivationMethod}; use std::marker::PhantomData; pub trait ActivationParams {} @@ -17,26 +17,31 @@ pub trait LinearActivation: ActivationMethod { } } -pub struct Activator where A: Activate { +pub struct Activator +where + A: Activate, +{ method: A, - _args: PhantomData + _args: PhantomData, } -impl Activator where A: Activate { +impl Activator +where + A: Activate, +{ pub fn new(method: A) -> Self { Activator { method, - _args: PhantomData + _args: PhantomData, } } } -impl Activate for Activator where A: Activate { +impl Activate for Activator +where + A: Activate, +{ fn activate(&self, x: T) -> T { self.method.activate(x) } } - - - - diff --git a/ml/neural/src/neurons/activate/nonlinear.rs b/ml/neural/src/neurons/activate/nonlinear.rs index 29edac7f..aae3c878 100644 --- a/ml/neural/src/neurons/activate/nonlinear.rs +++ b/ml/neural/src/neurons/activate/nonlinear.rs @@ -3,8 +3,8 @@ Contrib: FL03 */ use super::Activate; -use ndarray::{Dimension, RemoveAxis, ScalarOperand}; use ndarray::prelude::{Array, Array1, Axis}; +use ndarray::{Dimension, RemoveAxis, ScalarOperand}; use num::{Float, Zero}; use serde::{Deserialize, Serialize}; @@ -71,16 +71,16 @@ where } } -#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] pub struct Softmax { axis: Option, } impl Softmax { pub fn new(axis: Option) -> Self { - Self { - axis - } + Self { axis } } } diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index 686cb0d2..1b8062ef 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -16,7 +16,7 @@ pub(crate) mod utils {} #[cfg(test)] mod tests { - use super::activate::{Activate, Softmax, softmax}; + use super::activate::{softmax, Activate, Softmax}; use super::*; use ndarray::{array, Array1}; diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index 0d53686f..607c183f 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -2,14 +2,17 @@ Appellation: specs Contrib: FL03 */ -use ndarray::{Dimension, IntoDimension}; use ndarray::prelude::{Array, Array1, Array2}; -use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray::{Dimension, IntoDimension}; use ndarray_rand::rand_distr as dist; +use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::RandomExt; use num::Float; -pub trait InitUniform where T: Float + SampleUniform { +pub trait InitUniform +where + T: Float + SampleUniform, +{ type Dim: Dimension; fn uniform(axis: usize, dim: impl IntoDimension) -> Array { @@ -20,8 +23,6 @@ pub trait InitUniform where T: Float + SampleUniform { } } - - pub trait Trainable { fn train(&mut self, args: &Array2) -> Array2; } diff --git a/ml/neural/src/weights/mod.rs b/ml/neural/src/weights/mod.rs index 7551ccc0..96065754 100644 --- a/ml/neural/src/weights/mod.rs +++ b/ml/neural/src/weights/mod.rs @@ -3,21 +3,27 @@ Contrib: FL03 */ //! # Weights -pub use self::{weight::*, utils::*}; +pub use self::{utils::*, weight::*}; pub(crate) mod weight; - use ndarray::prelude::Array2; use num::Float; -pub trait Weighted where T: Float { +pub trait Weighted +where + T: Float, +{ fn weights(&self) -> &Array2; fn weights_mut(&mut self) -> &mut Array2; } -impl Weighted for S where S: AsMut> + AsRef>, T: Float { +impl Weighted for S +where + S: AsMut> + AsRef>, + T: Float, +{ fn weights(&self) -> &Array2 { self.as_ref() } @@ -27,9 +33,7 @@ impl Weighted for S where S: AsMut> + AsRef>, T: Fl } } - -pub(crate) mod utils { -} +pub(crate) mod utils {} #[cfg(test)] mod tests {} diff --git a/ml/neural/src/weights/weight.rs b/ml/neural/src/weights/weight.rs index ea94776f..0f86f02c 100644 --- a/ml/neural/src/weights/weight.rs +++ b/ml/neural/src/weights/weight.rs @@ -3,30 +3,34 @@ Contrib: FL03 */ use ndarray::prelude::Array2; -use ndarray_rand::RandomExt; -use ndarray_rand::rand_distr::Uniform; use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::Uniform; +use ndarray_rand::RandomExt; use num::Float; use serde::{Deserialize, Serialize}; - - #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Weight { - weights: Array2 + weights: Array2, } -impl Weight where T: Default { +impl Weight +where + T: Default, +{ pub fn new(m: usize, n: usize) -> Self { let weights = Array2::default((m, n)); Self { weights } } } -impl Weight where T: Float + SampleUniform { +impl Weight +where + T: Float + SampleUniform, +{ pub fn uniform(m: usize, n: usize) -> Array2 { let dk = (T::from(m).unwrap()).sqrt(); let dist = Uniform::new(-dk, dk); Array2::random((m, n), dist) } -} \ No newline at end of file +} diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index ca4cea49..07c6d664 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -32,8 +32,6 @@ impl AttentionHead { self.dim } - - pub fn mask_mut(&mut self) -> &mut Array2 { &mut self.mask } diff --git a/ml/transformers/src/attention/multi/attention.rs b/ml/transformers/src/attention/multi/attention.rs index e4e08a6a..23109b95 100644 --- a/ml/transformers/src/attention/multi/attention.rs +++ b/ml/transformers/src/attention/multi/attention.rs @@ -4,44 +4,60 @@ */ use super::{multihead, MultiHeadParams}; use crate::attention::Weight; -use crate::neural::prelude::{ Mask}; +use crate::neural::layers::linear::LinearLayer; +use crate::neural::prelude::Mask; use crate::ops::Split; -use ndarray::{ScalarOperand, ShapeError}; use ndarray::prelude::Array2; +use ndarray::{ScalarOperand, ShapeError}; +use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct MultiHeadAttention { + linear: LinearLayer, params: MultiHeadParams, weights: Weight, } -impl MultiHeadAttention { +impl MultiHeadAttention +where + T: Float, +{ + pub fn params(&self) -> MultiHeadParams { + self.params + } + + pub fn weights(&self) -> &Weight { + &self.weights + } +} + +impl MultiHeadAttention +where + T: Float + SampleUniform, +{ pub fn new(heads: usize, model: usize) -> Self { let params = MultiHeadParams::new(heads, model); let weights = Weight::new((model, model)); Self { + linear: LinearLayer::new(model, model), params, weights, } } - - pub fn params(&self) -> MultiHeadParams { - self.params - } - - pub fn weights(&self) -> &Weight { - &self.weights - } } -impl MultiHeadAttention { +impl MultiHeadAttention +where + T: Float + ScalarOperand, +{ pub fn attention(&self, data: &Array2, mask: &Mask) -> Result, ShapeError> { let weighted = data * self.weights(); let (q, k, v) = weighted.split(self.params().heads())?; let score = multihead(&q, &k, &v, mask)?; - Ok(score) + let res = self.linear.linear(&score); + Ok(res) } } diff --git a/ml/transformers/src/attention/multi/mod.rs b/ml/transformers/src/attention/multi/mod.rs index f828fba4..ab36daad 100644 --- a/ml/transformers/src/attention/multi/mod.rs +++ b/ml/transformers/src/attention/multi/mod.rs @@ -35,7 +35,7 @@ pub(crate) mod utils { use crate::attention::attention; use crate::neural::prelude::Mask; use crate::ops::Merge; - use ndarray::prelude::{Array2, Array3, Array4, Axis, azip, s}; + use ndarray::prelude::{s, Array2, Array3, Array4}; use ndarray::{ScalarOperand, ShapeError}; use num::Float; @@ -69,8 +69,7 @@ pub(crate) mod utils { where T: Float + ScalarOperand, { - let (heads, seq, dk) = query.dim(); - println!("heads: {}, seq: {}, query: {}", heads, seq, dk); + let (heads, _, _) = query.dim(); let mut score = Array3::::zeros(query.dim()); for h in 0..heads { let pos = s![h, .., ..]; @@ -78,13 +77,9 @@ pub(crate) mod utils { let k = key.slice(pos).to_owned(); let v = value.slice(pos).to_owned(); let head = attention(&q, &k, &v, mask.clone().into()); - println!("head dim: {:?}", &head.shape()); score.slice_mut(s![h, .., ..]).assign(&head); } - - let score = score.merge()?; - println!("score: {:?}", &score.shape()); - Ok(score) + score.merge() } } @@ -93,17 +88,16 @@ mod tests { use super::*; use crate::neural::prelude::Mask; - #[test] - fn test_multihead() { + fn test_multihead_shape() { let (heads, seq, model) = (8, 10, 512); let data = Array2::::zeros((seq, model)); - let _weights = Weight::::new((model, model)); - let _params = MultiHeadParams::new(heads, model); - let mask: Mask = Array2::::zeros((seq, seq)).into(); + let mask = Mask::::masked(seq).into(); let attention = MultiHeadAttention::new(heads, model); - let score = attention.attention(&data, &mask).expect("Failed to compute attention"); - assert_eq!(score, Array2::::zeros((10, model))); + let score = attention + .attention(&data, &mask) + .expect("Failed to compute attention"); + assert_eq!(score.dim(), (seq, model)); } } diff --git a/ml/transformers/src/attention/params/dim.rs b/ml/transformers/src/attention/params/dim.rs index ff665dda..910f156e 100644 --- a/ml/transformers/src/attention/params/dim.rs +++ b/ml/transformers/src/attention/params/dim.rs @@ -11,8 +11,8 @@ //! - `heads`: The number of attention heads //! - `model`: The dimension of the model (embedding size) use crate::{HEADS, MODEL_SIZE, QUERY_SIZE}; -use ndarray::IntoDimension; use ndarray::prelude::{Ix3, Ix4}; +use ndarray::IntoDimension; use serde::{Deserialize, Serialize}; pub trait Batched { @@ -31,8 +31,6 @@ impl Batched for Ix4 { } } - - pub enum AttentionDims { Base(BaseShape), // a 3d matrix (batch, seq, model) Head(HeadShape), // a 2d matrix (seq, query) @@ -91,7 +89,6 @@ impl BaseShape { } } - impl IntoDimension for BaseShape { type Dim = ndarray::Ix3; diff --git a/ml/transformers/src/attention/weights.rs b/ml/transformers/src/attention/weights.rs index 7d7fe587..4418ec3c 100644 --- a/ml/transformers/src/attention/weights.rs +++ b/ml/transformers/src/attention/weights.rs @@ -2,8 +2,8 @@ Appellation: weights Contrib: FL03 */ -use super::Weights; use super::params::QKV; +use super::Weights; use crate::ops::Split; use ndarray::prelude::{Array, Array2, Array3}; @@ -15,8 +15,6 @@ use strum::IntoEnumIterator; pub type WeightTensor = Array; // (seq, model) - - #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] pub struct Weight where @@ -101,8 +99,6 @@ impl From> for (Array2, Array2, Array2) { } } - - impl ops::Index for Weight { type Output = Array2; diff --git a/ml/transformers/src/codec/encode/mod.rs b/ml/transformers/src/codec/encode/mod.rs index 9298e844..c70c7402 100644 --- a/ml/transformers/src/codec/encode/mod.rs +++ b/ml/transformers/src/codec/encode/mod.rs @@ -16,14 +16,17 @@ pub(crate) mod utils {} #[cfg(test)] mod tests { use super::*; + use crate::neural::prelude::Mask; use ndarray::Array2; #[test] fn test_encoder() { - let (heads, model) = (8, 512); - let _data = Array2::::zeros((512, 512)); + let (heads, seq, model) = (8, 10, 512); + let _data = Array2::::zeros((seq, model)); + let _mask = Mask::::masked(seq); let params = EncoderParams::new(heads, model); let encoder = Encoder::new(params); + assert_eq!(encoder.params().heads(), heads); } } diff --git a/ml/transformers/src/ffn/network.rs b/ml/transformers/src/ffn/network.rs index 0cc8ad9e..8f449c81 100644 --- a/ml/transformers/src/ffn/network.rs +++ b/ml/transformers/src/ffn/network.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use super::FFNParams; -use crate::data::linear::LinearLayer; +use crate::neural::layers::linear::LinearLayer; use crate::neural::neurons::activate::{Activate, ReLU}; use crate::neural::prelude::Forward; use ndarray::prelude::Array2; @@ -32,6 +32,7 @@ impl Forward> for FFN { type Output = Array2; fn forward(&self, data: &Array2) -> Self::Output { - self.output.linear(&Activate::activate(&ReLU, self.input.linear(data))) + self.output + .linear(&Activate::activate(&ReLU, self.input.linear(data))) } } diff --git a/ml/transformers/src/lib.rs b/ml/transformers/src/lib.rs index cf4c8ff0..aa1d8e61 100644 --- a/ml/transformers/src/lib.rs +++ b/ml/transformers/src/lib.rs @@ -16,7 +16,6 @@ pub mod ops; pub mod transform; pub(crate) use concision_core as core; -pub(crate) use concision_data as data; pub(crate) use concision_neural as neural; pub mod prelude { diff --git a/ml/transformers/src/ops/merge.rs b/ml/transformers/src/ops/merge.rs index 4f87a01f..f7279298 100644 --- a/ml/transformers/src/ops/merge.rs +++ b/ml/transformers/src/ops/merge.rs @@ -2,8 +2,8 @@ Appellation: merge Contrib: FL03 */ -use ndarray::prelude::{Array2, Array3, Array4, Axis}; -use ndarray::ShapeError; +use ndarray::prelude::{Array2, Array3, Array4}; +use ndarray::{Order, ShapeError}; pub trait Merge { type Error; @@ -11,25 +11,20 @@ pub trait Merge { fn merge(&self) -> Result; } -impl Merge> for Array3 where T: Clone { +impl Merge> for Array3 +where + T: Clone, +{ type Error = ShapeError; fn merge(&self) -> Result, Self::Error> { - if self.ndim() < 3 { - return Err(ShapeError::from_kind(ndarray::ErrorKind::IncompatibleShape)); - } - let axes = (self.ndim() - 3, self.ndim() - 2, self.ndim() - 1); - + let (heads, seq, query) = self.dim(); let mut tmp = self.clone(); // swap the head and sequence axes - tmp.swap_axes(axes.0, axes.1); + tmp.swap_axes(0, 1); // reshape the qkv matrix into a 2d array - if tmp.merge_axes(Axis(axes.1), Axis(axes.2)) { - let res = tmp.remove_axis(Axis(axes.1)); - Ok(res) - } else { - Err(ShapeError::from_kind(ndarray::ErrorKind::IncompatibleShape)) - } + let res = tmp.to_shape(((seq, heads * query), Order::ColumnMajor))?; + Ok(res.to_owned()) } } @@ -42,6 +37,7 @@ impl Merge> for Array4 { // swap the head and sequence axes tmp.swap_axes(1, 2); // reshape the qkv matrix into a 2d array - tmp.into_shape((batch, seq, heads * query)) + let res = tmp.to_shape(((batch, seq, heads * query), Order::ColumnMajor))?; + Ok(res.to_owned()) } } From 87f006b35fe98ee842cd31438eeac5129e934348 Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 3 Nov 2023 15:15:26 -0500 Subject: [PATCH 039/118] update Signed-off-by: FL03 --- ml/neural/src/layers/linear/mod.rs | 1 + ml/neural/src/specs.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ml/neural/src/layers/linear/mod.rs b/ml/neural/src/layers/linear/mod.rs index 85745615..0f96a320 100644 --- a/ml/neural/src/layers/linear/mod.rs +++ b/ml/neural/src/layers/linear/mod.rs @@ -52,5 +52,6 @@ mod tests { let data = Array2::::ones((inputs, outputs)); let layer = LinearLayer::new(inputs, outputs); let linear = layer.linear(&data); + assert_eq!(linear.dim(), (inputs, outputs)); } } diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index 607c183f..bfa647d8 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -2,7 +2,7 @@ Appellation: specs Contrib: FL03 */ -use ndarray::prelude::{Array, Array1, Array2}; +use ndarray::prelude::{Array, Array2}; use ndarray::{Dimension, IntoDimension}; use ndarray_rand::rand_distr as dist; use ndarray_rand::rand_distr::uniform::SampleUniform; From 9930eac754e83d9dc63da5e1c1fa3ba712ff7a84 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 5 Nov 2023 12:49:27 -0600 Subject: [PATCH 040/118] update Signed-off-by: FL03 --- Cargo.toml | 1 + core/src/primitives.rs | 6 +- core/src/utils.rs | 4 +- ml/neural/src/grad/descent.rs | 29 ++++ ml/neural/src/grad/mod.rs | 19 +++ ml/neural/src/grad/sgd.rs | 105 ++++++++++++ ml/neural/src/layers/linear/layer.rs | 53 +++++- ml/neural/src/lib.rs | 1 + ml/neural/src/nn/loss/mod.rs | 45 ++---- ml/neural/src/prop/mod.rs | 3 +- ml/neural/src/specs.rs | 37 ++++- ml/nlp/Cargo.toml | 2 + ml/nlp/src/encode/positional.rs | 153 +++++++++++++++--- ml/transformers/src/attention/head.rs | 22 ++- .../src/attention/multi/attention.rs | 8 +- ml/transformers/src/attention/weights.rs | 97 ++++++++--- optim/Cargo.toml | 50 ++++++ optim/benches/default.rs | 52 ++++++ optim/src/lib.rs | 20 +++ optim/src/primitives.rs | 11 ++ optim/src/specs.rs | 4 + optim/src/utils.rs | 4 + optim/tests/default.rs | 8 + 23 files changed, 642 insertions(+), 92 deletions(-) create mode 100644 ml/neural/src/grad/descent.rs create mode 100644 ml/neural/src/grad/mod.rs create mode 100644 ml/neural/src/grad/sgd.rs create mode 100644 optim/Cargo.toml create mode 100644 optim/benches/default.rs create mode 100644 optim/src/lib.rs create mode 100644 optim/src/primitives.rs create mode 100644 optim/src/specs.rs create mode 100644 optim/src/utils.rs create mode 100644 optim/tests/default.rs diff --git a/Cargo.toml b/Cargo.toml index 3e638501..c741f9ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ members = [ "data", "derive", "macros", + "optim", "ml/*", ] diff --git a/core/src/primitives.rs b/core/src/primitives.rs index 3e1b7491..64c09104 100644 --- a/core/src/primitives.rs +++ b/core/src/primitives.rs @@ -5,13 +5,13 @@ pub use self::{constants::*, statics::*, types::*}; /// Collection of constants used throughout the system -pub(crate) mod constants {} +mod constants {} /// Collection of static references used throughout -pub(crate) mod statics {} +mod statics {} /// Collection of types used throughout the system -pub(crate) mod types { +mod types { /// pub type BoxError = Box; /// diff --git a/core/src/utils.rs b/core/src/utils.rs index 47b00354..0142d5bb 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -2,7 +2,8 @@ Appellation: utils Contrib: FL03 */ -use ndarray::{concatenate, Array, Axis, RemoveAxis}; +use ndarray::prelude::{Array, Axis}; +use ndarray::{concatenate, RemoveAxis}; // use num::Float; pub fn concat_iter(axis: usize, iter: impl IntoIterator>) -> Array @@ -17,6 +18,7 @@ where } out } + pub fn now() -> u128 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) diff --git a/ml/neural/src/grad/descent.rs b/ml/neural/src/grad/descent.rs new file mode 100644 index 00000000..3acfdb15 --- /dev/null +++ b/ml/neural/src/grad/descent.rs @@ -0,0 +1,29 @@ +/* + Appellation: grad + Contrib: FL03 +*/ + +use crate::layers::linear::LinearLayer; +use crate::nn::loss::mse; +use crate::prelude::Forward; +use ndarray::prelude::{s, Array1, Array2}; + +fn gradient_descent( + x: &Array2, + y: &Array1, + model: &mut LinearLayer, + learning_rate: f64, + epochs: usize, +) { + let n_samples = x.shape()[0]; + + for epoch in 0..epochs { + let predictions = model.forward(x); + let errors = &predictions - y; + let gradient = x.t().dot(&errors) / n_samples as f64; + + model.update_with_gradient(&gradient, learning_rate); + + // let err = mse(&predictions, y).expect("Error calculating MSE"); + } +} diff --git a/ml/neural/src/grad/mod.rs b/ml/neural/src/grad/mod.rs new file mode 100644 index 00000000..22709354 --- /dev/null +++ b/ml/neural/src/grad/mod.rs @@ -0,0 +1,19 @@ +/* + Appellation: grad + Contrib: FL03 +*/ +//! # Gradient Descent +pub use self::{descent::*, utils::*}; + +pub(crate) mod descent; + +pub mod sgd; + +pub trait Descent { + fn descent(&self, params: &[f64], grads: &[f64]) -> Vec; +} + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests {} diff --git a/ml/neural/src/grad/sgd.rs b/ml/neural/src/grad/sgd.rs new file mode 100644 index 00000000..bc1c6205 --- /dev/null +++ b/ml/neural/src/grad/sgd.rs @@ -0,0 +1,105 @@ +/* + Appellation: sgd + Contrib: FL03 +*/ +//! # Stochastic Gradient Descent (SGD) +//! +//! +use crate::layers::linear::LinearLayer; +use crate::nn::loss::mse; +use crate::prelude::Forward; +use ndarray::prelude::{s, Array1, Array2}; +use num::Float; +use rand::seq::SliceRandom; + +fn sgd( + x: &Array2, + y: &Array1, + model: &mut LinearLayer, + learning_rate: f64, + epochs: usize, + batch_size: usize, +) { + let n_samples = x.shape()[0]; + let input_size = x.shape()[1]; + + for epoch in 0..epochs { + let mut rng = rand::thread_rng(); + let mut indices: Vec = (0..n_samples).collect(); + indices.shuffle(&mut rng); + + for batch_start in (0..n_samples).step_by(batch_size) { + let batch_end = (batch_start + batch_size).min(n_samples); + let mut gradient = Array2::zeros(x.dim()); + + for i in batch_start..batch_end { + let sample_index = indices[i]; + let input = x.slice(s![sample_index, ..]).to_owned(); + let prediction = model.forward(&input); + let error = prediction - y[sample_index]; + gradient + .slice_mut(s![sample_index, ..]) + .assign(&(input * error)); + } + + gradient /= batch_size as f64; + model.update_with_gradient(&gradient, learning_rate); + } + + // println!("Epoch {}: Loss = {}", epoch, mse(&model.forward(x), y).unwrap()); + } +} + +pub struct StochasticGradientDescent +where + T: Float, +{ + batch_size: usize, + epochs: usize, + learning_rate: f64, + model: LinearLayer, +} + +impl StochasticGradientDescent +where + T: Float, +{ + pub fn new( + model: LinearLayer, + learning_rate: f64, + epochs: usize, + batch_size: usize, + ) -> Self { + Self { + batch_size, + epochs, + learning_rate, + model, + } + } + + pub fn train(&mut self, x: &Array2, y: &Array1) {} + + pub fn model(&self) -> &LinearLayer { + &self.model + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_sgd() { + // Generate some example data + let x = Array2::from_shape_vec((100, 2), (0..200).map(|x| x as f64).collect()).unwrap(); + let y = x.dot(&Array1::from_elem(2, 2.0)) + &Array1::from_elem(100, 1.0); + + let mut model = LinearLayer::new(200, 100); + let learning_rate = 0.01; + let epochs = 100; + let batch_size = 10; + + sgd(&x, &y, &mut model, learning_rate, epochs, batch_size); + } +} diff --git a/ml/neural/src/layers/linear/layer.rs b/ml/neural/src/layers/linear/layer.rs index ad0b7918..f1808a60 100644 --- a/ml/neural/src/layers/linear/layer.rs +++ b/ml/neural/src/layers/linear/layer.rs @@ -4,7 +4,8 @@ */ use crate::layers::Features; use crate::prelude::{Bias, Forward}; -use ndarray::prelude::Array2; +use ndarray::prelude::{Array1, Array2}; +use ndarray::ScalarOperand; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; @@ -24,6 +25,18 @@ where &self.bias } + pub fn bias_mut(&mut self) -> &mut Bias { + &mut self.bias + } + + pub fn params(&self) -> &Features { + &self.params + } + + pub fn params_mut(&mut self) -> &mut Features { + &mut self.params + } + pub fn linear(&self, data: &Array2) -> Array2 where T: 'static, @@ -34,6 +47,19 @@ where pub fn weights(&self) -> &Array2 { &self.weights } + + pub fn weights_mut(&mut self) -> &mut Array2 { + &mut self.weights + } + + pub fn update_weights(&mut self, weights: Array2) { + self.weights = weights; + } + + pub fn with_params(mut self, params: Features) -> Self { + self.params = params; + self + } } impl LinearLayer @@ -52,7 +78,30 @@ where } } -impl Forward> for LinearLayer { +impl LinearLayer +where + T: Float + ScalarOperand, +{ + pub fn update_with_gradient(&mut self, gradient: &Array2, lr: T) { + self.weights = self.weights() + gradient * lr; + } +} + +impl Forward> for LinearLayer +where + T: Float + ScalarOperand, +{ + type Output = Array1; + + fn forward(&self, data: &Array1) -> Self::Output { + data.dot(&self.weights().t()) + self.bias() + } +} + +impl Forward> for LinearLayer +where + T: Float + ScalarOperand, +{ type Output = Array2; fn forward(&self, data: &Array2) -> Self::Output { diff --git a/ml/neural/src/lib.rs b/ml/neural/src/lib.rs index 705b54ce..9955aaf3 100644 --- a/ml/neural/src/lib.rs +++ b/ml/neural/src/lib.rs @@ -15,6 +15,7 @@ pub(crate) mod utils; pub mod arch; pub mod bias; +pub mod grad; pub mod layers; pub mod models; pub mod neurons; diff --git a/ml/neural/src/nn/loss/mod.rs b/ml/neural/src/nn/loss/mod.rs index 6262565b..832f0c3f 100644 --- a/ml/neural/src/nn/loss/mod.rs +++ b/ml/neural/src/nn/loss/mod.rs @@ -19,35 +19,24 @@ pub trait Loss { } pub(crate) mod utils { - use ndarray::Array1; - - pub fn mae(pred: Array1, target: Array1) -> anyhow::Result { - if pred.shape() != target.shape() { - return Err(anyhow::anyhow!( - "Mismatched shapes: {:?} and {:?}", - pred.shape(), - target.shape() - )); - } - // the number of elements in the array - let n = pred.len() as f64; - let mut res = (target - pred).mapv(|x| x.abs()).sum(); - res /= n; - Ok(res) + use ndarray::prelude::Array; + use ndarray::{Dimension, ScalarOperand}; + use num::{Float, FromPrimitive}; + use std::ops; + + pub fn mae(pred: &Array, target: &Array) -> Option + where + T: Float + FromPrimitive + ScalarOperand + ops::DivAssign, + D: Dimension, + { + (target - pred).mapv(|x| x.abs()).mean() } - pub fn mse(pred: Array1, target: Array1) -> anyhow::Result { - if pred.shape() != target.shape() { - return Err(anyhow::anyhow!( - "Mismatched shapes: {:?} and {:?}", - pred.shape(), - target.shape() - )); - } - - let n = pred.len() as f64; - let mut res = (target - pred).mapv(|x| x.powi(2)).sum(); - res /= n; - Ok(res) + pub fn mse(pred: &Array, target: &Array) -> Option + where + T: Float + FromPrimitive + ScalarOperand + ops::DivAssign, + D: Dimension, + { + (target - pred).mapv(|x| x.powi(2)).mean() } } diff --git a/ml/neural/src/prop/mod.rs b/ml/neural/src/prop/mod.rs index 14826613..679c11b6 100644 --- a/ml/neural/src/prop/mod.rs +++ b/ml/neural/src/prop/mod.rs @@ -13,9 +13,10 @@ pub(crate) mod propagation; // pub mod forward; pub trait Backward { + type Params; type Output; - fn backward(&mut self, args: &T) -> Self::Output; + fn backward(&mut self, args: &T, params: &Self::Params) -> Self::Output; } pub trait Forward { diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index bfa647d8..c56d3618 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -4,22 +4,51 @@ */ use ndarray::prelude::{Array, Array2}; use ndarray::{Dimension, IntoDimension}; -use ndarray_rand::rand_distr as dist; use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Uniform}; use ndarray_rand::RandomExt; use num::Float; -pub trait InitUniform +pub trait GenerateRandom where T: Float + SampleUniform, { type Dim: Dimension; + fn bernoulli( + dim: impl IntoDimension, + p: Option, + ) -> Result, BernoulliError> { + let dist = Bernoulli::new(p.unwrap_or(0.5))?; + Ok(Array::random(dim.into_dimension(), dist)) + } + + fn uniform(axis: usize, dim: impl IntoDimension) -> Array { + let dim = dim.into_dimension(); + let k = (T::from(dim[axis]).unwrap()).sqrt(); + Array::random(dim, Uniform::new(-k, k)) + } +} + +impl GenerateRandom for Array +where + T: Float + SampleUniform, + D: Dimension, +{ + type Dim = D; + + fn bernoulli( + dim: impl IntoDimension, + p: Option, + ) -> Result, BernoulliError> { + let dist = Bernoulli::new(p.unwrap_or(0.5))?; + Ok(Array::random(dim.into_dimension(), dist)) + } + fn uniform(axis: usize, dim: impl IntoDimension) -> Array { let dim = dim.into_dimension(); let k = (T::from(dim[axis]).unwrap()).sqrt(); - let uniform = dist::Uniform::new(-k, k); - Array::random(dim, uniform) + Array::random(dim, Uniform::new(-k, k)) } } diff --git a/ml/nlp/Cargo.toml b/ml/nlp/Cargo.toml index d705fd8e..0955eea6 100644 --- a/ml/nlp/Cargo.toml +++ b/ml/nlp/Cargo.toml @@ -32,8 +32,10 @@ serde.workspace = true serde_json.workspace = true smart-default.workspace = true strum.workspace = true +tokenizers = { features = [], version = "0.14" } [dev-dependencies] +computare.workspace = true [package.metadata.docs.rs] all-features = true diff --git a/ml/nlp/src/encode/positional.rs b/ml/nlp/src/encode/positional.rs index 4d573bf0..9f343cb6 100644 --- a/ml/nlp/src/encode/positional.rs +++ b/ml/nlp/src/encode/positional.rs @@ -2,28 +2,92 @@ Appellation: positional Contrib: FL03 */ +use ndarray::prelude::Array2; +use ndarray::ScalarOperand; +use num::Float; +use serde::{Deserialize, Serialize}; -use ndarray::Array2; - -pub fn get_position_encoding(seq_len: usize, d: usize, n: f64) -> Array2 { - let denom = |i: usize| f64::powf(n, 2.0 * (i as f64) / d as f64); - let mut p = Array2::zeros((seq_len, d)); - for k in 0..seq_len { - for i in 0..d / 2 { - p[[k, 2 * i]] = (k as f64 / denom(i)).sin(); - p[[k, 2 * i + 1]] = (k as f64 / denom(i)).cos(); +pub fn create_positional( + model: usize, + seq_len: usize, + samples: Option, +) -> Array2 { + let n = T::from(samples.unwrap_or(10000)).unwrap(); + let d = T::from(model).unwrap(); + let denom = |pos: T, x: T| pos / T::powf(n, (T::from(2).unwrap() * x) / d); + let mut p = Array2::zeros((seq_len, model)); + for i in 0..seq_len { + for j in 0..model / 2 { + let u = T::from(i).unwrap(); + let v = T::from(j).unwrap(); + p[[i, 2 * j]] = denom(u, v).sin(); + p[[i, 2 * j + 1]] = denom(u, v + T::one()).cos(); } } p } -pub struct PositionalEncoder { - model: usize, - sequence: usize, - samples: usize, +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)] +pub struct PositionalEncoder { + params: PositionalEncoderParams, + pe: Array2, +} + +impl PositionalEncoder +where + T: Float, +{ + pub fn new(model: usize, sequence: usize, samples: usize) -> Self { + Self { + params: PositionalEncoderParams::new(model, sequence, samples), + pe: Array2::zeros((sequence, model)), + } + } + + pub fn init(mut self) -> Self { + self.pe = create_positional::( + self.params.model(), + self.params.sequence(), + Some(self.params.samples()), + ); + self + } + + pub fn params(&self) -> PositionalEncoderParams { + self.params + } + + pub fn positional(&self) -> &Array2 { + &self.pe + } +} + +impl PositionalEncoder +where + T: Float + ScalarOperand, +{ + pub fn encode(&self, data: &Array2) -> Array2 { + let x = data * T::from(self.params().model()).unwrap().sqrt(); + x + self.positional() + } +} + +pub trait IntoParams { + type Params; + + fn into_params(self) -> Self::Params; } -impl PositionalEncoder { +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] +pub struct PositionalEncoderParams { + pub model: usize, + pub sequence: usize, + pub samples: usize, +} + +impl PositionalEncoderParams { pub fn new(model: usize, sequence: usize, samples: usize) -> Self { Self { model, @@ -32,24 +96,69 @@ impl PositionalEncoder { } } - pub fn encode(&self, data: &Array2) -> Array2 { - let x = data * (self.model as f64).sqrt(); - x + self.positional() + pub fn std(sequence: usize) -> Self { + Self::new(512, sequence, 10000) + } + + pub fn model(&self) -> usize { + self.model + } + + pub fn sequence(&self) -> usize { + self.sequence + } + + pub fn samples(&self) -> usize { + self.samples + } + + pub fn set_model(&mut self, model: usize) { + self.model = model; } - pub fn positional(&self) -> Array2 { - get_position_encoding(self.sequence, self.model, self.samples as f64) + pub fn set_sequence(&mut self, sequence: usize) { + self.sequence = sequence; + } + + pub fn set_samples(&mut self, samples: usize) { + self.samples = samples; + } + + pub fn with_model(mut self, model: usize) -> Self { + self.model = model; + self + } + + pub fn with_sequence(mut self, sequence: usize) -> Self { + self.sequence = sequence; + self + } + + pub fn with_samples(mut self, samples: usize) -> Self { + self.samples = samples; + self } } #[cfg(test)] mod tests { use super::*; - use ndarray::array; + use computare::prelude::RoundTo; + use ndarray::prelude::{array, Array}; #[test] fn test_positional_encoding() { - let p = get_position_encoding(4, 4, 10000.); - assert_eq!(p.row(0), array![0.0, 1.0, 0.0, 1.0]); + let data = Array::linspace(1., 4., 4).into_shape((1, 4)).unwrap(); + let encoder = PositionalEncoder::new(4, 4, 10000).init(); + + let pe = encoder.positional(); + assert_eq!(pe.dim(), (4, 4)); + assert_eq!(pe.row(0), array![0.0, 1.0, 0.0, 1.0]); + + let encoded = encoder.encode(&data); + let rounded = encoded.mapv(|x| x.round_to(4)); + + assert_eq!(rounded[[0, 0]], 2.0); + assert_eq!(rounded[[1, 0]], 2.8415); } } diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index 07c6d664..6a10204d 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -7,6 +7,7 @@ use super::{Head, Weight}; use crate::neural::neurons::activate::{Activate, Softmax}; use ndarray::prelude::Array2; use ndarray::ScalarOperand; +use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; use std::ops; @@ -20,14 +21,6 @@ pub struct AttentionHead { } impl AttentionHead { - pub fn new(dim: HeadShape) -> Self { - Self { - dim, - mask: Array2::zeros((dim.sequence(), dim.sequence())), - weights: Weight::new(dim), - } - } - pub fn dim(&self) -> HeadShape { self.dim } @@ -54,6 +47,19 @@ impl AttentionHead { } } +impl AttentionHead +where + T: Float + SampleUniform, +{ + pub fn new(dim: HeadShape) -> Self { + Self { + dim, + mask: Array2::zeros((dim.sequence(), dim.sequence())), + weights: Weight::uniform(dim), + } + } +} + impl AttentionHead { pub fn attention(&mut self, data: &Array2) -> Array2 { // multiply the data by the wieghted query, key, and value matrices, respectively diff --git a/ml/transformers/src/attention/multi/attention.rs b/ml/transformers/src/attention/multi/attention.rs index 23109b95..fbe4323e 100644 --- a/ml/transformers/src/attention/multi/attention.rs +++ b/ml/transformers/src/attention/multi/attention.rs @@ -24,6 +24,10 @@ impl MultiHeadAttention where T: Float, { + pub fn linear(&self) -> &LinearLayer { + &self.linear + } + pub fn params(&self) -> MultiHeadParams { self.params } @@ -39,7 +43,7 @@ where { pub fn new(heads: usize, model: usize) -> Self { let params = MultiHeadParams::new(heads, model); - let weights = Weight::new((model, model)); + let weights = Weight::uniform((model, model)); Self { linear: LinearLayer::new(model, model), params, @@ -56,7 +60,7 @@ where let weighted = data * self.weights(); let (q, k, v) = weighted.split(self.params().heads())?; let score = multihead(&q, &k, &v, mask)?; - let res = self.linear.linear(&score); + let res = self.linear().linear(&score); Ok(res) } } diff --git a/ml/transformers/src/attention/weights.rs b/ml/transformers/src/attention/weights.rs index 4418ec3c..ba67a5f5 100644 --- a/ml/transformers/src/attention/weights.rs +++ b/ml/transformers/src/attention/weights.rs @@ -2,12 +2,36 @@ Appellation: weights Contrib: FL03 */ +//! # Weights +//! +//! ## Overview +//! +//! The `weights` module provides a `Weight` struct that is used to +//! group the `key`, `query`, and `value` matrices leveraged by the +//! attention mechanism. +//! +//! ## Dimensionality +//! +//! Each of the `key`, `query`, and `value` weight tensors are +//! initialized as square matrices (model, model) +//! +//! - W(model, model) +//! - Q/K/V(seq, model) * W(model, model) = (seq, model) +//! - Split(Q/K/V) = (heads, seq, model/heads) = (heads, seq, query) +//! - Q(seq, model) * Key(seq, model)^T = (seq, seq) +//! - (seq, seq) + Mask(seq, seq) = (seq, seq) +//! - (seq, seq) * V(seq, model) = (seq, model) +//! +//! +//! use super::params::QKV; use super::Weights; +use crate::neural::GenerateRandom; use crate::ops::Split; - -use ndarray::prelude::{Array, Array2, Array3}; -use ndarray::{IntoDimension, Ix2}; +use ndarray::linalg::Dot; +use ndarray::prelude::{Array, Array2, Array3, Ix2}; +use ndarray::IntoDimension; +use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; use std::ops; @@ -15,6 +39,11 @@ use strum::IntoEnumIterator; pub type WeightTensor = Array; // (seq, model) +pub enum AttentionTensor { + Embedding(Array2), + Multihead(Array3), +} + #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] pub struct Weight where @@ -27,9 +56,22 @@ where } impl Weight { + pub fn dim(&self) -> Ix2 { + self.dim + } + + pub fn qkv(&self) -> (Array2, Array2, Array2) { + self.clone().into() + } +} + +impl Weight +where + T: Default + Float, +{ pub fn new(dim: impl IntoDimension) -> Self { let dim = dim.into_dimension(); - let arr = Array2::ones(dim); + let arr = Array2::default(dim); Self { dim, key: arr.clone(), @@ -37,13 +79,26 @@ impl Weight { value: arr, } } +} - pub fn dim(&self) -> Ix2 { - self.dim +impl Weight +where + T: Float + SampleUniform, +{ + pub fn uniform(dim: impl IntoDimension) -> Self { + let dim = dim.into_dimension(); + Self { + dim: dim.clone(), + key: Array2::uniform(1, dim.clone()), + query: Array2::uniform(1, dim.clone()), + value: Array2::uniform(1, dim), + } } - - pub fn qkv(&self) -> (Array2, Array2, Array2) { - self.clone().into() + pub fn init_uniform(mut self) -> Self { + self.key = Array2::uniform(1, self.dim); + self.query = Array2::uniform(1, self.dim); + self.value = Array2::uniform(1, self.dim); + self } } @@ -99,6 +154,18 @@ impl From> for (Array2, Array2, Array2) { } } +impl Dot> for Weight { + type Output = Self; + + fn dot(&self, rhs: &Array2) -> Self::Output { + let mut ctx = self.clone(); + for qkv in QKV::iter() { + ctx[qkv] = ctx[qkv].dot(rhs); + } + ctx + } +} + impl ops::Index for Weight { type Output = Array2; @@ -123,18 +190,6 @@ impl ops::IndexMut for Weight { } } -impl ndarray::linalg::Dot> for Weight { - type Output = Self; - - fn dot(&self, rhs: &Array2) -> Self::Output { - let mut ctx = self.clone(); - for qkv in QKV::iter() { - ctx[qkv] = ctx[qkv].dot(rhs); - } - ctx - } -} - impl ops::Mul> for Array2 { type Output = Weight; diff --git a/optim/Cargo.toml b/optim/Cargo.toml new file mode 100644 index 00000000..2b1d9d78 --- /dev/null +++ b/optim/Cargo.toml @@ -0,0 +1,50 @@ +[package] +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "concision-optim" +readme.workspace = true +repository.workspace = true +version.workspace = true + +[features] +default = [] + + +[lib] +bench = true +crate-type = ["cdylib", "rlib"] +doctest = false +test = true + +[build-dependencies] + +[dependencies] +concision-core.workspace = true +concision-neural = { path = "../ml/neural" } + +anyhow.workspace = true +lazy_static.workspace = true +ndarray.workspace = true +# ndarray-linalg.workspace = true +ndarray-rand.workspace = true +ndarray-stats.workspace = true +num.workspace = true +serde.workspace = true +serde_json.workspace = true +smart-default.workspace = true +strum.workspace = true + +[dev-dependencies] + +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] + +[target.wasm32-unknown-unknown] + +[target.wasm32-wasi] diff --git a/optim/benches/default.rs b/optim/benches/default.rs new file mode 100644 index 00000000..937f2387 --- /dev/null +++ b/optim/benches/default.rs @@ -0,0 +1,52 @@ +// bench.rs +#![feature(test)] + +extern crate test; + +use std::mem::replace; +use test::Bencher; + +// bench: find the `BENCH_SIZE` first terms of the fibonacci sequence +static BENCH_SIZE: usize = 20; + +// recursive fibonacci +fn fibonacci(n: usize) -> u32 { + if n < 2 { + 1 + } else { + fibonacci(n - 1) + fibonacci(n - 2) + } +} + +// iterative fibonacci +struct Fibonacci { + curr: u32, + next: u32, +} + +impl Iterator for Fibonacci { + type Item = u32; + fn next(&mut self) -> Option { + let new_next = self.curr + self.next; + let new_curr = replace(&mut self.next, new_next); + + Some(replace(&mut self.curr, new_curr)) + } +} + +fn fibonacci_sequence() -> Fibonacci { + Fibonacci { curr: 1, next: 1 } +} + +// function to benchmark must be annotated with `#[bench]` +#[bench] +fn recursive_fibonacci(b: &mut Bencher) { + // exact code to benchmark must be passed as a closure to the iter + // method of Bencher + b.iter(|| (0..BENCH_SIZE).map(fibonacci).collect::>()) +} + +#[bench] +fn iterative_fibonacci(b: &mut Bencher) { + b.iter(|| fibonacci_sequence().take(BENCH_SIZE).collect::>()) +} diff --git a/optim/src/lib.rs b/optim/src/lib.rs new file mode 100644 index 00000000..3ea9c8f2 --- /dev/null +++ b/optim/src/lib.rs @@ -0,0 +1,20 @@ +/* + Appellation: optim + Contrib: FL03 +*/ +//! # Concision Optim +pub use self::{primitives::*, specs::*, utils::*}; + +pub(crate) mod primitives; +pub(crate) mod specs; +pub(crate) mod utils; + +pub(crate) use concision_core as core; +pub(crate) use concision_neural as neural; + +pub mod prelude { + + pub use crate::primitives::*; + pub use crate::specs::*; + pub use crate::utils::*; +} diff --git a/optim/src/primitives.rs b/optim/src/primitives.rs new file mode 100644 index 00000000..859023bb --- /dev/null +++ b/optim/src/primitives.rs @@ -0,0 +1,11 @@ +/* + Appellation: primitives + Contrib: FL03 +*/ +pub use self::{constants::*, statics::*, types::*}; + +mod constants {} + +mod statics {} + +mod types {} diff --git a/optim/src/specs.rs b/optim/src/specs.rs new file mode 100644 index 00000000..1d8faa71 --- /dev/null +++ b/optim/src/specs.rs @@ -0,0 +1,4 @@ +/* + Appellation: specs + Contrib: FL03 +*/ diff --git a/optim/src/utils.rs b/optim/src/utils.rs new file mode 100644 index 00000000..752dabaf --- /dev/null +++ b/optim/src/utils.rs @@ -0,0 +1,4 @@ +/* + Appellation: utils + Contrib: FL03 +*/ diff --git a/optim/tests/default.rs b/optim/tests/default.rs new file mode 100644 index 00000000..0cac1eb5 --- /dev/null +++ b/optim/tests/default.rs @@ -0,0 +1,8 @@ +#[cfg(test)] +#[test] +fn compiles() { + let f = |x: usize, y: usize| x + y; + + assert_eq!(f(10, 10), 20); + assert_ne!(f(1, 1), 3); +} From d5b067405d61b30b249c4e7ffa6ed3450a1e15d9 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 5 Nov 2023 13:20:52 -0600 Subject: [PATCH 041/118] update Signed-off-by: FL03 --- ml/neural/src/grad/sgd.rs | 6 +++++- ml/neural/src/nn/loss/mod.rs | 16 +++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/ml/neural/src/grad/sgd.rs b/ml/neural/src/grad/sgd.rs index bc1c6205..7c471359 100644 --- a/ml/neural/src/grad/sgd.rs +++ b/ml/neural/src/grad/sgd.rs @@ -19,9 +19,10 @@ fn sgd( learning_rate: f64, epochs: usize, batch_size: usize, -) { +) -> Array1 { let n_samples = x.shape()[0]; let input_size = x.shape()[1]; + let mut losses = Array1::::zeros(epochs); for epoch in 0..epochs { let mut rng = rand::thread_rng(); @@ -45,9 +46,12 @@ fn sgd( gradient /= batch_size as f64; model.update_with_gradient(&gradient, learning_rate); } + let loss = mse(&model.forward(x), y).unwrap(); + losses[epoch] = loss; // println!("Epoch {}: Loss = {}", epoch, mse(&model.forward(x), y).unwrap()); } + losses } pub struct StochasticGradientDescent diff --git a/ml/neural/src/nn/loss/mod.rs b/ml/neural/src/nn/loss/mod.rs index 832f0c3f..994762b1 100644 --- a/ml/neural/src/nn/loss/mod.rs +++ b/ml/neural/src/nn/loss/mod.rs @@ -19,24 +19,26 @@ pub trait Loss { } pub(crate) mod utils { - use ndarray::prelude::Array; + use ndarray::prelude::{Array, Array1}; use ndarray::{Dimension, ScalarOperand}; use num::{Float, FromPrimitive}; use std::ops; - pub fn mae(pred: &Array, target: &Array) -> Option + pub fn mae<'a, T, D>(pred: &Array, target: &Array1) -> Option where - T: Float + FromPrimitive + ScalarOperand + ops::DivAssign, + T: Float + FromPrimitive + ScalarOperand, D: Dimension, + Array1: ops::Sub, Output = Array>, { - (target - pred).mapv(|x| x.abs()).mean() + (target.clone() - pred.clone()).mapv(|x| x.abs()).mean() } - pub fn mse(pred: &Array, target: &Array) -> Option + pub fn mse(pred: &Array, target: &Array1) -> Option where - T: Float + FromPrimitive + ScalarOperand + ops::DivAssign, + T: Float + FromPrimitive + ScalarOperand, D: Dimension, + Array1: ops::Sub, Output = Array>, { - (target - pred).mapv(|x| x.powi(2)).mean() + (target.clone() - pred.clone()).mapv(|x| x.powi(2)).mean() } } From 481f040f067fd8d492fca82bfd92ab1ae9e77685 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 5 Nov 2023 13:27:30 -0600 Subject: [PATCH 042/118] update Signed-off-by: FL03 --- ml/neural/src/grad/sgd.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ml/neural/src/grad/sgd.rs b/ml/neural/src/grad/sgd.rs index 7c471359..a26d1f14 100644 --- a/ml/neural/src/grad/sgd.rs +++ b/ml/neural/src/grad/sgd.rs @@ -92,18 +92,21 @@ where #[cfg(test)] mod tests { use super::*; + use crate::bias::Bias; + // use crate::weights::Weight; + use ndarray::Array; #[test] fn test_sgd() { // Generate some example data - let x = Array2::from_shape_vec((100, 2), (0..200).map(|x| x as f64).collect()).unwrap(); - let y = x.dot(&Array1::from_elem(2, 2.0)) + &Array1::from_elem(100, 1.0); + let x = Array::linspace(1., 200., 200).into_shape((100, 2)).unwrap(); + let y = x.dot(&x) + &Bias::biased(100); - let mut model = LinearLayer::new(200, 100); + let mut model = LinearLayer::::new(200, 100); let learning_rate = 0.01; let epochs = 100; let batch_size = 10; - sgd(&x, &y, &mut model, learning_rate, epochs, batch_size); + // sgd(&x, &y, &mut model, learning_rate, epochs, batch_size); } } From 033380f084548cc844f69b732bf05d7f4a3ccdae Mon Sep 17 00:00:00 2001 From: FL03 Date: Mon, 6 Nov 2023 15:32:15 -0600 Subject: [PATCH 043/118] update Signed-off-by: FL03 --- Cargo.toml | 1 - concision/Cargo.toml | 6 + concision/src/lib.rs | 4 + ml/neural/src/bias/mod.rs | 21 ---- ml/neural/src/cost/kinds.rs | 33 +++++ ml/neural/src/cost/mod.rs | 16 +++ ml/neural/src/grad/descent.rs | 2 +- ml/neural/src/grad/mod.rs | 16 +++ ml/neural/src/grad/sgd.rs | 117 ++++++++++++------ ml/neural/src/layers/layer.rs | 2 +- ml/neural/src/layers/linear/layer.rs | 4 +- ml/neural/src/layers/linear/mod.rs | 3 +- ml/neural/src/lib.rs | 11 +- ml/neural/src/{nn => }/loss/kinds.rs | 0 ml/neural/src/{nn => }/loss/mod.rs | 0 ml/neural/src/{nn => }/loss/regress.rs | 0 ml/neural/src/{bias => masks}/mask.rs | 2 +- ml/neural/src/masks/mod.rs | 13 ++ ml/neural/src/nn/mod.rs | 2 - .../src/{bias/biases.rs => params/bias.rs} | 43 ++++++- ml/neural/src/{weights => params}/mod.rs | 15 ++- ml/neural/src/{weights => params}/weight.rs | 0 {optim => ml/optim}/Cargo.toml | 2 +- {optim => ml/optim}/benches/default.rs | 0 {optim => ml/optim}/src/lib.rs | 0 {optim => ml/optim}/src/primitives.rs | 0 ml/optim/src/specs.rs | 10 ++ {optim => ml/optim}/src/utils.rs | 0 {optim => ml/optim}/tests/default.rs | 0 optim/src/specs.rs | 4 - 30 files changed, 246 insertions(+), 81 deletions(-) delete mode 100644 ml/neural/src/bias/mod.rs create mode 100644 ml/neural/src/cost/kinds.rs create mode 100644 ml/neural/src/cost/mod.rs rename ml/neural/src/{nn => }/loss/kinds.rs (100%) rename ml/neural/src/{nn => }/loss/mod.rs (100%) rename ml/neural/src/{nn => }/loss/regress.rs (100%) rename ml/neural/src/{bias => masks}/mask.rs (99%) create mode 100644 ml/neural/src/masks/mod.rs rename ml/neural/src/{bias/biases.rs => params/bias.rs} (75%) rename ml/neural/src/{weights => params}/mod.rs (68%) rename ml/neural/src/{weights => params}/weight.rs (100%) rename {optim => ml/optim}/Cargo.toml (95%) rename {optim => ml/optim}/benches/default.rs (100%) rename {optim => ml/optim}/src/lib.rs (100%) rename {optim => ml/optim}/src/primitives.rs (100%) create mode 100644 ml/optim/src/specs.rs rename {optim => ml/optim}/src/utils.rs (100%) rename {optim => ml/optim}/tests/default.rs (100%) delete mode 100644 optim/src/specs.rs diff --git a/Cargo.toml b/Cargo.toml index c741f9ef..3e638501 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,6 @@ members = [ "data", "derive", "macros", - "optim", "ml/*", ] diff --git a/concision/Cargo.toml b/concision/Cargo.toml index ca65a13c..9055457a 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -45,6 +45,7 @@ macros = [ ml = [ "neural", "nlp", + "optim", "transformers", ] @@ -56,6 +57,10 @@ nlp = [ "concision-nlp" ] +optim = [ + "concision-optim", +] + transformers = [ "concision-transformers" ] @@ -76,6 +81,7 @@ concision-macros = { features = [], optional = true, path = "../macros", version concision-neural = { features = [], optional = true, path = "../ml/neural", version = "0.1.12" } concision-nlp = { features = [], optional = true, path = "../ml/nlp", version = "0.1.12" } +concision-optim = { features = [], optional = true, path = "../ml/optim", version = "0.1.12" } concision-transformers = { features = [], optional = true, path = "../ml/transformers", version = "0.1.12" } [dev-dependencies] diff --git a/concision/src/lib.rs b/concision/src/lib.rs index f5113520..070877cd 100644 --- a/concision/src/lib.rs +++ b/concision/src/lib.rs @@ -19,6 +19,8 @@ pub use concision_macros::*; pub use concision_neural as neural; #[cfg(feature = "nlp")] pub use concision_nlp as nlp; +#[cfg(feature = "optim")] +pub use concision_optim as optim; #[cfg(feature = "transformers")] pub use concision_transformers as transformers; @@ -35,6 +37,8 @@ pub mod prelude { pub use concision_neural::prelude::*; #[cfg(feature = "nlp")] pub use concision_nlp::prelude::*; + #[cfg(feature = "optim")] + pub use concision_optim::prelude::*; #[cfg(feature = "transformers")] pub use concision_transformers::prelude::*; } diff --git a/ml/neural/src/bias/mod.rs b/ml/neural/src/bias/mod.rs deleted file mode 100644 index 192adc34..00000000 --- a/ml/neural/src/bias/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -/* - Appellation: bias - Contrib: FL03 -*/ -//! # Bias -pub use self::{biases::*, mask::*, utils::*}; - -pub(crate) mod biases; -pub(crate) mod mask; - -use num::Float; - -pub trait Biased { - fn bias(&self) -> &Bias; - fn bias_mut(&mut self) -> &mut Bias; -} - -pub(crate) mod utils {} - -#[cfg(test)] -mod tests {} diff --git a/ml/neural/src/cost/kinds.rs b/ml/neural/src/cost/kinds.rs new file mode 100644 index 00000000..b815343e --- /dev/null +++ b/ml/neural/src/cost/kinds.rs @@ -0,0 +1,33 @@ +/* + Appellation: kinds + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum Costs { + BinaryCrossEntropy, + #[default] + None, +} diff --git a/ml/neural/src/cost/mod.rs b/ml/neural/src/cost/mod.rs new file mode 100644 index 00000000..8ea8a7ec --- /dev/null +++ b/ml/neural/src/cost/mod.rs @@ -0,0 +1,16 @@ +/* + Appellation: cost + Contrib: FL03 +*/ +//! # cost +//! +pub use self::{kinds::*, utils::*}; + +pub(crate) mod kinds; + +pub trait Cost {} + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests {} diff --git a/ml/neural/src/grad/descent.rs b/ml/neural/src/grad/descent.rs index 3acfdb15..ddc727d6 100644 --- a/ml/neural/src/grad/descent.rs +++ b/ml/neural/src/grad/descent.rs @@ -4,7 +4,7 @@ */ use crate::layers::linear::LinearLayer; -use crate::nn::loss::mse; +use crate::loss::mse; use crate::prelude::Forward; use ndarray::prelude::{s, Array1, Array2}; diff --git a/ml/neural/src/grad/mod.rs b/ml/neural/src/grad/mod.rs index 22709354..6ddc85ca 100644 --- a/ml/neural/src/grad/mod.rs +++ b/ml/neural/src/grad/mod.rs @@ -13,6 +13,22 @@ pub trait Descent { fn descent(&self, params: &[f64], grads: &[f64]) -> Vec; } +pub trait LearningRate { + fn gamma(&self) -> f64; +} + +pub trait Momentum { + fn mu(&self) -> f64; + + fn nestrov(&self) -> bool; +} + +pub struct DescentParams { + pub batch_size: usize, + pub epochs: usize, + pub gamma: f64, // learning rate +} + pub(crate) mod utils {} #[cfg(test)] diff --git a/ml/neural/src/grad/sgd.rs b/ml/neural/src/grad/sgd.rs index a26d1f14..b0c1f2e3 100644 --- a/ml/neural/src/grad/sgd.rs +++ b/ml/neural/src/grad/sgd.rs @@ -6,13 +6,15 @@ //! //! use crate::layers::linear::LinearLayer; -use crate::nn::loss::mse; +use crate::loss::mse; use crate::prelude::Forward; use ndarray::prelude::{s, Array1, Array2}; -use num::Float; +use ndarray::ScalarOperand; +use num::{Float, FromPrimitive}; use rand::seq::SliceRandom; +use std::ops::DivAssign; -fn sgd( +pub fn sgd( x: &Array2, y: &Array1, model: &mut LinearLayer, @@ -20,27 +22,22 @@ fn sgd( epochs: usize, batch_size: usize, ) -> Array1 { - let n_samples = x.shape()[0]; - let input_size = x.shape()[1]; + let (samples, _inputs) = x.dim(); + let mut indices: Vec = (0..samples).collect(); let mut losses = Array1::::zeros(epochs); for epoch in 0..epochs { - let mut rng = rand::thread_rng(); - let mut indices: Vec = (0..n_samples).collect(); - indices.shuffle(&mut rng); - - for batch_start in (0..n_samples).step_by(batch_size) { - let batch_end = (batch_start + batch_size).min(n_samples); + indices.shuffle(&mut rand::thread_rng()); + for batch_start in (0..samples).step_by(batch_size) { + let batch_end = (batch_start + batch_size).min(samples); let mut gradient = Array2::zeros(x.dim()); for i in batch_start..batch_end { - let sample_index = indices[i]; - let input = x.slice(s![sample_index, ..]).to_owned(); + let idx = indices[i]; + let input = x.slice(s![idx, ..]).to_owned(); let prediction = model.forward(&input); - let error = prediction - y[sample_index]; - gradient - .slice_mut(s![sample_index, ..]) - .assign(&(input * error)); + let error = prediction - y[idx]; + gradient.slice_mut(s![idx, ..]).assign(&(input * error)); } gradient /= batch_size as f64; @@ -49,7 +46,7 @@ fn sgd( let loss = mse(&model.forward(x), y).unwrap(); losses[epoch] = loss; - // println!("Epoch {}: Loss = {}", epoch, mse(&model.forward(x), y).unwrap()); + println!("Epoch {}: Loss = {}", epoch, loss); } losses } @@ -60,7 +57,7 @@ where { batch_size: usize, epochs: usize, - learning_rate: f64, + gamma: T, // learning rate model: LinearLayer, } @@ -68,44 +65,92 @@ impl StochasticGradientDescent where T: Float, { - pub fn new( - model: LinearLayer, - learning_rate: f64, - epochs: usize, - batch_size: usize, - ) -> Self { + pub fn new(batch_size: usize, epochs: usize, gamma: T, model: LinearLayer) -> Self { Self { batch_size, epochs, - learning_rate, + gamma, model, } } - pub fn train(&mut self, x: &Array2, y: &Array1) {} + pub fn batch_size(&self) -> usize { + self.batch_size + } + + pub fn epochs(&self) -> usize { + self.epochs + } + + pub fn gamma(&self) -> T { + self.gamma + } pub fn model(&self) -> &LinearLayer { &self.model } } +impl StochasticGradientDescent +where + T: DivAssign + Float + FromPrimitive + ScalarOperand + std::fmt::Debug, +{ + pub fn sgd(&mut self, x: &Array2, y: &Array1) -> Array1 + where + T: std::ops::DivAssign, + { + let (samples, _inputs) = x.dim(); + let mut indices: Vec = (0..samples).collect(); + let mut losses = Array1::::zeros(self.epochs); + + for epoch in 0..self.epochs { + indices.shuffle(&mut rand::thread_rng()); + for batch_start in (0..samples).step_by(self.batch_size) { + let batch_end = (batch_start + self.batch_size).min(samples); + let mut gradient = Array2::zeros(x.dim()); + + for i in batch_start..batch_end { + let idx = indices[i]; + let input = x.slice(s![idx, ..]).to_owned(); + let prediction = self.model.forward(&input); + let error = prediction - y[idx]; + gradient.slice_mut(s![idx, ..]).assign(&(input * error)); + } + + gradient /= T::from(self.batch_size).unwrap(); + self.model.update_with_gradient(&gradient, self.gamma); + + println!("Gadient:\n{:?}", &gradient); + } + let loss = mse(&self.model.forward(x), y).unwrap(); + losses[epoch] = loss; + + println!("Epoch {}: Loss = {:?}", epoch, loss); + } + losses + } +} + #[cfg(test)] mod tests { use super::*; - use crate::bias::Bias; - // use crate::weights::Weight; - use ndarray::Array; + use crate::GenerateRandom; + use ndarray::prelude::{Array, Array1}; #[test] fn test_sgd() { + let (samples, inputs) = (100, 2); + let shape = (samples, inputs); + + let (batch_size, epochs, gamma) = (10, 2, 0.01); // Generate some example data - let x = Array::linspace(1., 200., 200).into_shape((100, 2)).unwrap(); - let y = x.dot(&x) + &Bias::biased(100); + let x = Array::linspace(1., 200., 200).into_shape(shape).unwrap(); + let y = Array1::::uniform(0, 100); + + let model = LinearLayer::::new(inputs, 3); - let mut model = LinearLayer::::new(200, 100); - let learning_rate = 0.01; - let epochs = 100; - let batch_size = 10; + let mut sgd = StochasticGradientDescent::new(batch_size, epochs, gamma, model); + sgd.sgd(&x, &y); // sgd(&x, &y, &mut model, learning_rate, epochs, batch_size); } diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 44611d25..6bf7c2c4 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use super::{Features, LayerType}; -use crate::bias::Bias; +use crate::params::Bias; use crate::prop::Forward; use ndarray::prelude::Array2; diff --git a/ml/neural/src/layers/linear/layer.rs b/ml/neural/src/layers/linear/layer.rs index f1808a60..a433376a 100644 --- a/ml/neural/src/layers/linear/layer.rs +++ b/ml/neural/src/layers/linear/layer.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use crate::layers::Features; -use crate::prelude::{Bias, Forward}; +use crate::prelude::{Bias, Forward, GenerateRandom}; use ndarray::prelude::{Array1, Array2}; use ndarray::ScalarOperand; use ndarray_rand::rand_distr::uniform::SampleUniform; @@ -68,7 +68,7 @@ where { pub fn new(inputs: usize, outputs: usize) -> Self { let params = Features::new(inputs, outputs); - let weights = Array2::ones((inputs, outputs)); + let weights = Array2::uniform(1, (outputs, inputs)); let bias = Bias::biased(outputs); Self { bias, diff --git a/ml/neural/src/layers/linear/mod.rs b/ml/neural/src/layers/linear/mod.rs index 0f96a320..e6cad5d5 100644 --- a/ml/neural/src/layers/linear/mod.rs +++ b/ml/neural/src/layers/linear/mod.rs @@ -7,8 +7,7 @@ pub use self::{layer::*, utils::*}; pub(crate) mod layer; -use crate::bias::Biased; -use crate::weights::Weighted; +use crate::params::{Biased, Weighted}; use ndarray::prelude::Array2; use num::Float; diff --git a/ml/neural/src/lib.rs b/ml/neural/src/lib.rs index 9955aaf3..a5f78cc3 100644 --- a/ml/neural/src/lib.rs +++ b/ml/neural/src/lib.rs @@ -14,26 +14,31 @@ pub(crate) mod specs; pub(crate) mod utils; pub mod arch; -pub mod bias; +pub mod cost; pub mod grad; pub mod layers; +pub mod loss; +pub mod masks; pub mod models; pub mod neurons; pub mod nn; pub mod ops; +pub mod params; pub mod prop; -pub mod weights; // pub(crate) use concision_core as core; pub mod prelude { pub use crate::arch::*; - pub use crate::bias::*; + pub use crate::cost::*; pub use crate::layers::*; + pub use crate::loss::*; + pub use crate::masks::*; pub use crate::neurons::activate::*; pub use crate::neurons::{Neuron, Node, Weight}; pub use crate::nn::*; pub use crate::ops::*; + pub use crate::params::*; pub use crate::prop::*; pub use crate::primitives::*; diff --git a/ml/neural/src/nn/loss/kinds.rs b/ml/neural/src/loss/kinds.rs similarity index 100% rename from ml/neural/src/nn/loss/kinds.rs rename to ml/neural/src/loss/kinds.rs diff --git a/ml/neural/src/nn/loss/mod.rs b/ml/neural/src/loss/mod.rs similarity index 100% rename from ml/neural/src/nn/loss/mod.rs rename to ml/neural/src/loss/mod.rs diff --git a/ml/neural/src/nn/loss/regress.rs b/ml/neural/src/loss/regress.rs similarity index 100% rename from ml/neural/src/nn/loss/regress.rs rename to ml/neural/src/loss/regress.rs diff --git a/ml/neural/src/bias/mask.rs b/ml/neural/src/masks/mask.rs similarity index 99% rename from ml/neural/src/bias/mask.rs rename to ml/neural/src/masks/mask.rs index 00fc7db7..de49bcdd 100644 --- a/ml/neural/src/bias/mask.rs +++ b/ml/neural/src/masks/mask.rs @@ -1,5 +1,5 @@ /* - Appellation: bias + Appellation: mask Contrib: FL03 */ use ndarray::prelude::{Array, Array2}; diff --git a/ml/neural/src/masks/mod.rs b/ml/neural/src/masks/mod.rs new file mode 100644 index 00000000..d196bd78 --- /dev/null +++ b/ml/neural/src/masks/mod.rs @@ -0,0 +1,13 @@ +/* + Appellation: masks + Contrib: FL03 +*/ +//! # Mask +pub use self::{mask::*, utils::*}; + +pub(crate) mod mask; + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests {} diff --git a/ml/neural/src/nn/mod.rs b/ml/neural/src/nn/mod.rs index 5c0285bf..1a3c4ec3 100644 --- a/ml/neural/src/nn/mod.rs +++ b/ml/neural/src/nn/mod.rs @@ -5,8 +5,6 @@ //! # Neural Network pub use self::{network::*, utils::*}; -pub mod loss; - pub(crate) mod network; use crate::layers::Layer; diff --git a/ml/neural/src/bias/biases.rs b/ml/neural/src/params/bias.rs similarity index 75% rename from ml/neural/src/bias/biases.rs rename to ml/neural/src/params/bias.rs index f634e7e5..7f133b7a 100644 --- a/ml/neural/src/bias/biases.rs +++ b/ml/neural/src/params/bias.rs @@ -34,7 +34,7 @@ impl Bias { } } -impl Bias +impl Bias where T: Float + SampleUniform, { @@ -44,8 +44,43 @@ where } } -impl ops::Add> for Bias +impl From> for Bias where + T: Float, +{ + fn from(bias: Array1) -> Self { + Self::Biased(bias) + } +} + +impl From>> for Bias +where + T: Float, +{ + fn from(bias: Option>) -> Self { + match bias { + Some(bias) => Self::Biased(bias), + None => Self::Unbiased, + } + } +} + +impl From> for Option> +where + T: Float, +{ + fn from(bias: Bias) -> Self { + match bias { + Bias::Biased(bias) => Some(bias), + Bias::Unbiased => None, + } + } +} + +impl ops::Add> for Bias +where + D: Dimension, + T: Float, Array: ops::Add, Output = Array>, { type Output = Array; @@ -58,8 +93,10 @@ where } } -impl ops::Add<&Array> for Bias +impl ops::Add<&Array> for Bias where + D: Dimension, + T: Float, Array: ops::Add, Output = Array>, { type Output = Array; diff --git a/ml/neural/src/weights/mod.rs b/ml/neural/src/params/mod.rs similarity index 68% rename from ml/neural/src/weights/mod.rs rename to ml/neural/src/params/mod.rs index 96065754..1fbc96dc 100644 --- a/ml/neural/src/weights/mod.rs +++ b/ml/neural/src/params/mod.rs @@ -1,15 +1,24 @@ /* - Appellation: weights + Appellation: params Contrib: FL03 */ -//! # Weights -pub use self::{utils::*, weight::*}; +//! # Parameters +//! +//! ## Overview +//! +pub use self::{bias::*, utils::*, weight::*}; +pub(crate) mod bias; pub(crate) mod weight; use ndarray::prelude::Array2; use num::Float; +pub trait Biased { + fn bias(&self) -> &Bias; + fn bias_mut(&mut self) -> &mut Bias; +} + pub trait Weighted where T: Float, diff --git a/ml/neural/src/weights/weight.rs b/ml/neural/src/params/weight.rs similarity index 100% rename from ml/neural/src/weights/weight.rs rename to ml/neural/src/params/weight.rs diff --git a/optim/Cargo.toml b/ml/optim/Cargo.toml similarity index 95% rename from optim/Cargo.toml rename to ml/optim/Cargo.toml index 2b1d9d78..9d61dda2 100644 --- a/optim/Cargo.toml +++ b/ml/optim/Cargo.toml @@ -25,7 +25,7 @@ test = true [dependencies] concision-core.workspace = true -concision-neural = { path = "../ml/neural" } +concision-neural = { path = "../neural" } anyhow.workspace = true lazy_static.workspace = true diff --git a/optim/benches/default.rs b/ml/optim/benches/default.rs similarity index 100% rename from optim/benches/default.rs rename to ml/optim/benches/default.rs diff --git a/optim/src/lib.rs b/ml/optim/src/lib.rs similarity index 100% rename from optim/src/lib.rs rename to ml/optim/src/lib.rs diff --git a/optim/src/primitives.rs b/ml/optim/src/primitives.rs similarity index 100% rename from optim/src/primitives.rs rename to ml/optim/src/primitives.rs diff --git a/ml/optim/src/specs.rs b/ml/optim/src/specs.rs new file mode 100644 index 00000000..d4ebfe67 --- /dev/null +++ b/ml/optim/src/specs.rs @@ -0,0 +1,10 @@ +/* + Appellation: specs + Contrib: FL03 +*/ + +pub trait Optimize { + fn optimize(&self, params: &mut dyn Optimizable); +} + +pub trait Optimizable {} diff --git a/optim/src/utils.rs b/ml/optim/src/utils.rs similarity index 100% rename from optim/src/utils.rs rename to ml/optim/src/utils.rs diff --git a/optim/tests/default.rs b/ml/optim/tests/default.rs similarity index 100% rename from optim/tests/default.rs rename to ml/optim/tests/default.rs diff --git a/optim/src/specs.rs b/optim/src/specs.rs deleted file mode 100644 index 1d8faa71..00000000 --- a/optim/src/specs.rs +++ /dev/null @@ -1,4 +0,0 @@ -/* - Appellation: specs - Contrib: FL03 -*/ From f49510277ddbcb61b032b242b3f2201d4c633202 Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 10 Nov 2023 11:45:59 -0600 Subject: [PATCH 044/118] update Signed-off-by: FL03 --- core/Cargo.toml | 1 + core/src/specs.rs | 51 ++++++++++- ml/neural/src/layers/layer.rs | 3 +- ml/neural/src/layers/linear/layer.rs | 53 ++++++----- ml/neural/src/layers/linear/mod.rs | 3 +- ml/neural/src/layers/mod.rs | 7 ++ ml/neural/src/lib.rs | 7 +- ml/neural/src/loss/mod.rs | 32 ++++++- ml/neural/src/loss/regress.rs | 54 +++++++----- ml/neural/src/models/mod.rs | 4 +- ml/neural/src/neurons/mod.rs | 1 - ml/neural/src/params/bias.rs | 107 ++++++++++++++++++++--- ml/neural/src/params/mod.rs | 4 + ml/neural/src/params/weight.rs | 3 +- ml/neural/src/prop/mod.rs | 6 ++ ml/neural/src/specs.rs | 6 +- ml/optim/Cargo.toml | 1 + ml/optim/examples/sgd.rs | 21 +++++ ml/{neural => optim}/src/cost/kinds.rs | 0 ml/{neural => optim}/src/cost/mod.rs | 0 ml/{neural => optim}/src/grad/descent.rs | 7 +- ml/{neural => optim}/src/grad/mod.rs | 25 +++++- ml/{neural => optim}/src/grad/sgd.rs | 51 ++++++----- ml/optim/src/lib.rs | 4 + ml/optim/src/optimizer/mod.rs | 0 ml/optim/src/primitives.rs | 6 +- ml/transformers/src/attention/weights.rs | 2 +- 27 files changed, 356 insertions(+), 103 deletions(-) create mode 100644 ml/optim/examples/sgd.rs rename ml/{neural => optim}/src/cost/kinds.rs (100%) rename ml/{neural => optim}/src/cost/mod.rs (100%) rename ml/{neural => optim}/src/grad/descent.rs (81%) rename ml/{neural => optim}/src/grad/mod.rs (54%) rename ml/{neural => optim}/src/grad/sgd.rs (76%) create mode 100644 ml/optim/src/optimizer/mod.rs diff --git a/core/Cargo.toml b/core/Cargo.toml index f8bf1980..729c84d5 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -26,6 +26,7 @@ test = true anyhow.workspace = true chrono = "0.4" ndarray.workspace = true +ndarray-rand.workspace = true num.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/core/src/specs.rs b/core/src/specs.rs index 54b1c7d7..cb5c80b5 100644 --- a/core/src/specs.rs +++ b/core/src/specs.rs @@ -3,7 +3,11 @@ Contrib: FL03 */ use ndarray::prelude::Array; -use ndarray::{Dimension, ShapeError}; +use ndarray::{Dimension, IntoDimension, ShapeError}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Uniform}; +use ndarray_rand::RandomExt; +use num::Float; use num::{Num, One, Zero}; use std::ops::MulAssign; @@ -94,6 +98,51 @@ where // } // } + + +pub trait GenerateRandom +where + T: Float + SampleUniform, +{ + type Dim: Dimension; + + fn bernoulli( + dim: impl IntoDimension, + p: Option, + ) -> Result, BernoulliError> { + let dist = Bernoulli::new(p.unwrap_or(0.5))?; + Ok(Array::random(dim.into_dimension(), dist)) + } + + fn uniform(axis: usize, dim: impl IntoDimension) -> Array { + let dim = dim.into_dimension(); + let k = (T::one() / T::from(dim[axis]).unwrap()).sqrt(); + Array::random(dim, Uniform::new(-k, k)) + } +} + +impl GenerateRandom for Array +where + T: Float + SampleUniform, + D: Dimension, +{ + type Dim = D; + + fn bernoulli( + dim: impl IntoDimension, + p: Option, + ) -> Result, BernoulliError> { + let dist = Bernoulli::new(p.unwrap_or(0.5))?; + Ok(Array::random(dim.into_dimension(), dist)) + } + + fn uniform(axis: usize, dim: impl IntoDimension) -> Array { + let dim = dim.into_dimension(); + let k = (T::from(dim[axis]).unwrap()).sqrt(); + Array::random(dim, Uniform::new(-k, k)) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 6bf7c2c4..4bf03f9a 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -6,6 +6,7 @@ use super::{Features, LayerType}; use crate::params::Bias; use crate::prop::Forward; +use ndarray::ScalarOperand; use ndarray::prelude::Array2; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; @@ -76,7 +77,7 @@ where } } -impl Forward> for Layer { +impl Forward> for Layer where T: Float + ScalarOperand + 'static { type Output = Array2; fn forward(&self, data: &Array2) -> Self::Output { diff --git a/ml/neural/src/layers/linear/layer.rs b/ml/neural/src/layers/linear/layer.rs index a433376a..69d2006f 100644 --- a/ml/neural/src/layers/linear/layer.rs +++ b/ml/neural/src/layers/linear/layer.rs @@ -2,24 +2,28 @@ Appellation: layer Contrib: FL03 */ +use crate::core::prelude::GenerateRandom; use crate::layers::Features; -use crate::prelude::{Bias, Forward, GenerateRandom}; -use ndarray::prelude::{Array1, Array2}; -use ndarray::ScalarOperand; +use crate::prelude::{Bias, Forward}; + +use ndarray::linalg::Dot; +use ndarray::prelude::{Array, Array2}; +use ndarray::{Dimension, ScalarOperand}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; +use std::ops::{Add, Mul}; #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] pub struct LinearLayer { bias: Bias, - pub params: Features, + pub features: Features, weights: Array2, } impl LinearLayer where - T: Float, + T: Float + ScalarOperand, { pub fn bias(&self) -> &Bias { &self.bias @@ -29,12 +33,19 @@ where &mut self.bias } - pub fn params(&self) -> &Features { - &self.params + pub fn features(&self) -> &Features { + &self.features + } + + pub fn features_mut(&mut self) -> &mut Features { + &mut self.features } - pub fn params_mut(&mut self) -> &mut Features { - &mut self.params + pub fn fit(&mut self, data: &Array2) -> Array2 + where + T: 'static, + { + self.linear(data) } pub fn linear(&self, data: &Array2) -> Array2 @@ -57,7 +68,7 @@ where } pub fn with_params(mut self, params: Features) -> Self { - self.params = params; + self.features = params; self } } @@ -67,12 +78,12 @@ where T: Float + SampleUniform, { pub fn new(inputs: usize, outputs: usize) -> Self { - let params = Features::new(inputs, outputs); + let features = Features::new(inputs, outputs); let weights = Array2::uniform(1, (outputs, inputs)); let bias = Bias::biased(outputs); Self { bias, - params, + features, weights, } } @@ -87,24 +98,26 @@ where } } -impl Forward> for LinearLayer +impl Forward> for LinearLayer where + D: Dimension, T: Float + ScalarOperand, + Array: Add, Output = Array> + Dot, Output = Array>, { - type Output = Array1; + type Output = Array; - fn forward(&self, data: &Array1) -> Self::Output { - data.dot(&self.weights().t()) + self.bias() + fn forward(&self, data: &Array) -> Self::Output { + data.dot(&self.weights().t().to_owned()) + self.bias().clone() } } -impl Forward> for LinearLayer +impl Forward for LinearLayer where - T: Float + ScalarOperand, + T: Float + ScalarOperand + Mul, { type Output = Array2; - fn forward(&self, data: &Array2) -> Self::Output { - data.dot(&self.weights().t()) + self.bias() + fn forward(&self, data: &T) -> Self::Output { + self.weights().t().to_owned() * data.clone() + self.bias().clone() } } diff --git a/ml/neural/src/layers/linear/mod.rs b/ml/neural/src/layers/linear/mod.rs index e6cad5d5..804adcd2 100644 --- a/ml/neural/src/layers/linear/mod.rs +++ b/ml/neural/src/layers/linear/mod.rs @@ -8,6 +8,7 @@ pub use self::{layer::*, utils::*}; pub(crate) mod layer; use crate::params::{Biased, Weighted}; +use ndarray::ScalarOperand; use ndarray::prelude::Array2; use num::Float; @@ -21,7 +22,7 @@ where impl LinearTransformation for S where S: Biased + Weighted, - T: Float + 'static, + T: Float + ScalarOperand + 'static, { fn linear(&self, data: &Array2) -> Array2 { data.dot(&self.weights().t()) + self.bias() diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index c73c9e0a..9ac19ec5 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -17,6 +17,13 @@ use ndarray::prelude::{Array1, Array2}; use num::Float; pub trait L { + fn forward_slice(&self, args: &Array2, rho: impl Activate) -> Array2 + where + T: 'static, + { + let z = args.dot(self.weights()) + self.bias(); + z.mapv(|x| rho.activate(x)) + } // fn process(&self, args: &Array2, rho: impl Activate) -> Array2 where diff --git a/ml/neural/src/lib.rs b/ml/neural/src/lib.rs index a5f78cc3..67a4476d 100644 --- a/ml/neural/src/lib.rs +++ b/ml/neural/src/lib.rs @@ -14,8 +14,6 @@ pub(crate) mod specs; pub(crate) mod utils; pub mod arch; -pub mod cost; -pub mod grad; pub mod layers; pub mod loss; pub mod masks; @@ -26,16 +24,15 @@ pub mod ops; pub mod params; pub mod prop; -// pub(crate) use concision_core as core; +pub(crate) use concision_core as core; pub mod prelude { pub use crate::arch::*; - pub use crate::cost::*; pub use crate::layers::*; pub use crate::loss::*; pub use crate::masks::*; pub use crate::neurons::activate::*; - pub use crate::neurons::{Neuron, Node, Weight}; + pub use crate::neurons::{Neuron, Node}; pub use crate::nn::*; pub use crate::ops::*; pub use crate::params::*; diff --git a/ml/neural/src/loss/mod.rs b/ml/neural/src/loss/mod.rs index 994762b1..72fee3b9 100644 --- a/ml/neural/src/loss/mod.rs +++ b/ml/neural/src/loss/mod.rs @@ -14,10 +14,36 @@ pub(crate) mod kinds; pub mod regress; -pub trait Loss { - fn loss(&self, pred: &[f64], target: &[f64]) -> f64; +use ndarray::{Dimension, ScalarOperand}; +use ndarray::linalg::Dot; +use ndarray::prelude::{Array, Array1}; +use num::{Float, FromPrimitive}; +use std::ops; + +pub trait Loss { + + fn loss(&self, pred: &Array, target: &Array1) -> T; +} + +pub struct MSE; + +impl MSE { + pub fn partial_slope(data: &Array, bias: &Array1, slope: &Array, target: &Array1) -> Option + where + D: Dimension, + T: Float + FromPrimitive + ScalarOperand, + Array: Dot>, + as Dot>>::Output: ops::Add, Output = Array> + ops::Sub, Output = Array> + ops::Mul>, + Array1: ops::Sub, Output = Array>, + { + let predicted = data.dot(&slope.t().to_owned()) + bias.clone(); + let inner = data.dot(&(target.clone() - predicted)) * (- T::from(2).unwrap()); + inner.mean() + } } + + pub(crate) mod utils { use ndarray::prelude::{Array, Array1}; use ndarray::{Dimension, ScalarOperand}; @@ -41,4 +67,6 @@ pub(crate) mod utils { { (target.clone() - pred.clone()).mapv(|x| x.powi(2)).mean() } + + } diff --git a/ml/neural/src/loss/regress.rs b/ml/neural/src/loss/regress.rs index 00d7f393..63511be6 100644 --- a/ml/neural/src/loss/regress.rs +++ b/ml/neural/src/loss/regress.rs @@ -3,6 +3,10 @@ Contrib: FL03 */ use super::Loss; +use ndarray::{Dimension, ScalarOperand}; +use ndarray::prelude::{Array, Array1}; +use num::Float; +use std::ops; pub enum RegressiveLoss { Huber(HuberLoss), @@ -11,63 +15,65 @@ pub enum RegressiveLoss { Other(String), } -pub struct HuberLoss { - delta: f64, +pub struct HuberLoss { + delta: T, } -impl HuberLoss { - pub fn new(delta: f64) -> Self { +impl HuberLoss where T: Float { + pub fn new(delta: T) -> Self { Self { delta } } - pub fn delta(&self) -> f64 { + pub fn delta(&self) -> T { self.delta } - pub fn set_delta(&mut self, delta: f64) { + pub fn set_delta(&mut self, delta: T) { self.delta = delta; } } -impl Loss for HuberLoss { - fn loss(&self, pred: &[f64], target: &[f64]) -> f64 { - let mut loss = 0.0; - for (x, y) in pred.iter().zip(target.iter()) { +impl Loss for HuberLoss where T: Float + ops::AddAssign { + + fn loss(&self, pred: &Array, target: &Array1) -> T { + let half = T::from(0.5).unwrap(); + let mut loss = T::zero(); + for (x, y) in pred.iter().cloned().zip(target.iter().cloned()) { let diff = x - y; - if diff.abs() <= self.delta { + if diff.abs() <= self.delta() { // If the difference is sufficiently small, use the squared error. - loss += 0.5 * diff.powi(2); + loss += half * diff.powi(2); } else { // Otherwise, use a variant of the absolute error. - loss += self.delta * (diff.abs() - 0.5 * self.delta); + loss += self.delta * (diff.abs() - half * self.delta); } } - loss / pred.len() as f64 + loss / T::from(pred.len()).unwrap() } } pub struct MeanAbsoluteError; -impl Loss for MeanAbsoluteError { - fn loss(&self, pred: &[f64], target: &[f64]) -> f64 { - let mut res = 0.0; - for (p, t) in pred.iter().zip(target.iter()) { +impl Loss for MeanAbsoluteError where T: Float + ops::AddAssign + ops::DivAssign { + fn loss(&self, pred: &Array, target: &Array1) -> T { + let mut res = T::zero(); + for (p, t) in pred.iter().cloned().zip(target.iter().cloned()) { res += (p - t).abs(); } - res /= pred.len() as f64; + res /= T::from(pred.len()).unwrap(); res } } pub struct MeanSquaredError; -impl Loss for MeanSquaredError { - fn loss(&self, pred: &[f64], target: &[f64]) -> f64 { - let mut res = 0.0; - for (p, t) in pred.iter().zip(target.iter()) { +impl Loss for MeanSquaredError where T: Float + ops::AddAssign + ops::DivAssign { + fn loss(&self, pred: &Array, target: &Array1) -> T { + let mut res = T::zero(); + for (p, t) in pred.iter().cloned().zip(target.iter().cloned()) { res += (p - t).powi(2); } - res /= pred.len() as f64; + res /= T::from(pred.len()).unwrap(); res } } diff --git a/ml/neural/src/models/mod.rs b/ml/neural/src/models/mod.rs index 3dde2e9a..d009b3a6 100644 --- a/ml/neural/src/models/mod.rs +++ b/ml/neural/src/models/mod.rs @@ -8,6 +8,8 @@ pub use self::{model::*, utils::*}; pub(crate) mod model; -pub trait Model {} +pub trait Module { + fn add_module(&mut self, module: impl Module); +} pub(crate) mod utils {} diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index 1b8062ef..43d8e5ff 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -10,7 +10,6 @@ pub(crate) mod node; pub mod activate; -pub trait Weight {} pub(crate) mod utils {} diff --git a/ml/neural/src/params/bias.rs b/ml/neural/src/params/bias.rs index 7f133b7a..38bf6a93 100644 --- a/ml/neural/src/params/bias.rs +++ b/ml/neural/src/params/bias.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use ndarray::prelude::{Array, Array1}; -use ndarray::Dimension; +use ndarray::{Dimension, ScalarOperand}; use ndarray_rand::rand_distr::{uniform::SampleUniform, Uniform}; use ndarray_rand::RandomExt; use num::Float; @@ -13,8 +13,9 @@ use std::ops; use strum::EnumIs; fn _generate_bias(size: usize) -> Array1 { - let ds = (T::from(size).unwrap()).sqrt(); - let dist = Uniform::new(-ds, ds); + let k = T::one() / T::from(size).unwrap(); + let dk = k.sqrt(); + let dist = Uniform::new(-dk, dk); Array1::::random(size, dist) } @@ -23,13 +24,15 @@ pub enum Bias { Biased(Array1), #[default] Unbiased, + Value(T), } -impl Bias { +impl Bias where T: Float + ScalarOperand { pub fn forward(&self, data: &Array1) -> Array1 { match self { Self::Biased(bias) => data + bias, Self::Unbiased => data.clone(), + Self::Value(value) => data + value.clone(), } } } @@ -44,6 +47,12 @@ where } } +impl From for Bias where T: Float { + fn from(value: T) -> Self { + Self::Value(value) + } +} + impl From> for Bias where T: Float, @@ -73,14 +82,15 @@ where match bias { Bias::Biased(bias) => Some(bias), Bias::Unbiased => None, + Bias::Value(value) => Some(Array1::::from_elem(1, value)), } } } -impl ops::Add> for Bias +impl ops::Add> for Bias where D: Dimension, - T: Float, + T: Float + ScalarOperand, Array: ops::Add, Output = Array>, { type Output = Array; @@ -89,14 +99,15 @@ where match self { Self::Biased(bias) => rhs + bias, Self::Unbiased => rhs, + Self::Value(value) => &rhs + value, } } } -impl ops::Add<&Array> for Bias +impl ops::Add<&Array> for Bias where D: Dimension, - T: Float, + T: Float + ScalarOperand, Array: ops::Add, Output = Array>, { type Output = Array; @@ -105,12 +116,15 @@ where match self { Self::Biased(bias) => rhs.clone() + bias, Self::Unbiased => rhs.clone(), + Self::Value(value) => rhs + value, } } } -impl ops::Add> for Array +impl ops::Add> for Array where + D: Dimension, + T: Float + ScalarOperand, Array: ops::Add, Output = Array>, { type Output = Array; @@ -119,12 +133,15 @@ where match bias.clone() { Bias::Biased(bias) => self.clone() + bias, Bias::Unbiased => self.clone(), + Bias::Value(value) => &self + value, } } } -impl ops::Add<&Bias> for Array +impl ops::Add<&Bias> for Array where + D: Dimension, + T: Float + ScalarOperand, Array: ops::Add, Output = Array>, { type Output = Array; @@ -133,6 +150,76 @@ where match bias.clone() { Bias::Biased(bias) => self.clone() + bias, Bias::Unbiased => self.clone(), + Bias::Value(value) => &self + value, + } + } +} + + +impl ops::Sub> for Bias +where + D: Dimension, + T: Float + ScalarOperand, + Array: ops::Sub, Output = Array>, +{ + type Output = Array; + + fn sub(self, rhs: Array) -> Self::Output { + match self { + Self::Biased(bias) => rhs - bias, + Self::Unbiased => rhs, + Self::Value(value) => &rhs - value, + } + } +} + +impl ops::Sub<&Array> for Bias +where + D: Dimension, + T: Float + ScalarOperand, + Array: ops::Sub, Output = Array>, +{ + type Output = Array; + + fn sub(self, rhs: &Array) -> Self::Output { + match self { + Self::Biased(bias) => rhs.clone() - bias, + Self::Unbiased => rhs.clone(), + Self::Value(value) => rhs - value, + } + } +} + +impl ops::Sub> for Array +where + D: Dimension, + T: Float + ScalarOperand, + Array: ops::Sub, Output = Array>, +{ + type Output = Array; + + fn sub(self, bias: Bias) -> Self::Output { + match bias.clone() { + Bias::Biased(bias) => self.clone() - bias, + Bias::Unbiased => self.clone(), + Bias::Value(value) => &self - value, + } + } +} + +impl ops::Sub<&Bias> for Array +where + D: Dimension, + T: Float + ScalarOperand, + Array: ops::Sub, Output = Array>, +{ + type Output = Array; + + fn sub(self, bias: &Bias) -> Self::Output { + match bias.clone() { + Bias::Biased(bias) => self.clone() - bias, + Bias::Unbiased => self.clone(), + Bias::Value(value) => &self - value, } } } diff --git a/ml/neural/src/params/mod.rs b/ml/neural/src/params/mod.rs index 1fbc96dc..96f24894 100644 --- a/ml/neural/src/params/mod.rs +++ b/ml/neural/src/params/mod.rs @@ -14,6 +14,10 @@ pub(crate) mod weight; use ndarray::prelude::Array2; use num::Float; +pub trait Parameter { + +} + pub trait Biased { fn bias(&self) -> &Bias; fn bias_mut(&mut self) -> &mut Bias; diff --git a/ml/neural/src/params/weight.rs b/ml/neural/src/params/weight.rs index 0f86f02c..68cd6248 100644 --- a/ml/neural/src/params/weight.rs +++ b/ml/neural/src/params/weight.rs @@ -29,7 +29,8 @@ where T: Float + SampleUniform, { pub fn uniform(m: usize, n: usize) -> Array2 { - let dk = (T::from(m).unwrap()).sqrt(); + let k = T::one() / T::from(m).unwrap(); + let dk = k.sqrt(); let dist = Uniform::new(-dk, dk); Array2::random((m, n), dist) } diff --git a/ml/neural/src/prop/mod.rs b/ml/neural/src/prop/mod.rs index 679c11b6..b40bd16f 100644 --- a/ml/neural/src/prop/mod.rs +++ b/ml/neural/src/prop/mod.rs @@ -25,4 +25,10 @@ pub trait Forward { fn forward(&self, args: &T) -> Self::Output; } +pub trait ForwardIter { + type Output; + + fn forward_iter(&self, args: &T) -> Self::Output; +} + pub(crate) mod utils {} diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index c56d3618..7221e86f 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -9,7 +9,7 @@ use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Uniform}; use ndarray_rand::RandomExt; use num::Float; -pub trait GenerateRandom +pub trait InitRandom where T: Float + SampleUniform, { @@ -25,12 +25,12 @@ where fn uniform(axis: usize, dim: impl IntoDimension) -> Array { let dim = dim.into_dimension(); - let k = (T::from(dim[axis]).unwrap()).sqrt(); + let k = (T::one() / T::from(dim[axis]).unwrap()).sqrt(); Array::random(dim, Uniform::new(-k, k)) } } -impl GenerateRandom for Array +impl InitRandom for Array where T: Float + SampleUniform, D: Dimension, diff --git a/ml/optim/Cargo.toml b/ml/optim/Cargo.toml index 9d61dda2..3275588c 100644 --- a/ml/optim/Cargo.toml +++ b/ml/optim/Cargo.toml @@ -34,6 +34,7 @@ ndarray.workspace = true ndarray-rand.workspace = true ndarray-stats.workspace = true num.workspace = true +rand.workspace = true serde.workspace = true serde_json.workspace = true smart-default.workspace = true diff --git a/ml/optim/examples/sgd.rs b/ml/optim/examples/sgd.rs new file mode 100644 index 00000000..38a8f8ca --- /dev/null +++ b/ml/optim/examples/sgd.rs @@ -0,0 +1,21 @@ +use concision_core::prelude::{GenerateRandom, BoxResult}; +use concision_neural::layers::linear::LinearLayer; +use concision_optim::grad::sgd::StochasticGradientDescent; +use ndarray::prelude::{Array, Array1}; + +fn main() -> BoxResult { + let (samples, inputs) = (20, 5); + let shape = (samples, inputs); + + let (batch_size, epochs, gamma) = (10, 1, 0.01); + // Generate some example data + let x = Array::linspace(1., 100., 100).into_shape(shape).unwrap(); + let y = Array::linspace(1., 100., 100).into_shape(100).unwrap(); + + let model = LinearLayer::::new(inputs, 5); + + let mut sgd = StochasticGradientDescent::new(batch_size, epochs, gamma, model); + let losses = sgd.sgd(&x, &y); + println!("Losses {:?}", losses); + Ok(()) +} \ No newline at end of file diff --git a/ml/neural/src/cost/kinds.rs b/ml/optim/src/cost/kinds.rs similarity index 100% rename from ml/neural/src/cost/kinds.rs rename to ml/optim/src/cost/kinds.rs diff --git a/ml/neural/src/cost/mod.rs b/ml/optim/src/cost/mod.rs similarity index 100% rename from ml/neural/src/cost/mod.rs rename to ml/optim/src/cost/mod.rs diff --git a/ml/neural/src/grad/descent.rs b/ml/optim/src/grad/descent.rs similarity index 81% rename from ml/neural/src/grad/descent.rs rename to ml/optim/src/grad/descent.rs index ddc727d6..6617dbb3 100644 --- a/ml/neural/src/grad/descent.rs +++ b/ml/optim/src/grad/descent.rs @@ -3,10 +3,9 @@ Contrib: FL03 */ -use crate::layers::linear::LinearLayer; -use crate::loss::mse; -use crate::prelude::Forward; -use ndarray::prelude::{s, Array1, Array2}; +use crate::neural::layers::linear::LinearLayer; +use crate::neural::prelude::Forward; +use ndarray::prelude::{Array1, Array2}; fn gradient_descent( x: &Array2, diff --git a/ml/neural/src/grad/mod.rs b/ml/optim/src/grad/mod.rs similarity index 54% rename from ml/neural/src/grad/mod.rs rename to ml/optim/src/grad/mod.rs index 6ddc85ca..2290c367 100644 --- a/ml/neural/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -9,8 +9,12 @@ pub(crate) mod descent; pub mod sgd; -pub trait Descent { - fn descent(&self, params: &[f64], grads: &[f64]) -> Vec; +use num::Float; + +pub trait Descent { + type Params; + + fn descent(&self, ) -> Vec; } pub trait LearningRate { @@ -18,9 +22,24 @@ pub trait LearningRate { } pub trait Momentum { - fn mu(&self) -> f64; + + fn mu(&self) -> f64; // Momentum Rate fn nestrov(&self) -> bool; + + fn tau(&self) -> f64; // Momentum Damper +} + +pub trait Nesterov { + fn nestrov(&self) -> bool; +} + +pub trait Decay { + fn lambda(&self) -> f64; // Decay Rate +} + +pub trait Dampener { + fn tau(&self) -> f64; // Momentum Damper } pub struct DescentParams { diff --git a/ml/neural/src/grad/sgd.rs b/ml/optim/src/grad/sgd.rs similarity index 76% rename from ml/neural/src/grad/sgd.rs rename to ml/optim/src/grad/sgd.rs index b0c1f2e3..31f2e65a 100644 --- a/ml/neural/src/grad/sgd.rs +++ b/ml/optim/src/grad/sgd.rs @@ -5,14 +5,14 @@ //! # Stochastic Gradient Descent (SGD) //! //! -use crate::layers::linear::LinearLayer; -use crate::loss::mse; -use crate::prelude::Forward; +use crate::neural::layers::linear::LinearLayer; +use crate::neural::prelude::{mse, Forward}; +// use crate::prelude::ObjectiveFn; use ndarray::prelude::{s, Array1, Array2}; use ndarray::ScalarOperand; use num::{Float, FromPrimitive}; use rand::seq::SliceRandom; -use std::ops::DivAssign; +use std::ops; pub fn sgd( x: &Array2, @@ -51,6 +51,12 @@ pub fn sgd( losses } +pub trait Objective { + type Model; + + fn objective(&self, x: &Array2, y: &Array1) -> Array1; +} + pub struct StochasticGradientDescent where T: Float, @@ -93,39 +99,36 @@ where impl StochasticGradientDescent where - T: DivAssign + Float + FromPrimitive + ScalarOperand + std::fmt::Debug, + T: Default + Float + FromPrimitive + ScalarOperand + std::fmt::Debug + ops::AddAssign + ops::DivAssign, { - pub fn sgd(&mut self, x: &Array2, y: &Array1) -> Array1 - where - T: std::ops::DivAssign, - { - let (samples, _inputs) = x.dim(); + pub fn sgd(&mut self, x: &Array2, y: &Array1) -> Array1 { + let (samples, inputs) = x.dim(); let mut indices: Vec = (0..samples).collect(); let mut losses = Array1::::zeros(self.epochs); for epoch in 0..self.epochs { indices.shuffle(&mut rand::thread_rng()); + for batch_start in (0..samples).step_by(self.batch_size) { let batch_end = (batch_start + self.batch_size).min(samples); - let mut gradient = Array2::zeros(x.dim()); + let mut gradient = Array2::zeros((inputs, self.model().features().outputs())); for i in batch_start..batch_end { let idx = indices[i]; - let input = x.slice(s![idx, ..]).to_owned(); - let prediction = self.model.forward(&input); - let error = prediction - y[idx]; - gradient.slice_mut(s![idx, ..]).assign(&(input * error)); + let input = x.slice(s![idx, ..]).to_owned(); // (1, inputs) + let prediction = self.model.forward(&input); // (1, outputs) + println!("Predicted Shape:\n{:?}", &prediction.shape()); + let error = &prediction - y[idx]; + gradient += input.dot(&error); } gradient /= T::from(self.batch_size).unwrap(); self.model.update_with_gradient(&gradient, self.gamma); println!("Gadient:\n{:?}", &gradient); + losses[epoch] += gradient.mean().unwrap_or_default(); } - let loss = mse(&self.model.forward(x), y).unwrap(); - losses[epoch] = loss; - - println!("Epoch {}: Loss = {:?}", epoch, loss); + losses[epoch] /= T::from(self.batch_size).unwrap(); } losses } @@ -134,20 +137,20 @@ where #[cfg(test)] mod tests { use super::*; - use crate::GenerateRandom; + use crate::core::prelude::GenerateRandom; use ndarray::prelude::{Array, Array1}; #[test] fn test_sgd() { - let (samples, inputs) = (100, 2); + let (samples, inputs) = (20, 5); let shape = (samples, inputs); - let (batch_size, epochs, gamma) = (10, 2, 0.01); + let (batch_size, epochs, gamma) = (10, 1, 0.01); // Generate some example data - let x = Array::linspace(1., 200., 200).into_shape(shape).unwrap(); + let x = Array::linspace(1., 100., 100).into_shape(shape).unwrap(); let y = Array1::::uniform(0, 100); - let model = LinearLayer::::new(inputs, 3); + let model = LinearLayer::::new(inputs, 5); let mut sgd = StochasticGradientDescent::new(batch_size, epochs, gamma, model); sgd.sgd(&x, &y); diff --git a/ml/optim/src/lib.rs b/ml/optim/src/lib.rs index 3ea9c8f2..357ccb18 100644 --- a/ml/optim/src/lib.rs +++ b/ml/optim/src/lib.rs @@ -12,6 +12,10 @@ pub(crate) mod utils; pub(crate) use concision_core as core; pub(crate) use concision_neural as neural; +pub mod cost; +pub mod grad; +pub mod optimizer; + pub mod prelude { pub use crate::primitives::*; diff --git a/ml/optim/src/optimizer/mod.rs b/ml/optim/src/optimizer/mod.rs new file mode 100644 index 00000000..e69de29b diff --git a/ml/optim/src/primitives.rs b/ml/optim/src/primitives.rs index 859023bb..008b50ce 100644 --- a/ml/optim/src/primitives.rs +++ b/ml/optim/src/primitives.rs @@ -8,4 +8,8 @@ mod constants {} mod statics {} -mod types {} +mod types { + use ndarray::prelude::{Array1, Array2}; + + pub type ObjectiveFn = fn(&Array2, &Array1) -> Array1; +} diff --git a/ml/transformers/src/attention/weights.rs b/ml/transformers/src/attention/weights.rs index ba67a5f5..9c8b892c 100644 --- a/ml/transformers/src/attention/weights.rs +++ b/ml/transformers/src/attention/weights.rs @@ -26,7 +26,7 @@ //! use super::params::QKV; use super::Weights; -use crate::neural::GenerateRandom; +use crate::core::GenerateRandom; use crate::ops::Split; use ndarray::linalg::Dot; use ndarray::prelude::{Array, Array2, Array3, Ix2}; From 227351d5245b1186976197ff70369a8e20bfc2da Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 10 Nov 2023 12:25:02 -0600 Subject: [PATCH 045/118] update Signed-off-by: FL03 --- ml/optim/examples/sgd.rs | 19 +++++++++++-------- ml/optim/src/grad/sgd.rs | 9 ++++----- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/ml/optim/examples/sgd.rs b/ml/optim/examples/sgd.rs index 38a8f8ca..81312d92 100644 --- a/ml/optim/examples/sgd.rs +++ b/ml/optim/examples/sgd.rs @@ -1,18 +1,21 @@ -use concision_core::prelude::{GenerateRandom, BoxResult}; + use concision_neural::layers::linear::LinearLayer; use concision_optim::grad::sgd::StochasticGradientDescent; -use ndarray::prelude::{Array, Array1}; +use ndarray::prelude::Array; -fn main() -> BoxResult { - let (samples, inputs) = (20, 5); +fn main() -> anyhow::Result<()> { + let (samples, inputs) = (20, 10); let shape = (samples, inputs); - let (batch_size, epochs, gamma) = (10, 1, 0.01); + let n = samples * inputs; + + let (batch_size, epochs, gamma) = (10, 10, 0.01); // Generate some example data - let x = Array::linspace(1., 100., 100).into_shape(shape).unwrap(); - let y = Array::linspace(1., 100., 100).into_shape(100).unwrap(); + let base = Array::linspace(1., n as f64, n); + let x = base.clone().into_shape(shape).unwrap(); + let y = base.clone().into_shape(n).unwrap() + 1.0; - let model = LinearLayer::::new(inputs, 5); + let model = LinearLayer::::new(inputs, 8); let mut sgd = StochasticGradientDescent::new(batch_size, epochs, gamma, model); let losses = sgd.sgd(&x, &y); diff --git a/ml/optim/src/grad/sgd.rs b/ml/optim/src/grad/sgd.rs index 31f2e65a..66a44096 100644 --- a/ml/optim/src/grad/sgd.rs +++ b/ml/optim/src/grad/sgd.rs @@ -115,17 +115,16 @@ where for i in batch_start..batch_end { let idx = indices[i]; - let input = x.slice(s![idx, ..]).to_owned(); // (1, inputs) + let input = x.slice(s![idx, ..]).to_shape((1, inputs)).expect("").to_owned(); // (1, inputs) let prediction = self.model.forward(&input); // (1, outputs) - println!("Predicted Shape:\n{:?}", &prediction.shape()); let error = &prediction - y[idx]; - gradient += input.dot(&error); + gradient += &(&input * &error.t()).t(); } gradient /= T::from(self.batch_size).unwrap(); - self.model.update_with_gradient(&gradient, self.gamma); + self.model.update_with_gradient(&gradient.t().to_owned(), self.gamma); - println!("Gadient:\n{:?}", &gradient); + println!("Gradient:\n{:?}", &gradient); losses[epoch] += gradient.mean().unwrap_or_default(); } losses[epoch] /= T::from(self.batch_size).unwrap(); From c26e4c8b752b15c3db1669c644cdf93ca63a9d4b Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 10 Nov 2023 12:51:08 -0600 Subject: [PATCH 046/118] update Signed-off-by: FL03 --- ml/neural/src/layers/linear/layer.rs | 4 ++-- ml/optim/examples/sgd.rs | 2 +- ml/optim/src/grad/sgd.rs | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ml/neural/src/layers/linear/layer.rs b/ml/neural/src/layers/linear/layer.rs index 69d2006f..88b062eb 100644 --- a/ml/neural/src/layers/linear/layer.rs +++ b/ml/neural/src/layers/linear/layer.rs @@ -113,11 +113,11 @@ where impl Forward for LinearLayer where - T: Float + ScalarOperand + Mul, + T: Float + ScalarOperand, { type Output = Array2; fn forward(&self, data: &T) -> Self::Output { - self.weights().t().to_owned() * data.clone() + self.bias().clone() + &self.weights().t().to_owned() * data.clone() + self.bias().clone() } } diff --git a/ml/optim/examples/sgd.rs b/ml/optim/examples/sgd.rs index 81312d92..f30fb08b 100644 --- a/ml/optim/examples/sgd.rs +++ b/ml/optim/examples/sgd.rs @@ -9,7 +9,7 @@ fn main() -> anyhow::Result<()> { let n = samples * inputs; - let (batch_size, epochs, gamma) = (10, 10, 0.01); + let (batch_size, epochs, gamma) = (20, 4, 0.01); // Generate some example data let base = Array::linspace(1., n as f64, n); let x = base.clone().into_shape(shape).unwrap(); diff --git a/ml/optim/src/grad/sgd.rs b/ml/optim/src/grad/sgd.rs index 66a44096..648ebc8d 100644 --- a/ml/optim/src/grad/sgd.rs +++ b/ml/optim/src/grad/sgd.rs @@ -119,12 +119,15 @@ where let prediction = self.model.forward(&input); // (1, outputs) let error = &prediction - y[idx]; gradient += &(&input * &error.t()).t(); + } gradient /= T::from(self.batch_size).unwrap(); self.model.update_with_gradient(&gradient.t().to_owned(), self.gamma); println!("Gradient:\n{:?}", &gradient); + // let loss = mse(&self.model.forward(x), y).unwrap(); + // println!("Epoch: {:?}\nLoss:\n{:?}", &epoch, &loss); losses[epoch] += gradient.mean().unwrap_or_default(); } losses[epoch] /= T::from(self.batch_size).unwrap(); From c2b9c9dcecd09c069eb0944e0ca49dc6a164cb7c Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 12 Nov 2023 09:49:33 -0600 Subject: [PATCH 047/118] update Signed-off-by: FL03 --- core/src/epochs/epoch.rs | 6 + core/src/epochs/mod.rs | 12 ++ core/src/lib.rs | 2 + core/src/specs.rs | 30 ++--- ml/neural/src/layers/features.rs | 41 +++++++ ml/neural/src/layers/layer.rs | 7 +- ml/neural/src/layers/linear/layer.rs | 63 +++++++--- ml/neural/src/layers/linear/mod.rs | 4 +- ml/neural/src/loss/mod.rs | 23 ++-- ml/neural/src/loss/regress.rs | 23 +++- ml/neural/src/neurons/mod.rs | 1 - ml/neural/src/params/bias.rs | 38 ++++-- ml/neural/src/params/mod.rs | 25 +++- ml/neural/src/params/weight.rs | 2 + ml/neural/src/prop/propagation.rs | 7 ++ ml/neural/src/utils.rs | 20 ++++ ml/optim/examples/sgd.rs | 5 +- ml/optim/src/cost/mod.rs | 21 +++- ml/optim/src/grad/mod.rs | 37 +++--- ml/optim/src/grad/sgd.rs | 110 ++++++++++++++---- ml/optim/src/optimizer/mod.rs | 1 + ml/optim/src/specs.rs | 13 +++ .../src/attention/multi/attention.rs | 2 +- ml/transformers/src/ffn/network.rs | 2 +- 24 files changed, 381 insertions(+), 114 deletions(-) create mode 100644 core/src/epochs/epoch.rs create mode 100644 core/src/epochs/mod.rs diff --git a/core/src/epochs/epoch.rs b/core/src/epochs/epoch.rs new file mode 100644 index 00000000..27fd6c58 --- /dev/null +++ b/core/src/epochs/epoch.rs @@ -0,0 +1,6 @@ +/* + Appellation: epoch + Contrib: FL03 +*/ + +pub struct Epoch {} diff --git a/core/src/epochs/mod.rs b/core/src/epochs/mod.rs new file mode 100644 index 00000000..25ad6560 --- /dev/null +++ b/core/src/epochs/mod.rs @@ -0,0 +1,12 @@ +/* + Appellation: epochs + Contrib: FL03 +*/ +pub use self::{epoch::*, utils::*}; + +pub(crate) mod epoch; + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests {} diff --git a/core/src/lib.rs b/core/src/lib.rs index a55f15ba..66b3a49c 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -9,11 +9,13 @@ pub(crate) mod primitives; pub(crate) mod specs; pub(crate) mod utils; +pub mod epochs; pub mod errors; pub mod states; pub mod step; pub mod prelude { + pub use crate::epochs::*; pub use crate::errors::*; pub use crate::states::*; pub use crate::step::*; diff --git a/core/src/specs.rs b/core/src/specs.rs index cb5c80b5..9171e75f 100644 --- a/core/src/specs.rs +++ b/core/src/specs.rs @@ -7,12 +7,17 @@ use ndarray::{Dimension, IntoDimension, ShapeError}; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Uniform}; use ndarray_rand::RandomExt; -use num::Float; -use num::{Num, One, Zero}; -use std::ops::MulAssign; +use num::traits::NumOps; +use num::{Float, Num, One, Zero}; +use std::ops; pub trait BinaryNum: One + Zero {} +pub trait NumOpsAssign: + Num + NumOps + Sized + ops::AddAssign + ops::DivAssign + ops::MulAssign + ops::SubAssign +{ +} + pub trait Pair { fn pair(&self) -> (A, B); } @@ -26,20 +31,19 @@ where } } -pub trait Product { - type Item: Num; - - fn product(&self) -> Self::Item; +pub trait Product +where + T: Num, +{ + fn product(&self) -> T; } -impl Product for I +impl Product for I where I: Clone + IntoIterator, - T: One + Num + MulAssign, + T: One + Num + ops::MulAssign, { - type Item = T; - - fn product(&self) -> Self::Item { + fn product(&self) -> T { let mut res = T::one(); for i in self.clone().into_iter() { res *= i; @@ -98,8 +102,6 @@ where // } // } - - pub trait GenerateRandom where T: Float + SampleUniform, diff --git a/ml/neural/src/layers/features.rs b/ml/neural/src/layers/features.rs index fd263495..9e1eff14 100644 --- a/ml/neural/src/layers/features.rs +++ b/ml/neural/src/layers/features.rs @@ -25,6 +25,32 @@ impl Features { pub fn outputs(&self) -> usize { self.outputs } + + pub fn set_inputs(&mut self, inputs: usize) { + self.inputs = inputs; + } + + pub fn set_outputs(&mut self, outputs: usize) { + self.outputs = outputs; + } + + pub fn with_inputs(mut self, inputs: usize) -> Self { + self.inputs = inputs; + self + } + + pub fn with_outputs(mut self, outputs: usize) -> Self { + self.outputs = outputs; + self + } + + pub fn in_by_out(&self) -> (usize, usize) { + (self.inputs, self.outputs) + } + + pub fn out_by_in(&self) -> (usize, usize) { + (self.outputs, self.inputs) + } } impl std::fmt::Display for Features { @@ -59,8 +85,23 @@ impl From for [usize; 2] { } } +impl From<[usize; 2]> for Features { + fn from(features: [usize; 2]) -> Self { + Self { + inputs: features[0], + outputs: features[1], + } + } +} + impl From for (usize, usize) { fn from(features: Features) -> Self { (features.inputs, features.outputs) } } + +impl From<(usize, usize)> for Features { + fn from((inputs, outputs): (usize, usize)) -> Self { + Self { inputs, outputs } + } +} diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 4bf03f9a..62b97458 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -6,8 +6,8 @@ use super::{Features, LayerType}; use crate::params::Bias; use crate::prop::Forward; -use ndarray::ScalarOperand; use ndarray::prelude::Array2; +use ndarray::ScalarOperand; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; @@ -77,7 +77,10 @@ where } } -impl Forward> for Layer where T: Float + ScalarOperand + 'static { +impl Forward> for Layer +where + T: Float + ScalarOperand + 'static, +{ type Output = Array2; fn forward(&self, data: &Array2) -> Self::Output { diff --git a/ml/neural/src/layers/linear/layer.rs b/ml/neural/src/layers/linear/layer.rs index 88b062eb..0026cfd1 100644 --- a/ml/neural/src/layers/linear/layer.rs +++ b/ml/neural/src/layers/linear/layer.rs @@ -12,7 +12,7 @@ use ndarray::{Dimension, ScalarOperand}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; -use std::ops::{Add, Mul}; +use std::ops::Add; #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] pub struct LinearLayer { @@ -23,8 +23,16 @@ pub struct LinearLayer { impl LinearLayer where - T: Float + ScalarOperand, + T: Float, { + pub fn new(bias: Bias, features: Features, weights: Array2) -> Self { + Self { + bias, + features, + weights, + } + } + pub fn bias(&self) -> &Bias { &self.bias } @@ -41,20 +49,6 @@ where &mut self.features } - pub fn fit(&mut self, data: &Array2) -> Array2 - where - T: 'static, - { - self.linear(data) - } - - pub fn linear(&self, data: &Array2) -> Array2 - where - T: 'static, - { - data.dot(&self.weights.t()) + &self.bias - } - pub fn weights(&self) -> &Array2 { &self.weights } @@ -63,7 +57,15 @@ where &mut self.weights } - pub fn update_weights(&mut self, weights: Array2) { + pub fn set_bias(&mut self, bias: Bias) { + self.bias = bias; + } + + pub fn set_features(&mut self, features: Features) { + self.features = features; + } + + pub fn set_weights(&mut self, weights: Array2) { self.weights = weights; } @@ -71,13 +73,24 @@ where self.features = params; self } + + pub fn update_bias_at(&mut self, index: usize, value: T) { + self.bias_mut(); + } } impl LinearLayer where T: Float + SampleUniform, { - pub fn new(inputs: usize, outputs: usize) -> Self { + pub fn init(mut self) -> Self { + let (inputs, outputs) = self.features().in_by_out(); + self.bias = Bias::biased(outputs); + self.weights = Array2::uniform(1, (outputs, inputs)); + self + } + + pub fn new_biased(inputs: usize, outputs: usize) -> Self { let features = Features::new(inputs, outputs); let weights = Array2::uniform(1, (outputs, inputs)); let bias = Bias::biased(outputs); @@ -93,6 +106,20 @@ impl LinearLayer where T: Float + ScalarOperand, { + pub fn fit(&mut self, data: &Array2) -> Array2 + where + T: 'static, + { + self.linear(data) + } + + pub fn linear(&self, data: &Array2) -> Array2 + where + T: 'static, + { + data.dot(&self.weights.t()) + &self.bias + } + pub fn update_with_gradient(&mut self, gradient: &Array2, lr: T) { self.weights = self.weights() + gradient * lr; } diff --git a/ml/neural/src/layers/linear/mod.rs b/ml/neural/src/layers/linear/mod.rs index 804adcd2..dec67b25 100644 --- a/ml/neural/src/layers/linear/mod.rs +++ b/ml/neural/src/layers/linear/mod.rs @@ -8,8 +8,8 @@ pub use self::{layer::*, utils::*}; pub(crate) mod layer; use crate::params::{Biased, Weighted}; -use ndarray::ScalarOperand; use ndarray::prelude::Array2; +use ndarray::ScalarOperand; use num::Float; pub trait LinearTransformation @@ -50,7 +50,7 @@ mod tests { fn test_linear_layer() { let (inputs, outputs) = (2, 2); let data = Array2::::ones((inputs, outputs)); - let layer = LinearLayer::new(inputs, outputs); + let layer = LinearLayer::new_biased(inputs, outputs); let linear = layer.linear(&data); assert_eq!(linear.dim(), (inputs, outputs)); } diff --git a/ml/neural/src/loss/mod.rs b/ml/neural/src/loss/mod.rs index 72fee3b9..b85c72fd 100644 --- a/ml/neural/src/loss/mod.rs +++ b/ml/neural/src/loss/mod.rs @@ -14,36 +14,41 @@ pub(crate) mod kinds; pub mod regress; -use ndarray::{Dimension, ScalarOperand}; use ndarray::linalg::Dot; use ndarray::prelude::{Array, Array1}; +use ndarray::{Dimension, ScalarOperand}; use num::{Float, FromPrimitive}; use std::ops; pub trait Loss { - fn loss(&self, pred: &Array, target: &Array1) -> T; } pub struct MSE; impl MSE { - pub fn partial_slope(data: &Array, bias: &Array1, slope: &Array, target: &Array1) -> Option + pub fn partial_slope( + data: &Array, + bias: &Array1, + slope: &Array, + target: &Array1, + ) -> (T, T) where D: Dimension, T: Float + FromPrimitive + ScalarOperand, Array: Dot>, - as Dot>>::Output: ops::Add, Output = Array> + ops::Sub, Output = Array> + ops::Mul>, + as Dot>>::Output: ops::Add, Output = Array> + + ops::Sub, Output = Array> + + ops::Mul>, Array1: ops::Sub, Output = Array>, { let predicted = data.dot(&slope.t().to_owned()) + bias.clone(); - let inner = data.dot(&(target.clone() - predicted)) * (- T::from(2).unwrap()); - inner.mean() + let w = data.dot(&(target.clone() - predicted.clone())) * (-T::from(2).unwrap()); + let b = (target.clone() - predicted) * (-T::from(2).unwrap()); + (w.mean().unwrap(), b.mean().unwrap()) } } - - pub(crate) mod utils { use ndarray::prelude::{Array, Array1}; use ndarray::{Dimension, ScalarOperand}; @@ -67,6 +72,4 @@ pub(crate) mod utils { { (target.clone() - pred.clone()).mapv(|x| x.powi(2)).mean() } - - } diff --git a/ml/neural/src/loss/regress.rs b/ml/neural/src/loss/regress.rs index 63511be6..ab0e9654 100644 --- a/ml/neural/src/loss/regress.rs +++ b/ml/neural/src/loss/regress.rs @@ -3,8 +3,8 @@ Contrib: FL03 */ use super::Loss; -use ndarray::{Dimension, ScalarOperand}; use ndarray::prelude::{Array, Array1}; +use ndarray::{Dimension, ScalarOperand}; use num::Float; use std::ops; @@ -19,7 +19,10 @@ pub struct HuberLoss { delta: T, } -impl HuberLoss where T: Float { +impl HuberLoss +where + T: Float, +{ pub fn new(delta: T) -> Self { Self { delta } } @@ -33,8 +36,10 @@ impl HuberLoss where T: Float { } } -impl Loss for HuberLoss where T: Float + ops::AddAssign { - +impl Loss for HuberLoss +where + T: Float + ops::AddAssign, +{ fn loss(&self, pred: &Array, target: &Array1) -> T { let half = T::from(0.5).unwrap(); let mut loss = T::zero(); @@ -54,7 +59,10 @@ impl Loss for HuberLoss where T: Float + ops::AddAssign { pub struct MeanAbsoluteError; -impl Loss for MeanAbsoluteError where T: Float + ops::AddAssign + ops::DivAssign { +impl Loss for MeanAbsoluteError +where + T: Float + ops::AddAssign + ops::DivAssign, +{ fn loss(&self, pred: &Array, target: &Array1) -> T { let mut res = T::zero(); for (p, t) in pred.iter().cloned().zip(target.iter().cloned()) { @@ -67,7 +75,10 @@ impl Loss for MeanAbsoluteError where T: Float + ops::AddAssign + ops::Div pub struct MeanSquaredError; -impl Loss for MeanSquaredError where T: Float + ops::AddAssign + ops::DivAssign { +impl Loss for MeanSquaredError +where + T: Float + ops::AddAssign + ops::DivAssign, +{ fn loss(&self, pred: &Array, target: &Array1) -> T { let mut res = T::zero(); for (p, t) in pred.iter().cloned().zip(target.iter().cloned()) { diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index 43d8e5ff..74df7790 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -10,7 +10,6 @@ pub(crate) mod node; pub mod activate; - pub(crate) mod utils {} #[cfg(test)] diff --git a/ml/neural/src/params/bias.rs b/ml/neural/src/params/bias.rs index 38bf6a93..926baf7c 100644 --- a/ml/neural/src/params/bias.rs +++ b/ml/neural/src/params/bias.rs @@ -2,21 +2,19 @@ Appellation: bias Contrib: FL03 */ +use crate::generate_uniform_arr; use ndarray::prelude::{Array, Array1}; use ndarray::{Dimension, ScalarOperand}; -use ndarray_rand::rand_distr::{uniform::SampleUniform, Uniform}; -use ndarray_rand::RandomExt; +use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; use smart_default::SmartDefault; use std::ops; use strum::EnumIs; -fn _generate_bias(size: usize) -> Array1 { - let k = T::one() / T::from(size).unwrap(); - let dk = k.sqrt(); - let dist = Uniform::new(-dk, dk); - Array1::::random(size, dist) +pub struct Belief { + pub bias: Bias, + pub features: usize, } #[derive(Clone, Debug, Deserialize, EnumIs, PartialEq, Serialize, SmartDefault)] @@ -27,7 +25,23 @@ pub enum Bias { Value(T), } -impl Bias where T: Float + ScalarOperand { +impl Bias +where + T: Float, +{ + pub fn update_at(&mut self, index: usize, value: T) { + match self { + Self::Biased(bias) => bias[index] = value, + Self::Unbiased => (), + Self::Value(bias) => *bias = value, + } + } +} + +impl Bias +where + T: Float + ScalarOperand, +{ pub fn forward(&self, data: &Array1) -> Array1 { match self { Self::Biased(bias) => data + bias, @@ -42,12 +56,15 @@ where T: Float + SampleUniform, { pub fn biased(size: usize) -> Self { - let bias = _generate_bias(size); + let bias = generate_uniform_arr(0, size); Self::Biased(bias) } } -impl From for Bias where T: Float { +impl From for Bias +where + T: Float, +{ fn from(value: T) -> Self { Self::Value(value) } @@ -155,7 +172,6 @@ where } } - impl ops::Sub> for Bias where D: Dimension, diff --git a/ml/neural/src/params/mod.rs b/ml/neural/src/params/mod.rs index 96f24894..75f49173 100644 --- a/ml/neural/src/params/mod.rs +++ b/ml/neural/src/params/mod.rs @@ -11,18 +11,35 @@ pub use self::{bias::*, utils::*, weight::*}; pub(crate) mod bias; pub(crate) mod weight; -use ndarray::prelude::Array2; +use ndarray::linalg::Dot; +use ndarray::prelude::{Array, Array2, Ix2}; +use ndarray::Dimension; use num::Float; -pub trait Parameter { - +pub enum ParameterShapes { + Thick { features: usize, outputs: usize }, } -pub trait Biased { +pub trait Parameter {} + +pub trait Biased +where + T: Float, +{ fn bias(&self) -> &Bias; fn bias_mut(&mut self) -> &mut Bias; } +pub trait W +where + Self: Dot>, + D: Dimension, + T: Float, +{ + fn weights(&self) -> &Self; + fn weights_mut(&mut self) -> &Self; +} + pub trait Weighted where T: Float, diff --git a/ml/neural/src/params/weight.rs b/ml/neural/src/params/weight.rs index 68cd6248..e8a41294 100644 --- a/ml/neural/src/params/weight.rs +++ b/ml/neural/src/params/weight.rs @@ -9,6 +9,8 @@ use ndarray_rand::RandomExt; use num::Float; use serde::{Deserialize, Serialize}; +pub enum WeightShape {} + #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Weight { weights: Array2, diff --git a/ml/neural/src/prop/propagation.rs b/ml/neural/src/prop/propagation.rs index 7048eb46..589b2cd1 100644 --- a/ml/neural/src/prop/propagation.rs +++ b/ml/neural/src/prop/propagation.rs @@ -5,5 +5,12 @@ use super::PropagationMode; pub struct Propagator { + pub epochs: usize, pub mode: PropagationMode, } + +impl Propagator { + pub fn new(epochs: usize, mode: PropagationMode) -> Self { + Self { epochs, mode } + } +} diff --git a/ml/neural/src/utils.rs b/ml/neural/src/utils.rs index f8840c69..9b7b2532 100644 --- a/ml/neural/src/utils.rs +++ b/ml/neural/src/utils.rs @@ -2,3 +2,23 @@ Appellation: utils Contrib: FL03 */ +use ndarray::prelude::Array; +use ndarray::{Dimension, IntoDimension}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::Uniform; +use ndarray_rand::RandomExt; +use num::Float; + +pub fn generate_uniform_arr(axis: usize, dim: impl IntoDimension) -> Array +where + D: Dimension, + T: Float + SampleUniform, +{ + let shape: D = dim.into_dimension(); + let dk = { + let k = T::from(shape[axis]).unwrap(); + (T::one() / k).sqrt() + }; + let dist = Uniform::new(-dk, dk); + Array::::random(shape, dist) +} diff --git a/ml/optim/examples/sgd.rs b/ml/optim/examples/sgd.rs index f30fb08b..9e8df137 100644 --- a/ml/optim/examples/sgd.rs +++ b/ml/optim/examples/sgd.rs @@ -1,4 +1,3 @@ - use concision_neural::layers::linear::LinearLayer; use concision_optim::grad::sgd::StochasticGradientDescent; use ndarray::prelude::Array; @@ -15,10 +14,10 @@ fn main() -> anyhow::Result<()> { let x = base.clone().into_shape(shape).unwrap(); let y = base.clone().into_shape(n).unwrap() + 1.0; - let model = LinearLayer::::new(inputs, 8); + let model = LinearLayer::::new_biased(inputs, 8); let mut sgd = StochasticGradientDescent::new(batch_size, epochs, gamma, model); let losses = sgd.sgd(&x, &y); println!("Losses {:?}", losses); Ok(()) -} \ No newline at end of file +} diff --git a/ml/optim/src/cost/mod.rs b/ml/optim/src/cost/mod.rs index 8ea8a7ec..3df94f42 100644 --- a/ml/optim/src/cost/mod.rs +++ b/ml/optim/src/cost/mod.rs @@ -8,7 +8,26 @@ pub use self::{kinds::*, utils::*}; pub(crate) mod kinds; -pub trait Cost {} +use ndarray::prelude::Array; +use ndarray::Dimension; +use num::Float; + +pub trait Cost +where + T: Float, +{ + fn cost(&self, pred: &T, target: &T) -> T; +} + +pub trait CostArr +where + T: Float, +{ + type Dim: Dimension; + + fn cost(&self, pred: &Array, target: &Array) + -> Array; +} pub(crate) mod utils {} diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index 2290c367..302f3d2d 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -14,32 +14,39 @@ use num::Float; pub trait Descent { type Params; - fn descent(&self, ) -> Vec; + fn descent(&self) -> Vec; } -pub trait LearningRate { - fn gamma(&self) -> f64; +pub trait LearningRate +where + T: Float, +{ + fn gamma(&self) -> T; } -pub trait Momentum { - - fn mu(&self) -> f64; // Momentum Rate - - fn nestrov(&self) -> bool; - - fn tau(&self) -> f64; // Momentum Damper +pub trait Momentum +where + T: Float, +{ + fn mu(&self) -> T; // Momentum Rate } -pub trait Nesterov { +pub trait Nesterov: Momentum { fn nestrov(&self) -> bool; } -pub trait Decay { - fn lambda(&self) -> f64; // Decay Rate +pub trait Decay +where + T: Float, +{ + fn lambda(&self) -> T; // Decay Rate } -pub trait Dampener { - fn tau(&self) -> f64; // Momentum Damper +pub trait Dampener +where + T: Float, +{ + fn tau(&self) -> T; // Momentum Damper } pub struct DescentParams { diff --git a/ml/optim/src/grad/sgd.rs b/ml/optim/src/grad/sgd.rs index 648ebc8d..6c79d56d 100644 --- a/ml/optim/src/grad/sgd.rs +++ b/ml/optim/src/grad/sgd.rs @@ -8,7 +8,7 @@ use crate::neural::layers::linear::LinearLayer; use crate::neural::prelude::{mse, Forward}; // use crate::prelude::ObjectiveFn; -use ndarray::prelude::{s, Array1, Array2}; +use ndarray::prelude::{s, Array1, Array2, Axis}; use ndarray::ScalarOperand; use num::{Float, FromPrimitive}; use rand::seq::SliceRandom; @@ -18,43 +18,75 @@ pub fn sgd( x: &Array2, y: &Array1, model: &mut LinearLayer, - learning_rate: f64, epochs: usize, + learning_rate: f64, batch_size: usize, -) -> Array1 { +) -> anyhow::Result> { + let layer = model.clone(); + let features = layer.features(); let (samples, _inputs) = x.dim(); let mut indices: Vec = (0..samples).collect(); let mut losses = Array1::::zeros(epochs); for epoch in 0..epochs { indices.shuffle(&mut rand::thread_rng()); - for batch_start in (0..samples).step_by(batch_size) { - let batch_end = (batch_start + batch_size).min(samples); - let mut gradient = Array2::zeros(x.dim()); + let pos = &indices[..batch_size]; + + let xs = x.select(Axis(0), pos); + let ys = y.select(Axis(0), pos); + + let pred = model.forward(&xs); + let error = &pred - &ys; + let grad_w = xs.dot(&error.t()).sum() * (-2.0 / batch_size as f64); + let grad_b = error.sum() * (-2.0 / batch_size as f64); - for i in batch_start..batch_end { + for batch in (0..samples).step_by(batch_size) { + let mut gradient = Array2::zeros((features.outputs(), features.inputs())); + + for i in batch..(batch + batch_size).min(samples) { let idx = indices[i]; - let input = x.slice(s![idx, ..]).to_owned(); - let prediction = model.forward(&input); - let error = prediction - y[idx]; - gradient.slice_mut(s![idx, ..]).assign(&(input * error)); - } + let input = x + .slice(s![idx, ..]) + .to_shape((1, features.inputs()))? + .to_owned(); // (1, inputs) + let prediction = model.forward(&input); // (1, outputs) + + let inner = y[idx] - &prediction; + let partial_w = -2.0 * input.dot(&inner).mean().unwrap(); + let partial_b = -2.0 * inner.mean().unwrap(); + // let mut weights = model.weights_mut().slice_mut(s![]) + // model.set_weights(weights) + + let cost = mse(&prediction, y).unwrap(); + losses[epoch] += cost; + // let error = &prediction - y[idx]; + println!("Cost:\t{:?}", &cost); + gradient += &(input * cost); + } gradient /= batch_size as f64; model.update_with_gradient(&gradient, learning_rate); - } - let loss = mse(&model.forward(x), y).unwrap(); - losses[epoch] = loss; - println!("Epoch {}: Loss = {}", epoch, loss); + println!("Gradient:\n{:?}", &gradient); + } + losses /= batch_size as f64; } - losses + + Ok(losses) } -pub trait Objective { - type Model; +pub struct Sgd { + batch_size: usize, + gamma: f64, // learning rate + model: LinearLayer, +} + +impl Iterator for Sgd { + type Item = Array1; - fn objective(&self, x: &Array2, y: &Array1) -> Array1; + fn next(&mut self) -> Option { + None + } } pub struct StochasticGradientDescent @@ -99,7 +131,13 @@ where impl StochasticGradientDescent where - T: Default + Float + FromPrimitive + ScalarOperand + std::fmt::Debug + ops::AddAssign + ops::DivAssign, + T: Default + + Float + + FromPrimitive + + ScalarOperand + + std::fmt::Debug + + ops::AddAssign + + ops::DivAssign, { pub fn sgd(&mut self, x: &Array2, y: &Array1) -> Array1 { let (samples, inputs) = x.dim(); @@ -115,15 +153,19 @@ where for i in batch_start..batch_end { let idx = indices[i]; - let input = x.slice(s![idx, ..]).to_shape((1, inputs)).expect("").to_owned(); // (1, inputs) + let input = x + .slice(s![idx, ..]) + .to_shape((1, inputs)) + .expect("") + .to_owned(); // (1, inputs) let prediction = self.model.forward(&input); // (1, outputs) let error = &prediction - y[idx]; gradient += &(&input * &error.t()).t(); - } gradient /= T::from(self.batch_size).unwrap(); - self.model.update_with_gradient(&gradient.t().to_owned(), self.gamma); + self.model + .update_with_gradient(&gradient.t().to_owned(), self.gamma); println!("Gradient:\n{:?}", &gradient); // let loss = mse(&self.model.forward(x), y).unwrap(); @@ -152,11 +194,29 @@ mod tests { let x = Array::linspace(1., 100., 100).into_shape(shape).unwrap(); let y = Array1::::uniform(0, 100); - let model = LinearLayer::::new(inputs, 5); + let model = LinearLayer::::new_biased(inputs, 5); let mut sgd = StochasticGradientDescent::new(batch_size, epochs, gamma, model); sgd.sgd(&x, &y); // sgd(&x, &y, &mut model, learning_rate, epochs, batch_size); } + + #[test] + fn test_stochastic() { + let (samples, inputs) = (20, 5); + let shape = (samples, inputs); + + let (batch_size, epochs, gamma) = (10, 1, 0.01); + // Generate some example data + let x = Array::linspace(1., 100., 100).into_shape(shape).unwrap(); + let y = Array1::::uniform(0, 100); + + let mut model = LinearLayer::::new_biased(inputs, 5); + + let grad = sgd(&x, &y, &mut model, epochs, gamma, batch_size); + assert!(grad.is_ok()); + + // sgd(&x, &y, &mut model, learning_rate, epochs, batch_size); + } } diff --git a/ml/optim/src/optimizer/mod.rs b/ml/optim/src/optimizer/mod.rs index e69de29b..8b137891 100644 --- a/ml/optim/src/optimizer/mod.rs +++ b/ml/optim/src/optimizer/mod.rs @@ -0,0 +1 @@ + diff --git a/ml/optim/src/specs.rs b/ml/optim/src/specs.rs index d4ebfe67..e8c02e7c 100644 --- a/ml/optim/src/specs.rs +++ b/ml/optim/src/specs.rs @@ -2,6 +2,19 @@ Appellation: specs Contrib: FL03 */ +use ndarray::prelude::{Array1, Array2}; + +pub trait Gradient { + type Model; + + fn gradient(&self, x: &Array2, y: &Array1) -> Array1; +} + +pub trait Objective { + type Model; + + fn objective(&self, x: &Array2, y: &Array1) -> Array1; +} pub trait Optimize { fn optimize(&self, params: &mut dyn Optimizable); diff --git a/ml/transformers/src/attention/multi/attention.rs b/ml/transformers/src/attention/multi/attention.rs index fbe4323e..753ea7bd 100644 --- a/ml/transformers/src/attention/multi/attention.rs +++ b/ml/transformers/src/attention/multi/attention.rs @@ -45,7 +45,7 @@ where let params = MultiHeadParams::new(heads, model); let weights = Weight::uniform((model, model)); Self { - linear: LinearLayer::new(model, model), + linear: LinearLayer::new_biased(model, model), params, weights, } diff --git a/ml/transformers/src/ffn/network.rs b/ml/transformers/src/ffn/network.rs index 8f449c81..db13324c 100644 --- a/ml/transformers/src/ffn/network.rs +++ b/ml/transformers/src/ffn/network.rs @@ -19,7 +19,7 @@ pub struct FFN { impl FFN { pub fn new(model: usize, network: Option) -> Self { let params = FFNParams::new(model, network.unwrap_or(crate::NETWORK_SIZE)); - let layer = LinearLayer::new(params.model, params.network); + let layer = LinearLayer::new_biased(params.model, params.network); Self { input: layer.clone(), output: layer, From fa56cd555afd45b916da423b336f042bd80a17cf Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 12 Nov 2023 14:17:19 -0600 Subject: [PATCH 048/118] update Signed-off-by: FL03 --- core/src/specs.rs | 4 ++ ml/neural/src/layers/features.rs | 4 ++ ml/neural/src/layers/linear/layer.rs | 41 +++++++++---- ml/neural/src/loss/mod.rs | 2 + ml/neural/src/neurons/mod.rs | 57 ++++++++--------- ml/neural/src/neurons/neuron.rs | 20 +++--- ml/neural/src/neurons/node.rs | 4 +- ml/neural/src/params/bias.rs | 17 ++++++ ml/optim/examples/descent.rs | 27 +++++++++ ml/optim/examples/sgd.rs | 13 ++-- ml/optim/src/cost/mod.rs | 28 +++++++-- ml/optim/src/grad/descent.rs | 91 ++++++++++++++++++++++++---- ml/optim/src/grad/gradient.rs | 38 ++++++++++++ ml/optim/src/grad/mod.rs | 3 +- ml/optim/src/grad/sgd.rs | 66 +++++++++++++------- ml/optim/src/specs.rs | 2 +- 16 files changed, 315 insertions(+), 102 deletions(-) create mode 100644 ml/optim/examples/descent.rs create mode 100644 ml/optim/src/grad/gradient.rs diff --git a/core/src/specs.rs b/core/src/specs.rs index 9171e75f..142aa009 100644 --- a/core/src/specs.rs +++ b/core/src/specs.rs @@ -121,6 +121,10 @@ where let k = (T::one() / T::from(dim[axis]).unwrap()).sqrt(); Array::random(dim, Uniform::new(-k, k)) } + + fn uniform_between(dk: T, dim: impl IntoDimension) -> Array { + Array::random(dim, Uniform::new(-dk, dk)) + } } impl GenerateRandom for Array diff --git a/ml/neural/src/layers/features.rs b/ml/neural/src/layers/features.rs index 9e1eff14..d876fe2b 100644 --- a/ml/neural/src/layers/features.rs +++ b/ml/neural/src/layers/features.rs @@ -18,6 +18,10 @@ impl Features { Self { inputs, outputs } } + pub fn uniform_scale(&self) -> T { + (T::one() / T::from(self.inputs).unwrap()).sqrt() + } + pub fn inputs(&self) -> usize { self.inputs } diff --git a/ml/neural/src/layers/linear/layer.rs b/ml/neural/src/layers/linear/layer.rs index 0026cfd1..7990c3bd 100644 --- a/ml/neural/src/layers/linear/layer.rs +++ b/ml/neural/src/layers/linear/layer.rs @@ -7,7 +7,7 @@ use crate::layers::Features; use crate::prelude::{Bias, Forward}; use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Array2}; +use ndarray::prelude::{Array, Array1, Array2}; use ndarray::{Dimension, ScalarOperand}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; @@ -25,11 +25,11 @@ impl LinearLayer where T: Float, { - pub fn new(bias: Bias, features: Features, weights: Array2) -> Self { + pub fn new(features: Features) -> Self { Self { - bias, + bias: Array1::zeros(features.outputs()).into(), features, - weights, + weights: Array2::zeros(features.out_by_in()), } } @@ -83,10 +83,18 @@ impl LinearLayer where T: Float + SampleUniform, { - pub fn init(mut self) -> Self { + pub fn init_bias(mut self) -> Self { + let (inputs, outputs) = self.features().in_by_out(); + let dk = (T::one() / T::from(inputs).unwrap()).sqrt(); + self.bias = ndarray::Array1::uniform_between(dk, outputs).into(); + self + } + + pub fn init_weight(mut self) -> Self { let (inputs, outputs) = self.features().in_by_out(); - self.bias = Bias::biased(outputs); - self.weights = Array2::uniform(1, (outputs, inputs)); + let dk = (T::one() / T::from(inputs).unwrap()).sqrt(); + self.bias = ndarray::Array1::uniform_between(dk, outputs).into(); + self.weights = Array2::uniform_between(dk, (outputs, inputs)); self } @@ -120,18 +128,29 @@ where data.dot(&self.weights.t()) + &self.bias } + pub fn update_params_gradient(&mut self, bias: &Array1, weights: &Array2, lr: T) { + // self.bias = self.bias() + bias * lr; + self.weights = self.weights() - weights * lr; + } + pub fn update_with_gradient(&mut self, gradient: &Array2, lr: T) { - self.weights = self.weights() + gradient * lr; + self.weights = &self.weights - gradient * lr; + } + + pub fn apply_gradient(&mut self, gradient: &Array1, lr: T) { + self.weights = &self.weights - gradient * lr; } } -impl Forward> for LinearLayer +impl Forward> for LinearLayer where D: Dimension, + S: Dimension, T: Float + ScalarOperand, - Array: Add, Output = Array> + Dot, Output = Array>, + Array: Add, Output = Array> + Dot, Output = Array>, + Array: Add, Output = Array>, { - type Output = Array; + type Output = Array; fn forward(&self, data: &Array) -> Self::Output { data.dot(&self.weights().t().to_owned()) + self.bias().clone() diff --git a/ml/neural/src/loss/mod.rs b/ml/neural/src/loss/mod.rs index b85c72fd..a3c19635 100644 --- a/ml/neural/src/loss/mod.rs +++ b/ml/neural/src/loss/mod.rs @@ -24,6 +24,8 @@ pub trait Loss { fn loss(&self, pred: &Array, target: &Array1) -> T; } +// pub type LinearWeightGradient = fn() + pub struct MSE; impl MSE { diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index 74df7790..c4604b3d 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -25,52 +25,43 @@ mod tests { weights: &Array1, ) -> Array1 { rho.activate( - args.dot(weights) - bias.unwrap_or_else(|| Array1::::zeros(args.shape()[0])), + args.dot(weights) + bias.unwrap_or_else(|| Array1::::zeros(args.shape()[0])), ) } #[test] fn test_neuron() { - let bias = ndarray::Array1::::zeros(4); + let bias = 0.0; - let a_data = array![10.0, 10.0, 6.0, 1.0, 8.0]; - let a_weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; - let a = Neuron::new(softmax, bias.clone(), a_weights.clone()); + let data = array![10.0, 10.0, 6.0, 1.0, 8.0]; + let weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; + let neuron = Neuron::new(|x| x, bias.clone(), weights.clone()); - let exp = _artificial(&a_data, Some(bias.clone()), Softmax::default(), &a_weights); - assert_eq!(a.compute(&a_data), exp); + let exp = data.clone().dot(&weights) + bias; - let b_data = array![0.0, 9.0, 3.0, 5.0, 3.0]; - let b_weights = array![2.0, 8.0, 8.0, 0.0, 3.0]; - - let b = Neuron::new(softmax, bias.clone(), b_weights.clone()); - - let exp = _artificial(&b_data, Some(bias), Softmax::default(), &b_weights); - assert_eq!(b.compute(&b_data), exp); - - // assert_eq!(a.dot() + b.dot(), 252.0); + assert_eq!(exp, neuron.process(&data)); } - #[test] - fn test_node() { - let bias = ndarray::Array1::::zeros(4); + // #[test] + // fn test_node() { + // let bias = ndarray::Array1::::zeros(4); - let a_data = array![10.0, 10.0, 6.0, 1.0, 8.0]; - let a_weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; - let a = Neuron::new(softmax, bias.clone(), a_weights.clone()); - let node_a = Node::new(a.clone()).with_data(a_data.clone()); + // let a_data = array![10.0, 10.0, 6.0, 1.0, 8.0]; + // let a_weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; + // let a = Neuron::new(softmax, bias.clone(), a_weights.clone()); + // let node_a = Node::new(a.clone()).with_data(a_data.clone()); - let exp = _artificial(&a_data, Some(bias.clone()), Softmax::default(), &a_weights); - assert_eq!(node_a.process(), exp); + // let exp = _artificial(&a_data, Some(bias.clone()), Softmax::default(), &a_weights); + // assert_eq!(node_a.process(), exp); - let b_data = array![0.0, 9.0, 3.0, 5.0, 3.0]; - let b_weights = array![2.0, 8.0, 8.0, 0.0, 3.0]; + // let b_data = array![0.0, 9.0, 3.0, 5.0, 3.0]; + // let b_weights = array![2.0, 8.0, 8.0, 0.0, 3.0]; - let b = Neuron::new(softmax, bias.clone(), b_weights.clone()); - let node_b = Node::new(b.clone()).with_data(b_data.clone()); - let exp = _artificial(&b_data, Some(bias), Softmax::default(), &b_weights); - assert_eq!(node_b.process(), exp); + // let b = Neuron::new(softmax, bias.clone(), b_weights.clone()); + // let node_b = Node::new(b.clone()).with_data(b_data.clone()); + // let exp = _artificial(&b_data, Some(bias), Softmax::default(), &b_weights); + // assert_eq!(node_b.process(), exp); - assert_eq!(node_a.dot() + node_b.dot(), 252.0); - } + // assert_eq!(node_a.dot() + node_b.dot(), 252.0); + // } } diff --git a/ml/neural/src/neurons/neuron.rs b/ml/neural/src/neurons/neuron.rs index e620e68f..e84d4a2c 100644 --- a/ml/neural/src/neurons/neuron.rs +++ b/ml/neural/src/neurons/neuron.rs @@ -8,17 +8,13 @@ use ndarray::prelude::Array1; /// Artificial Neuron #[derive(Clone, Debug, PartialEq)] pub struct Neuron { - activation: ActivationFn>, - bias: Array1, + activation: ActivationFn, + bias: f64, weights: Array1, } impl Neuron { - pub fn new( - activation: ActivationFn>, - bias: Array1, - weights: Array1, - ) -> Self { + pub fn new(activation: ActivationFn, bias: f64, weights: Array1) -> Self { Self { activation, bias, @@ -26,15 +22,15 @@ impl Neuron { } } - pub fn bias(&self) -> &Array1 { + pub fn bias(&self) -> &f64 { &self.bias } - pub fn compute(&self, args: &Array1) -> Array1 { - self.rho()(args.dot(&self.weights) - self.bias()) + pub fn process(&self, args: &Array1) -> f64 { + self.rho()(args.dot(&self.weights.t()) + self.bias()) } - pub fn rho(&self) -> ActivationFn> { + pub fn rho(&self) -> ActivationFn { self.activation } @@ -42,7 +38,7 @@ impl Neuron { &self.weights } - pub fn set_bias(&mut self, bias: Array1) { + pub fn set_bias(&mut self, bias: f64) { self.bias = bias; } diff --git a/ml/neural/src/neurons/node.rs b/ml/neural/src/neurons/node.rs index 5dcb3bf3..01915686 100644 --- a/ml/neural/src/neurons/node.rs +++ b/ml/neural/src/neurons/node.rs @@ -33,8 +33,8 @@ impl Node { &self.neuron } - pub fn process(&self) -> Array1 { - self.neuron.compute(&self.data) + pub fn process(&self) -> f64 { + self.neuron.process(&self.data) } pub fn set_data(&mut self, data: Array1) { diff --git a/ml/neural/src/params/bias.rs b/ml/neural/src/params/bias.rs index 926baf7c..b01b9574 100644 --- a/ml/neural/src/params/bias.rs +++ b/ml/neural/src/params/bias.rs @@ -121,6 +121,23 @@ where } } +impl ops::Add> for &Bias +where + D: Dimension, + T: Float + ScalarOperand, + Array: ops::Add, Output = Array>, +{ + type Output = Array; + + fn add(self, rhs: Array) -> Self::Output { + match self.clone() { + Bias::Biased(bias) => rhs + bias, + Bias::Unbiased => rhs, + Bias::Value(value) => &rhs + value, + } + } +} + impl ops::Add<&Array> for Bias where D: Dimension, diff --git a/ml/optim/examples/descent.rs b/ml/optim/examples/descent.rs new file mode 100644 index 00000000..5d5b8226 --- /dev/null +++ b/ml/optim/examples/descent.rs @@ -0,0 +1,27 @@ +use concision_neural::layers::linear::LinearLayer; +use concision_optim::grad::gradient_descent_step; +use ndarray::prelude::Array; + +fn main() -> anyhow::Result<()> { + let (samples, inputs) = (20, 5); + let outputs = 1; + let n = samples * inputs; + + let (_epochs, gamma) = (10, 0.5); + // Generate some example data + let x = Array::linspace(1., n as f64, n) + .into_shape((samples, inputs)) + .unwrap(); + let y = Array::linspace(1., n as f64, outputs) + .into_shape(outputs) + .unwrap(); + + let features = (inputs, outputs).into(); + let mut model = LinearLayer::::new(features).init_weight(); + + for e in 0.._epochs { + let cost = gradient_descent_step(&x, &y, &mut model, gamma); + println!("Step ({}) Cost {:?}", e, cost); + } + Ok(()) +} diff --git a/ml/optim/examples/sgd.rs b/ml/optim/examples/sgd.rs index 9e8df137..d12837c0 100644 --- a/ml/optim/examples/sgd.rs +++ b/ml/optim/examples/sgd.rs @@ -4,17 +4,22 @@ use ndarray::prelude::Array; fn main() -> anyhow::Result<()> { let (samples, inputs) = (20, 10); - let shape = (samples, inputs); + let outputs = 5; let n = samples * inputs; let (batch_size, epochs, gamma) = (20, 4, 0.01); // Generate some example data let base = Array::linspace(1., n as f64, n); - let x = base.clone().into_shape(shape).unwrap(); - let y = base.clone().into_shape(n).unwrap() + 1.0; + let x = Array::linspace(1., n as f64, n) + .into_shape((samples, inputs)) + .unwrap(); + let y = Array::linspace(1., n as f64, outputs) + .into_shape(outputs) + .unwrap() + + 1.0; - let model = LinearLayer::::new_biased(inputs, 8); + let model = LinearLayer::::new_biased(inputs, outputs); let mut sgd = StochasticGradientDescent::new(batch_size, epochs, gamma, model); let losses = sgd.sgd(&x, &y); diff --git a/ml/optim/src/cost/mod.rs b/ml/optim/src/cost/mod.rs index 3df94f42..bba10846 100644 --- a/ml/optim/src/cost/mod.rs +++ b/ml/optim/src/cost/mod.rs @@ -8,7 +8,7 @@ pub use self::{kinds::*, utils::*}; pub(crate) mod kinds; -use ndarray::prelude::Array; +use ndarray::prelude::{Array, Array1}; use ndarray::Dimension; use num::Float; @@ -25,11 +25,31 @@ where { type Dim: Dimension; - fn cost(&self, pred: &Array, target: &Array) - -> Array; + fn cost(&self, pred: &Array, target: &Array1) -> Array; } -pub(crate) mod utils {} +pub(crate) mod utils { + + use ndarray::prelude::Array; + use ndarray::Dimension; + use num::Float; + + pub fn mse<'a, T, D>(pred: &Array, target: &Array) -> Array + where + T: Float, + D: Dimension, + { + (pred - target).mapv(|x| x.powi(2)) + } + + pub fn mae<'a, T, D>(pred: &Array, target: &Array) -> Array + where + T: Float, + D: Dimension, + { + (pred - target).mapv(|x| x.abs()) + } +} #[cfg(test)] mod tests {} diff --git a/ml/optim/src/grad/descent.rs b/ml/optim/src/grad/descent.rs index 6617dbb3..2a7224ce 100644 --- a/ml/optim/src/grad/descent.rs +++ b/ml/optim/src/grad/descent.rs @@ -2,27 +2,94 @@ Appellation: grad Contrib: FL03 */ - use crate::neural::layers::linear::LinearLayer; -use crate::neural::prelude::Forward; +use crate::neural::prelude::{mse, Forward}; use ndarray::prelude::{Array1, Array2}; -fn gradient_descent( +pub fn cost(target: &Array1, prediction: &Array1) -> f64 { + mse(prediction, target).unwrap_or_default() +} + +pub fn gradient_descent_step( x: &Array2, y: &Array1, model: &mut LinearLayer, learning_rate: f64, - epochs: usize, -) { - let n_samples = x.shape()[0]; +) -> f64 { + let (_samples, _inputs) = x.dim(); + + let predictions = model.forward(x); // fully connected + let errors = y - &predictions; + let scale = -1.0 / x.len() as f64; + let _bias = scale * &errors; + let weights = scale * x.t().dot(&errors); + // let wg = (scale * x.t().dot(&errors)).sum_axis(Axis(1)); + + println!("Weights: {:?}", &weights); + model.update_with_gradient(&weights.t().to_owned(), learning_rate); + // model.apply_gradient(&wg, learning_rate); + + mse(&predictions, y).unwrap_or_default() +} + +pub struct GradientDescent { + pub gamma: f64, + model: LinearLayer, +} + +impl GradientDescent { + pub fn new(gamma: f64, model: LinearLayer) -> Self { + Self { gamma, model } + } + + pub fn gradient(&self, x: &Array2, y: &Array1) -> Array2 { + let (samples, _inputs) = x.dim(); + + let predictions = x.dot(&self.model.weights().t()) + self.model.bias(); // fully connected + let errors = y - &predictions; + let scale = -2.0 / samples as f64; + scale * x.t().dot(&errors) + } + + pub fn step(&mut self, x: &Array2, y: &Array1) -> f64 { + let (samples, _inputs) = x.dim(); + + let predictions = x.dot(&self.model.weights().t()) + self.model.bias(); // fully connected + let errors = y - &predictions; + let scale = -2.0 / samples as f64; + let _bias = scale * &errors; + let weights = scale * x.t().dot(&errors); + self.model + .update_with_gradient(&weights.t().to_owned(), self.gamma); + + mse(&self.model.forward(x), y).unwrap_or_default() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ndarray::prelude::Array; + + #[test] + fn test_gradient_descent() { + let (samples, inputs) = (20, 5); + let outputs = 8; + let n = samples * inputs; + + let (_epochs, gamma) = (1, 0.01); + // Generate some example data + let x = Array::linspace(1., n as f64, n) + .into_shape((samples, inputs)) + .unwrap(); + let y = Array::linspace(1., n as f64, outputs) + .into_shape(outputs) + .unwrap(); - for epoch in 0..epochs { - let predictions = model.forward(x); - let errors = &predictions - y; - let gradient = x.t().dot(&errors) / n_samples as f64; + let mut model = LinearLayer::::new_biased(inputs, outputs); - model.update_with_gradient(&gradient, learning_rate); + let _grad = gradient_descent_step(&x, &y, &mut model, gamma); - // let err = mse(&predictions, y).expect("Error calculating MSE"); + // sgd(&x, &y, &mut model, learning_rate, epochs, batch_size); } } diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs new file mode 100644 index 00000000..36aa0861 --- /dev/null +++ b/ml/optim/src/grad/gradient.rs @@ -0,0 +1,38 @@ +/* + Appellation: grad + Contrib: FL03 +*/ + +use ndarray::prelude::{Array1, Array2}; + +pub type BiasGradient = Array1; + +pub type WeightGradient = Array2; + +pub struct Gradient { + pub bias: BiasGradient, + pub weights: WeightGradient, +} + +impl Gradient { + pub fn new(bias: BiasGradient, weights: WeightGradient) -> Self { + Self { bias, weights } + } + + pub fn zeros(inputs: usize, outputs: usize) -> Self { + Self { + bias: Array1::zeros(outputs), + weights: Array2::zeros((inputs, outputs)), + } + } +} + +#[cfg(test)] +mod tests { + + #[test] + fn test_gradient() { + let (samples, inputs) = (20, 5); + let _shape = (samples, inputs); + } +} diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index 302f3d2d..d787e653 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -3,9 +3,10 @@ Contrib: FL03 */ //! # Gradient Descent -pub use self::{descent::*, utils::*}; +pub use self::{descent::*, gradient::*, utils::*}; pub(crate) mod descent; +pub(crate) mod gradient; pub mod sgd; diff --git a/ml/optim/src/grad/sgd.rs b/ml/optim/src/grad/sgd.rs index 6c79d56d..c26c8af4 100644 --- a/ml/optim/src/grad/sgd.rs +++ b/ml/optim/src/grad/sgd.rs @@ -9,10 +9,9 @@ use crate::neural::layers::linear::LinearLayer; use crate::neural::prelude::{mse, Forward}; // use crate::prelude::ObjectiveFn; use ndarray::prelude::{s, Array1, Array2, Axis}; -use ndarray::ScalarOperand; +use ndarray::NdFloat; use num::{Float, FromPrimitive}; use rand::seq::SliceRandom; -use std::ops; pub fn sgd( x: &Array2, @@ -53,8 +52,9 @@ pub fn sgd( let prediction = model.forward(&input); // (1, outputs) let inner = y[idx] - &prediction; - let partial_w = -2.0 * input.dot(&inner).mean().unwrap(); - let partial_b = -2.0 * inner.mean().unwrap(); + let partial_w = (-2.0 / batch_size as f64) * input.dot(&inner); + let partial_b = (-2.0 / batch_size as f64) * inner; + gradient -= partial_w.sum(); // let mut weights = model.weights_mut().slice_mut(s![]) // model.set_weights(weights) @@ -62,7 +62,7 @@ pub fn sgd( losses[epoch] += cost; // let error = &prediction - y[idx]; println!("Cost:\t{:?}", &cost); - gradient += &(input * cost); + // gradient += &(input * cost); } gradient /= batch_size as f64; model.update_with_gradient(&gradient, learning_rate); @@ -75,12 +75,44 @@ pub fn sgd( Ok(losses) } +pub fn sgd_step( + x: &Array2, + y: &Array1, + model: &mut LinearLayer, + learning_rate: f64, + batch_size: usize, +) -> anyhow::Result { + let layer = model.clone(); + let features = layer.features(); + let (samples, _inputs) = x.dim(); + let mut indices: Vec = (0..features.outputs()).collect(); + let mut losses = 0.0; + + indices.shuffle(&mut rand::thread_rng()); + let pos = &indices[..batch_size]; + + let xs = x.select(Axis(0), pos); + let ys = y.select(Axis(0), pos); + + let pred = model.forward(&xs); + + Ok(losses) +} + pub struct Sgd { batch_size: usize, gamma: f64, // learning rate model: LinearLayer, } +impl Sgd { + pub fn step(&mut self) -> f64 { + let mut loss = 0.0; + + loss + } +} + impl Iterator for Sgd { type Item = Array1; @@ -131,13 +163,7 @@ where impl StochasticGradientDescent where - T: Default - + Float - + FromPrimitive - + ScalarOperand - + std::fmt::Debug - + ops::AddAssign - + ops::DivAssign, + T: Default + FromPrimitive + NdFloat, { pub fn sgd(&mut self, x: &Array2, y: &Array1) -> Array1 { let (samples, inputs) = x.dim(); @@ -159,7 +185,7 @@ where .expect("") .to_owned(); // (1, inputs) let prediction = self.model.forward(&input); // (1, outputs) - let error = &prediction - y[idx]; + let error = &prediction - y; gradient += &(&input * &error.t()).t(); } @@ -168,8 +194,8 @@ where .update_with_gradient(&gradient.t().to_owned(), self.gamma); println!("Gradient:\n{:?}", &gradient); - // let loss = mse(&self.model.forward(x), y).unwrap(); - // println!("Epoch: {:?}\nLoss:\n{:?}", &epoch, &loss); + let loss = mse(&self.model.forward(x), y).unwrap(); + println!("Epoch: {:?}\nLoss:\n{:?}", &epoch, &loss); losses[epoch] += gradient.mean().unwrap_or_default(); } losses[epoch] /= T::from(self.batch_size).unwrap(); @@ -192,14 +218,12 @@ mod tests { let (batch_size, epochs, gamma) = (10, 1, 0.01); // Generate some example data let x = Array::linspace(1., 100., 100).into_shape(shape).unwrap(); - let y = Array1::::uniform(0, 100); + let y = Array::linspace(1., 100., 5).into_shape(5).unwrap(); let model = LinearLayer::::new_biased(inputs, 5); let mut sgd = StochasticGradientDescent::new(batch_size, epochs, gamma, model); sgd.sgd(&x, &y); - - // sgd(&x, &y, &mut model, learning_rate, epochs, batch_size); } #[test] @@ -214,9 +238,7 @@ mod tests { let mut model = LinearLayer::::new_biased(inputs, 5); - let grad = sgd(&x, &y, &mut model, epochs, gamma, batch_size); - assert!(grad.is_ok()); - - // sgd(&x, &y, &mut model, learning_rate, epochs, batch_size); + // let grad = sgd(&x, &y, &mut model, epochs, gamma, batch_size); + // assert!(grad.is_ok()); } } diff --git a/ml/optim/src/specs.rs b/ml/optim/src/specs.rs index e8c02e7c..09af9eb5 100644 --- a/ml/optim/src/specs.rs +++ b/ml/optim/src/specs.rs @@ -7,7 +7,7 @@ use ndarray::prelude::{Array1, Array2}; pub trait Gradient { type Model; - fn gradient(&self, x: &Array2, y: &Array1) -> Array1; + fn gradient(&self, x: &Array2, y: &Array1) -> T; } pub trait Objective { From c1fdf283ce7e4bc72c542ff40fd3ca8bc843ad67 Mon Sep 17 00:00:00 2001 From: FL03 Date: Mon, 13 Nov 2023 10:28:37 -0600 Subject: [PATCH 049/118] update Signed-off-by: FL03 --- Cargo.toml | 1 + core/src/specs.rs | 11 ++- ml/neural/src/layers/linear/mod.rs | 3 +- ml/neural/src/layers/linear/regress.rs | 92 ++++++++++++++++++++++++++ ml/neural/src/neurons/mod.rs | 9 ++- ml/neural/src/neurons/neuron.rs | 68 ++++++++++++++++--- ml/neural/src/neurons/node.rs | 4 -- ml/optim/examples/descent.rs | 79 ++++++++++++++++++++-- ml/optim/src/grad/descent.rs | 90 +++++++++++++++++++++---- ml/optim/src/grad/gradient.rs | 4 +- ml/optim/src/lib.rs | 1 + 11 files changed, 320 insertions(+), 42 deletions(-) create mode 100644 ml/neural/src/layers/linear/regress.rs diff --git a/Cargo.toml b/Cargo.toml index 3e638501..27d9b960 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ computare = { features = ["full"], branch = "v0.1.0", git = "https://github.com/ anyhow = "1" lazy_static = "1" +linfa = { features = [], version = "0.7" } ndarray = { features = ["serde-1"], version = "0.15" } # ndarray-linalg = { features = [], version = "0.16" } ndarray-rand = { features = [], version = "0.14" } diff --git a/core/src/specs.rs b/core/src/specs.rs index 142aa009..a41c34cb 100644 --- a/core/src/specs.rs +++ b/core/src/specs.rs @@ -3,7 +3,8 @@ Contrib: FL03 */ use ndarray::prelude::Array; -use ndarray::{Dimension, IntoDimension, ShapeError}; +use ndarray::{Dimension, IntoDimension, NdFloat, ShapeError}; +// use ndarray::linalg::Dot; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Uniform}; use ndarray_rand::RandomExt; @@ -149,6 +150,14 @@ where } } +pub trait Mean {} + +pub trait Stats +where + T: NdFloat, +{ +} + #[cfg(test)] mod tests { use super::*; diff --git a/ml/neural/src/layers/linear/mod.rs b/ml/neural/src/layers/linear/mod.rs index dec67b25..457932ab 100644 --- a/ml/neural/src/layers/linear/mod.rs +++ b/ml/neural/src/layers/linear/mod.rs @@ -3,9 +3,10 @@ Contrib: FL03 */ //! # Linear Layer -pub use self::{layer::*, utils::*}; +pub use self::{layer::*, regress::*, utils::*}; pub(crate) mod layer; +pub(crate) mod regress; use crate::params::{Biased, Weighted}; use ndarray::prelude::Array2; diff --git a/ml/neural/src/layers/linear/regress.rs b/ml/neural/src/layers/linear/regress.rs new file mode 100644 index 00000000..27326fff --- /dev/null +++ b/ml/neural/src/layers/linear/regress.rs @@ -0,0 +1,92 @@ +/* + Appellation: regress + Contrib: FL03 +*/ +use ndarray::linalg::Dot; +use ndarray::prelude::{Array, Array1, Array2, Axis}; +use ndarray::{Dimension, NdFloat}; +use ndarray_stats::CorrelationExt; +use num::FromPrimitive; + +pub fn covariance(ddof: T, x: &Array, y: &Array) -> anyhow::Result> +where + D: Dimension, + T: Default + FromPrimitive + NdFloat, + Array: Dot, Output = Array>, +{ + let x_mean = x.mean().unwrap_or_default(); + let y_mean = y.mean().unwrap_or_default(); + let xs = x - x_mean; + let ys = y - y_mean; + let cov = xs.dot(&ys.t().to_owned()); + let scale = T::one() / (T::from(x.len()).unwrap() - ddof); + Ok(cov * scale) +} + +pub struct LinearRegression { + bias: f64, + pub features: usize, + weights: Array1, +} + +impl LinearRegression { + pub fn new(features: usize) -> Self { + Self { + bias: 0.0, + features, + weights: Array1::zeros(features), + } + } + + pub fn bias(&self) -> f64 { + self.bias + } + + pub fn bias_mut(&mut self) -> &mut f64 { + &mut self.bias + } + + pub fn features(&self) -> usize { + self.features + } + + pub fn features_mut(&mut self) -> &mut usize { + &mut self.features + } + + pub fn weights(&self) -> &Array1 { + &self.weights + } + + pub fn weights_mut(&mut self) -> &mut Array1 { + &mut self.weights + } + + pub fn set_bias(&mut self, bias: f64) { + self.bias = bias; + } + + pub fn set_features(&mut self, features: usize) { + self.features = features; + } + + pub fn set_weights(&mut self, weights: Array1) { + self.weights = weights; + } + + pub fn fit(&mut self, data: &Array2, targets: &Array1) -> anyhow::Result<()> { + let _m = data.cov(0.0)? / data.var_axis(Axis(0), 0.0); + // let covar = covariance(0.0, x, y); + // self.bias = targets.mean().unwrap_or_default() - m * data.mean().unwrap_or_default(); + // self.weights -= m; + Ok(()) + } + + pub fn predict(&self, data: &Array2) -> Array1 { + data.dot(&self.weights().t()) + self.bias() + } + + pub fn update_with_gradient(&mut self, gradient: &Array1, gamma: f64) { + self.weights = &self.weights - gradient * gamma; + } +} diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index c4604b3d..40489423 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -16,6 +16,7 @@ pub(crate) mod utils {} mod tests { use super::activate::{softmax, Activate, Softmax}; use super::*; + // use lazy_static::lazy_static; use ndarray::{array, Array1}; fn _artificial( @@ -33,11 +34,13 @@ mod tests { fn test_neuron() { let bias = 0.0; - let data = array![10.0, 10.0, 6.0, 1.0, 8.0]; + let data = array![[10.0, 10.0, 6.0, 1.0, 8.0]]; let weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; - let neuron = Neuron::new(|x| x, bias.clone(), weights.clone()); + let neuron = Neuron::new(5) + .with_rho(softmax) + .with_weights(weights.clone()); - let exp = data.clone().dot(&weights) + bias; + let exp = softmax(data.clone().dot(&weights) + bias); assert_eq!(exp, neuron.process(&data)); } diff --git a/ml/neural/src/neurons/neuron.rs b/ml/neural/src/neurons/neuron.rs index e84d4a2c..b3a28f74 100644 --- a/ml/neural/src/neurons/neuron.rs +++ b/ml/neural/src/neurons/neuron.rs @@ -2,35 +2,49 @@ Appellation: neuron Contrib: FL03 */ -use super::activate::ActivationFn; -use ndarray::prelude::Array1; +use super::activate::{Activate, ActivationFn}; +use crate::core::GenerateRandom; +use crate::prelude::Forward; +use ndarray::prelude::{Array1, Array2}; /// Artificial Neuron #[derive(Clone, Debug, PartialEq)] pub struct Neuron { - activation: ActivationFn, + activation: ActivationFn>, bias: f64, + features: usize, weights: Array1, } impl Neuron { - pub fn new(activation: ActivationFn, bias: f64, weights: Array1) -> Self { + pub fn new(features: usize) -> Self { Self { - activation, - bias, - weights, + activation: |x| x, + bias: 0.0, + features, + weights: Array1::zeros(features), } } - pub fn bias(&self) -> &f64 { - &self.bias + pub fn with_rho(mut self, rho: ActivationFn>) -> Self { + self.activation = rho; + self } - pub fn process(&self, args: &Array1) -> f64 { + pub fn init_weights(mut self) -> Self { + self.weights = Array1::uniform(0, self.features); + self + } + + pub fn bias(&self) -> f64 { + self.bias + } + + pub fn process(&self, args: &Array2) -> Array1 { self.rho()(args.dot(&self.weights.t()) + self.bias()) } - pub fn rho(&self) -> ActivationFn { + pub fn rho(&self) -> ActivationFn> { self.activation } @@ -45,4 +59,36 @@ impl Neuron { pub fn set_weights(&mut self, weights: Array1) { self.weights = weights; } + + pub fn with_bias(mut self, bias: f64) -> Self { + self.bias = bias; + self + } + + pub fn with_weights(mut self, weights: Array1) -> Self { + self.weights = weights; + self + } + + pub fn apply_weight_gradient(&mut self, gamma: f64, gradient: &Array1) { + self.weights = &self.weights - gamma * gradient; + } +} + +// impl Forward> for Neuron { +// type Output = f64; + +// fn forward(&self, args: &Array1) -> Self::Output { +// self.rho().activate(args.dot(&self.weights().t().to_owned()) + self.bias) +// } + +// } + +impl Forward> for Neuron { + type Output = Array1; + + fn forward(&self, args: &Array2) -> Self::Output { + let linstep = args.dot(&self.weights().t().to_owned()) + self.bias; + self.rho().activate(linstep) + } } diff --git a/ml/neural/src/neurons/node.rs b/ml/neural/src/neurons/node.rs index 01915686..8ecc8bfd 100644 --- a/ml/neural/src/neurons/node.rs +++ b/ml/neural/src/neurons/node.rs @@ -33,10 +33,6 @@ impl Node { &self.neuron } - pub fn process(&self) -> f64 { - self.neuron.process(&self.data) - } - pub fn set_data(&mut self, data: Array1) { self.data = data; } diff --git a/ml/optim/examples/descent.rs b/ml/optim/examples/descent.rs index 5d5b8226..24168995 100644 --- a/ml/optim/examples/descent.rs +++ b/ml/optim/examples/descent.rs @@ -1,13 +1,38 @@ -use concision_neural::layers::linear::LinearLayer; -use concision_optim::grad::gradient_descent_step; +use concision_neural::layers::linear; +use concision_neural::prelude::{Features, Neuron}; +use concision_optim::prelude::{ + gradient_descent, gradient_descent_node, gradient_descent_step, GradientDescent, +}; use ndarray::prelude::Array; fn main() -> anyhow::Result<()> { let (samples, inputs) = (20, 5); let outputs = 1; + + let features = Features::new(inputs, outputs); + + let n = samples * inputs; + + let (epochs, gamma) = (100, 0.5); + let mut x = Array::linspace(1., n as f64, samples) + .into_shape(samples) + .unwrap(); + println!( + "{:?}", + gradient_descent(&mut x, epochs, gamma, |a| a.clone()) + ); + // sample_descent(epochs, features, gamma)?; + // sample_node(epochs, features.outputs(), gamma)?; + // sample_steps(epochs, features.outputs(), gamma)?; + + Ok(()) +} + +pub fn sample_descent(epochs: usize, features: Features, gamma: f64) -> anyhow::Result<()> { + let (samples, inputs) = (20, features.inputs()); + let outputs = features.outputs(); let n = samples * inputs; - let (_epochs, gamma) = (10, 0.5); // Generate some example data let x = Array::linspace(1., n as f64, n) .into_shape((samples, inputs)) @@ -16,10 +41,52 @@ fn main() -> anyhow::Result<()> { .into_shape(outputs) .unwrap(); - let features = (inputs, outputs).into(); - let mut model = LinearLayer::::new(features).init_weight(); + let model = linear::LinearLayer::::new(features).init_weight(); + let mut grad = GradientDescent::new(gamma, model); + + for e in 0..epochs { + let cost = grad.step(&x, &y); + println!("Step ({}) Cost {:?}", e, cost); + } + Ok(()) +} + +pub fn sample_node(epochs: usize, features: usize, gamma: f64) -> anyhow::Result<()> { + let mut node = Neuron::new(features).init_weights(); + let (samples, inputs) = (20, features); + let n = samples * inputs; + + // Generate some example data + let x = Array::linspace(1., n as f64, n) + .into_shape((samples, inputs)) + .unwrap(); + let y = Array::linspace(1., n as f64, samples) + .into_shape(samples) + .unwrap(); + + for e in 0..epochs { + let cost = gradient_descent_node(gamma, &mut node, &x, &y); + println!("Step ({}) Cost {:?}", e, cost); + } + Ok(()) +} + +// pub fn sample_descent(gamma: f64, inputs: usize, outputs: usize, ) + +pub fn sample_steps(epochs: usize, features: usize, gamma: f64) -> anyhow::Result<()> { + let mut model = linear::LinearRegression::new(features); + let (samples, inputs) = (20, features); + let n = samples * inputs; + + // Generate some example data + let x = Array::linspace(1., n as f64, n) + .into_shape((samples, inputs)) + .unwrap(); + let y = Array::linspace(1., n as f64, samples) + .into_shape(samples) + .unwrap(); - for e in 0.._epochs { + for e in 0..epochs { let cost = gradient_descent_step(&x, &y, &mut model, gamma); println!("Step ({}) Cost {:?}", e, cost); } diff --git a/ml/optim/src/grad/descent.rs b/ml/optim/src/grad/descent.rs index 2a7224ce..e1c5bfc1 100644 --- a/ml/optim/src/grad/descent.rs +++ b/ml/optim/src/grad/descent.rs @@ -2,34 +2,71 @@ Appellation: grad Contrib: FL03 */ -use crate::neural::layers::linear::LinearLayer; -use crate::neural::prelude::{mse, Forward}; +use crate::neural::layers::linear::{LinearLayer, LinearRegression}; +use crate::neural::prelude::{mse, Forward, Neuron}; use ndarray::prelude::{Array1, Array2}; +use ndarray_stats::DeviationExt; pub fn cost(target: &Array1, prediction: &Array1) -> f64 { mse(prediction, target).unwrap_or_default() } +pub fn gradient_descent( + data: &mut Array1, + epochs: usize, + gamma: f64, + obj: impl Fn(&Array1) -> Array1, +) -> Array1 { + let mut losses = Array1::zeros(epochs); + for e in 0..epochs { + let gradient = obj(data); + for (xs, g) in data.iter_mut().zip(gradient.iter()) { + *xs -= gamma * g; + losses[e] += *xs; + } + losses[e] /= data.len() as f64; + } + losses +} + +pub fn gradient_descent_node( + gamma: f64, + node: &mut Neuron, + x: &Array2, + y: &Array1, +) -> f64 { + let pred = node.forward(x); + let error = pred.clone() - y; + let scale = -2.0 / x.len() as f64; + let wg = scale * x.t().dot(&error); + node.apply_weight_gradient(gamma, &wg); + node.forward(x).mean_sq_err(y).unwrap_or_default() +} + pub fn gradient_descent_step( x: &Array2, y: &Array1, - model: &mut LinearLayer, + model: &mut LinearRegression, learning_rate: f64, ) -> f64 { - let (_samples, _inputs) = x.dim(); - - let predictions = model.forward(x); // fully connected - let errors = y - &predictions; - let scale = -1.0 / x.len() as f64; + let (samples, _inputs) = x.dim(); + // forward the provided data to the model + let predictions = model.predict(x); // fully connected + println!("Predictions (dim): {:?}", &predictions.shape()); + // calculate the error + let errors = &predictions - y; + println!("Errors (dim): {:?}", &errors.shape()); + // calculate a scaling factor + let scale = -2.0 / samples as f64; let _bias = scale * &errors; + // calculate the gradient of the weights let weights = scale * x.t().dot(&errors); // let wg = (scale * x.t().dot(&errors)).sum_axis(Axis(1)); - println!("Weights: {:?}", &weights); + println!("Weights (dim): {:?}", &weights.shape()); model.update_with_gradient(&weights.t().to_owned(), learning_rate); // model.apply_gradient(&wg, learning_rate); - - mse(&predictions, y).unwrap_or_default() + predictions.mean_sq_err(y).unwrap_or_default() } pub struct GradientDescent { @@ -72,9 +109,9 @@ mod tests { use ndarray::prelude::Array; #[test] - fn test_gradient_descent() { + fn test_descent() { let (samples, inputs) = (20, 5); - let outputs = 8; + let outputs = 3; let n = samples * inputs; let (_epochs, gamma) = (1, 0.01); @@ -86,7 +123,32 @@ mod tests { .into_shape(outputs) .unwrap(); - let mut model = LinearLayer::::new_biased(inputs, outputs); + let model = LinearLayer::::new_biased(inputs, outputs); + + let mut grad = GradientDescent::new(gamma, model); + + let _s = grad.step(&x, &y); + + // sgd(&x, &y, &mut model, learning_rate, epochs, batch_size); + } + + #[test] + fn test_gradient_descent() { + let (samples, inputs) = (20, 5); + let outputs = 5; + let n = samples * inputs; + + let (_epochs, gamma) = (1, 0.01); + // Generate some example data + let x = Array::linspace(1., n as f64, n) + .into_shape((samples, inputs)) + .unwrap(); + let y = Array::linspace(1., n as f64, samples) + .into_shape(samples) + .unwrap(); + + // let mut model = LinearLayer::::new_biased(inputs, outputs); + let mut model = LinearRegression::new(outputs); let _grad = gradient_descent_step(&x, &y, &mut model, gamma); diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs index 36aa0861..c53b5f4f 100644 --- a/ml/optim/src/grad/gradient.rs +++ b/ml/optim/src/grad/gradient.rs @@ -9,12 +9,12 @@ pub type BiasGradient = Array1; pub type WeightGradient = Array2; -pub struct Gradient { +pub struct Gradients { pub bias: BiasGradient, pub weights: WeightGradient, } -impl Gradient { +impl Gradients { pub fn new(bias: BiasGradient, weights: WeightGradient) -> Self { Self { bias, weights } } diff --git a/ml/optim/src/lib.rs b/ml/optim/src/lib.rs index 357ccb18..4a8b2031 100644 --- a/ml/optim/src/lib.rs +++ b/ml/optim/src/lib.rs @@ -17,6 +17,7 @@ pub mod grad; pub mod optimizer; pub mod prelude { + pub use crate::grad::*; pub use crate::primitives::*; pub use crate::specs::*; From a89edfeab040225c9b881b2a6bf4730375f04f65 Mon Sep 17 00:00:00 2001 From: FL03 Date: Wed, 15 Nov 2023 15:26:15 -0600 Subject: [PATCH 050/118] update Signed-off-by: FL03 --- core/src/step/mod.rs | 2 + core/src/utils.rs | 22 ++- ml/neural/src/layers/linear/layer.rs | 25 ++- ml/neural/src/layers/linear/regress.rs | 148 ++++++++++++++---- ml/neural/src/layers/mod.rs | 6 - ml/neural/src/lib.rs | 1 + ml/neural/src/loss/regress.rs | 2 +- ml/neural/src/models/model.rs | 6 + ml/neural/src/neurons/mod.rs | 2 +- ml/optim/examples/descent.rs | 50 ++++-- ml/optim/src/grad/descent.rs | 109 ++++++++++--- ml/optim/src/grad/gradient.rs | 4 + ml/optim/src/primitives.rs | 6 +- ml/optim/src/specs.rs | 6 + ml/optim/src/utils.rs | 30 ++++ .../src/attention/multi/attention.rs | 4 +- 16 files changed, 321 insertions(+), 102 deletions(-) diff --git a/core/src/step/mod.rs b/core/src/step/mod.rs index 6cb2c8e0..5b0fb408 100644 --- a/core/src/step/mod.rs +++ b/core/src/step/mod.rs @@ -8,4 +8,6 @@ pub(crate) mod stepper; pub mod linspace; +pub trait Step {} + pub(crate) mod utils {} diff --git a/core/src/utils.rs b/core/src/utils.rs index 0142d5bb..71d1c06e 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -2,9 +2,10 @@ Appellation: utils Contrib: FL03 */ -use ndarray::prelude::{Array, Axis}; -use ndarray::{concatenate, RemoveAxis}; -// use num::Float; +use ndarray::linalg::Dot; +use ndarray::prelude::{Array, Axis, NdFloat}; +use ndarray::{concatenate, Dimension, RemoveAxis}; +use num::FromPrimitive; pub fn concat_iter(axis: usize, iter: impl IntoIterator>) -> Array where @@ -19,6 +20,21 @@ where out } +pub fn covariance(ddof: T, x: &Array, y: &Array) -> anyhow::Result> +where + D: Dimension, + T: Default + FromPrimitive + NdFloat, + Array: Dot, Output = Array>, +{ + let x_mean = x.mean().unwrap_or_default(); + let y_mean = y.mean().unwrap_or_default(); + let xs = x - x_mean; + let ys = y - y_mean; + let cov = xs.dot(&ys.t().to_owned()); + let scale = T::one() / (T::from(x.len()).unwrap() - ddof); + Ok(cov * scale) +} + pub fn now() -> u128 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) diff --git a/ml/neural/src/layers/linear/layer.rs b/ml/neural/src/layers/linear/layer.rs index 7990c3bd..0cdc7eb3 100644 --- a/ml/neural/src/layers/linear/layer.rs +++ b/ml/neural/src/layers/linear/layer.rs @@ -7,8 +7,8 @@ use crate::layers::Features; use crate::prelude::{Bias, Forward}; use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Array1, Array2}; -use ndarray::{Dimension, ScalarOperand}; +use ndarray::prelude::{Array, Array1, Array2, NdFloat}; +use ndarray::Dimension; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; @@ -73,10 +73,6 @@ where self.features = params; self } - - pub fn update_bias_at(&mut self, index: usize, value: T) { - self.bias_mut(); - } } impl LinearLayer @@ -112,7 +108,7 @@ where impl LinearLayer where - T: Float + ScalarOperand, + T: NdFloat, { pub fn fit(&mut self, data: &Array2) -> Array2 where @@ -128,17 +124,14 @@ where data.dot(&self.weights.t()) + &self.bias } - pub fn update_params_gradient(&mut self, bias: &Array1, weights: &Array2, lr: T) { - // self.bias = self.bias() + bias * lr; - self.weights = self.weights() - weights * lr; - } - pub fn update_with_gradient(&mut self, gradient: &Array2, lr: T) { self.weights = &self.weights - gradient * lr; } - pub fn apply_gradient(&mut self, gradient: &Array1, lr: T) { - self.weights = &self.weights - gradient * lr; + pub fn apply_gradient(&mut self, gradient: &Array1, gamma: T) { + for (ws, g) in self.weights_mut().iter_mut().zip(gradient.iter()) { + *ws -= *g * gamma; + } } } @@ -146,7 +139,7 @@ impl Forward> for LinearLayer where D: Dimension, S: Dimension, - T: Float + ScalarOperand, + T: NdFloat, Array: Add, Output = Array> + Dot, Output = Array>, Array: Add, Output = Array>, { @@ -159,7 +152,7 @@ where impl Forward for LinearLayer where - T: Float + ScalarOperand, + T: NdFloat, { type Output = Array2; diff --git a/ml/neural/src/layers/linear/regress.rs b/ml/neural/src/layers/linear/regress.rs index 27326fff..0a9c549c 100644 --- a/ml/neural/src/layers/linear/regress.rs +++ b/ml/neural/src/layers/linear/regress.rs @@ -2,47 +2,52 @@ Appellation: regress Contrib: FL03 */ -use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Array1, Array2, Axis}; -use ndarray::{Dimension, NdFloat}; +use crate::core::prelude::GenerateRandom; +use crate::prelude::Forward; +use ndarray::prelude::{Array1, Array2, Axis, NdFloat}; +use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_stats::CorrelationExt; -use num::FromPrimitive; +use num::{Float, FromPrimitive}; +use rand::Rng; -pub fn covariance(ddof: T, x: &Array, y: &Array) -> anyhow::Result> -where - D: Dimension, - T: Default + FromPrimitive + NdFloat, - Array: Dot, Output = Array>, -{ - let x_mean = x.mean().unwrap_or_default(); - let y_mean = y.mean().unwrap_or_default(); - let xs = x - x_mean; - let ys = y - y_mean; - let cov = xs.dot(&ys.t().to_owned()); - let scale = T::one() / (T::from(x.len()).unwrap() - ddof); - Ok(cov * scale) +pub enum Params { + Layer { + bias: Array1, // (outputs,) + weights: Array2, // (outputs, inputs) + }, + Node { + bias: f64, + weights: Array1, // (inputs,) + }, } -pub struct LinearRegression { - bias: f64, +#[derive(Clone, Debug, PartialEq)] +pub struct Linear +where + T: Float, +{ + bias: T, pub features: usize, - weights: Array1, + weights: Array1, } -impl LinearRegression { +impl Linear +where + T: Float, +{ pub fn new(features: usize) -> Self { Self { - bias: 0.0, + bias: T::zero(), features, weights: Array1::zeros(features), } } - pub fn bias(&self) -> f64 { - self.bias + pub fn bias(&self) -> &T { + &self.bias } - pub fn bias_mut(&mut self) -> &mut f64 { + pub fn bias_mut(&mut self) -> &mut T { &mut self.bias } @@ -54,15 +59,15 @@ impl LinearRegression { &mut self.features } - pub fn weights(&self) -> &Array1 { + pub fn weights(&self) -> &Array1 { &self.weights } - pub fn weights_mut(&mut self) -> &mut Array1 { + pub fn weights_mut(&mut self) -> &mut Array1 { &mut self.weights } - pub fn set_bias(&mut self, bias: f64) { + pub fn set_bias(&mut self, bias: T) { self.bias = bias; } @@ -70,23 +75,98 @@ impl LinearRegression { self.features = features; } - pub fn set_weights(&mut self, weights: Array1) { + pub fn set_weights(&mut self, weights: Array1) { + self.weights = weights; + } + + pub fn with_bias(mut self, bias: T) -> Self { + self.bias = bias; + self + } + + pub fn with_features(mut self, features: usize) -> Self { + self.features = features; + self + } + + pub fn with_weights(mut self, weights: Array1) -> Self { self.weights = weights; + self } +} - pub fn fit(&mut self, data: &Array2, targets: &Array1) -> anyhow::Result<()> { - let _m = data.cov(0.0)? / data.var_axis(Axis(0), 0.0); +impl Linear +where + T: Float + SampleUniform, +{ + pub fn init(mut self, biased: bool) -> Self { + if biased { + self = self.init_bias(); + } + self.init_weight() + } + + pub fn init_bias(mut self) -> Self { + let dk = (T::one() / T::from(self.features).unwrap()).sqrt(); + self.bias = rand::thread_rng().gen_range(-dk..dk); + self + } + + pub fn init_weight(mut self) -> Self { + let features = self.features; + let dk = (T::one() / T::from(features).unwrap()).sqrt(); + self.weights = Array1::uniform_between(dk, features); + self + } +} + +impl Linear +where + T: FromPrimitive + NdFloat, +{ + pub fn fit(&mut self, data: &Array2, _targets: &Array1) -> anyhow::Result<()> { + let _m = data.cov(T::zero())? / data.var_axis(Axis(0), T::zero()); // let covar = covariance(0.0, x, y); // self.bias = targets.mean().unwrap_or_default() - m * data.mean().unwrap_or_default(); // self.weights -= m; Ok(()) } - pub fn predict(&self, data: &Array2) -> Array1 { - data.dot(&self.weights().t()) + self.bias() + pub fn predict(&self, data: &Array2) -> Array1 { + data.dot(&self.weights().t()) + *self.bias() } - pub fn update_with_gradient(&mut self, gradient: &Array1, gamma: f64) { + + pub fn apply_gradient(&mut self, gamma: T, gradient: impl Fn(&Array1) -> (T, Array1)) -> T { + let (cost, grad) = gradient(self.weights()); + self.weights_mut().scaled_add(-gamma, &grad); + cost + } + + pub fn update_with_gradient(&mut self, gamma: T, gradient: &Array1) { self.weights = &self.weights - gradient * gamma; } } + +impl Forward> for Linear +where + T: NdFloat, +{ + type Output = Array1; + + fn forward(&self, data: &Array2) -> Self::Output { + data.dot(&self.weights().t().to_owned()) + self.bias().clone() + } +} + +// impl Forward> for Linear +// where +// D: Dimension, +// Array: Dot, Output = Array>, +// { +// type Output = Array1; + +// fn forward(&self, data: &Array) -> Self::Output { +// data.dot(&self.weights().t().to_owned()) + self.bias().clone() +// } +// } diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index 9ac19ec5..c70cfeca 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -38,12 +38,6 @@ pub trait L { fn weights(&self) -> &Array2; } -pub trait Linear { - fn linear(&self, data: &Array2) -> Array2 - where - T: 'static; -} - pub(crate) mod utils {} #[cfg(test)] diff --git a/ml/neural/src/lib.rs b/ml/neural/src/lib.rs index 67a4476d..6bba9f13 100644 --- a/ml/neural/src/lib.rs +++ b/ml/neural/src/lib.rs @@ -28,6 +28,7 @@ pub(crate) use concision_core as core; pub mod prelude { pub use crate::arch::*; + pub use crate::layers::linear::*; pub use crate::layers::*; pub use crate::loss::*; pub use crate::masks::*; diff --git a/ml/neural/src/loss/regress.rs b/ml/neural/src/loss/regress.rs index ab0e9654..1a59ba33 100644 --- a/ml/neural/src/loss/regress.rs +++ b/ml/neural/src/loss/regress.rs @@ -4,7 +4,7 @@ */ use super::Loss; use ndarray::prelude::{Array, Array1}; -use ndarray::{Dimension, ScalarOperand}; +use ndarray::Dimension; use num::Float; use std::ops; diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs index e2a0d764..7a759a87 100644 --- a/ml/neural/src/models/model.rs +++ b/ml/neural/src/models/model.rs @@ -2,3 +2,9 @@ Appellation: model Contrib: FL03 */ +use ndarray::prelude::Array1; + +pub struct Model { + pub bias: T, + pub weights: Array1, +} \ No newline at end of file diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index 40489423..3763cbd9 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -14,7 +14,7 @@ pub(crate) mod utils {} #[cfg(test)] mod tests { - use super::activate::{softmax, Activate, Softmax}; + use super::activate::{softmax, Activate}; use super::*; // use lazy_static::lazy_static; use ndarray::{array, Array1}; diff --git a/ml/optim/examples/descent.rs b/ml/optim/examples/descent.rs index 24168995..b3547283 100644 --- a/ml/optim/examples/descent.rs +++ b/ml/optim/examples/descent.rs @@ -1,9 +1,10 @@ use concision_neural::layers::linear; -use concision_neural::prelude::{Features, Neuron}; +use concision_neural::prelude::{Features, Linear, Neuron}; +use concision_neural::prop::Forward; use concision_optim::prelude::{ gradient_descent, gradient_descent_node, gradient_descent_step, GradientDescent, }; -use ndarray::prelude::Array; +use ndarray::prelude::{Array, s}; fn main() -> anyhow::Result<()> { let (samples, inputs) = (20, 5); @@ -13,41 +14,58 @@ fn main() -> anyhow::Result<()> { let n = samples * inputs; - let (epochs, gamma) = (100, 0.5); - let mut x = Array::linspace(1., n as f64, samples) - .into_shape(samples) + let (epochs, gamma) = (10, 0.5); + + // basic_descent(epochs, features, gamma)?; + + sample_descent(epochs, features, gamma)?; + // sample_node(epochs, features.outputs(), gamma)?; + // sample_steps(epochs, features.outputs(), gamma)?; + + Ok(()) +} + +pub fn basic_descent(epochs: usize, features: Features, gamma: f64) -> anyhow::Result<()> { + let (samples, inputs) = (20, features.inputs()); + let n = samples * inputs; + + let x = Array::linspace(1., n as f64, n) + .into_shape((samples, inputs)) .unwrap(); + let mut model = Linear::new(features.inputs()).init_weight(); + println!( "{:?}", - gradient_descent(&mut x, epochs, gamma, |a| a.clone()) + gradient_descent(model.weights_mut(), epochs, gamma, |a| a.clone()) ); - // sample_descent(epochs, features, gamma)?; - // sample_node(epochs, features.outputs(), gamma)?; - // sample_steps(epochs, features.outputs(), gamma)?; - Ok(()) } pub fn sample_descent(epochs: usize, features: Features, gamma: f64) -> anyhow::Result<()> { let (samples, inputs) = (20, features.inputs()); - let outputs = features.outputs(); let n = samples * inputs; // Generate some example data let x = Array::linspace(1., n as f64, n) .into_shape((samples, inputs)) .unwrap(); - let y = Array::linspace(1., n as f64, outputs) - .into_shape(outputs) + let y = Array::linspace(1., samples as f64, samples) + .into_shape(samples) .unwrap(); - let model = linear::LinearLayer::::new(features).init_weight(); + let model = Linear::new(features.inputs()).init_weight(); + + println!( + "Initial Prediction: {:?}", &y - model.forward(&x) + ); + let mut grad = GradientDescent::new(gamma, model); for e in 0..epochs { - let cost = grad.step(&x, &y); + let cost = grad.descent(&x, &y); println!("Step ({}) Cost {:?}", e, cost); } + println!("Model:\n\nWeights:\n{:?}", grad.model().weights()); Ok(()) } @@ -74,7 +92,7 @@ pub fn sample_node(epochs: usize, features: usize, gamma: f64) -> anyhow::Result // pub fn sample_descent(gamma: f64, inputs: usize, outputs: usize, ) pub fn sample_steps(epochs: usize, features: usize, gamma: f64) -> anyhow::Result<()> { - let mut model = linear::LinearRegression::new(features); + let mut model = linear::Linear::new(features); let (samples, inputs) = (20, features); let n = samples * inputs; diff --git a/ml/optim/src/grad/descent.rs b/ml/optim/src/grad/descent.rs index e1c5bfc1..0b8c8294 100644 --- a/ml/optim/src/grad/descent.rs +++ b/ml/optim/src/grad/descent.rs @@ -2,29 +2,37 @@ Appellation: grad Contrib: FL03 */ -use crate::neural::layers::linear::{LinearLayer, LinearRegression}; +use crate::neural::layers::linear::Linear; use crate::neural::prelude::{mse, Forward, Neuron}; -use ndarray::prelude::{Array1, Array2}; +use ndarray::prelude::{Array1, Array2, Axis}; use ndarray_stats::DeviationExt; pub fn cost(target: &Array1, prediction: &Array1) -> f64 { - mse(prediction, target).unwrap_or_default() + (target - prediction) + .map(|x| x.powi(2)) + .mean() + .unwrap_or_default() } + +pub fn grad(data: &Array2, target: &Array1, prediction: &Array1) -> Array1 { + let error = prediction - target; + let scale = -2.0 / data.len() as f64; + scale * data.t().dot(&error) +} + + + pub fn gradient_descent( - data: &mut Array1, + weights: &mut Array1, epochs: usize, gamma: f64, - obj: impl Fn(&Array1) -> Array1, + partial: impl Fn(&Array1) -> Array1, ) -> Array1 { let mut losses = Array1::zeros(epochs); for e in 0..epochs { - let gradient = obj(data); - for (xs, g) in data.iter_mut().zip(gradient.iter()) { - *xs -= gamma * g; - losses[e] += *xs; - } - losses[e] /= data.len() as f64; + *weights = weights.clone() - gamma * partial(&weights.clone()); + losses[e] = weights.mean().unwrap_or_default(); } losses } @@ -46,7 +54,7 @@ pub fn gradient_descent_node( pub fn gradient_descent_step( x: &Array2, y: &Array1, - model: &mut LinearRegression, + model: &mut Linear, learning_rate: f64, ) -> f64 { let (samples, _inputs) = x.dim(); @@ -64,25 +72,83 @@ pub fn gradient_descent_step( // let wg = (scale * x.t().dot(&errors)).sum_axis(Axis(1)); println!("Weights (dim): {:?}", &weights.shape()); - model.update_with_gradient(&weights.t().to_owned(), learning_rate); + model.update_with_gradient(learning_rate, &weights.t().to_owned()); // model.apply_gradient(&wg, learning_rate); predictions.mean_sq_err(y).unwrap_or_default() } +pub type BaseObjective = fn(&Array1) -> Array1; + +#[derive(Clone)] pub struct GradientDescent { pub gamma: f64, - model: LinearLayer, + model: Linear, } impl GradientDescent { - pub fn new(gamma: f64, model: LinearLayer) -> Self { + pub fn new(gamma: f64, model: Linear) -> Self { Self { gamma, model } } - pub fn gradient(&self, x: &Array2, y: &Array1) -> Array2 { + pub fn gamma(&self) -> f64 { + self.gamma + } + + pub fn gamma_mut(&mut self) -> &mut f64 { + &mut self.gamma + } + + pub fn model(&self) -> &Linear { + &self.model + } + + pub fn model_mut(&mut self) -> &mut Linear { + &mut self.model + } + + pub fn set_gamma(&mut self, gamma: f64) { + self.gamma = gamma; + } + + pub fn set_model(&mut self, model: Linear) { + self.model = model; + } + + pub fn with_gamma(mut self, gamma: f64) -> Self { + self.gamma = gamma; + self + } + + pub fn with_model(mut self, model: Linear) -> Self { + self.model = model; + self + } + + pub fn descent(&mut self, data: &Array2, targets: &Array1) -> anyhow::Result { + let lr = self.gamma; + let pred = self.model.forward(data); + let scale = -1.0 / data.len() as f64; + // let errors = targets - &pred; + // println!("Errors (dim): {:?}", &errors.shape()); + let gradient = |p: &Array1| { + let pred = data.dot(p); + let dist = targets.l2_dist(&pred).expect("L2 distance"); + + let errors = targets - &pred; + (dist, p.clone()) + }; + let c = self.model.apply_gradient(self.gamma, &gradient); + + println!("Dist: {:?}", self.model().weights() - gradient(self.model().weights()).1 * self.gamma() ); + let loss = targets.mean_sq_err(&self.model.forward(data))?; + Ok(loss) + } + + + pub fn gradient(&self, x: &Array2, y: &Array1) -> Array1 { let (samples, _inputs) = x.dim(); - let predictions = x.dot(&self.model.weights().t()) + self.model.bias(); // fully connected + let predictions = x.dot(&self.model.weights().t()) + *self.model.bias(); // fully connected let errors = y - &predictions; let scale = -2.0 / samples as f64; scale * x.t().dot(&errors) @@ -91,13 +157,13 @@ impl GradientDescent { pub fn step(&mut self, x: &Array2, y: &Array1) -> f64 { let (samples, _inputs) = x.dim(); - let predictions = x.dot(&self.model.weights().t()) + self.model.bias(); // fully connected + let predictions = x.dot(&self.model.weights().t()) + *self.model.bias(); // fully connected let errors = y - &predictions; let scale = -2.0 / samples as f64; let _bias = scale * &errors; let weights = scale * x.t().dot(&errors); self.model - .update_with_gradient(&weights.t().to_owned(), self.gamma); + .update_with_gradient(self.gamma, &weights.t().to_owned()); mse(&self.model.forward(x), y).unwrap_or_default() } @@ -123,8 +189,7 @@ mod tests { .into_shape(outputs) .unwrap(); - let model = LinearLayer::::new_biased(inputs, outputs); - + let model = Linear::new(outputs).init_weight(); let mut grad = GradientDescent::new(gamma, model); let _s = grad.step(&x, &y); @@ -148,7 +213,7 @@ mod tests { .unwrap(); // let mut model = LinearLayer::::new_biased(inputs, outputs); - let mut model = LinearRegression::new(outputs); + let mut model = Linear::new(outputs); let _grad = gradient_descent_step(&x, &y, &mut model, gamma); diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs index c53b5f4f..e183505f 100644 --- a/ml/optim/src/grad/gradient.rs +++ b/ml/optim/src/grad/gradient.rs @@ -9,6 +9,10 @@ pub type BiasGradient = Array1; pub type WeightGradient = Array2; +pub struct Grad { + +} + pub struct Gradients { pub bias: BiasGradient, pub weights: WeightGradient, diff --git a/ml/optim/src/primitives.rs b/ml/optim/src/primitives.rs index 008b50ce..0ecc612c 100644 --- a/ml/optim/src/primitives.rs +++ b/ml/optim/src/primitives.rs @@ -4,7 +4,11 @@ */ pub use self::{constants::*, statics::*, types::*}; -mod constants {} +mod constants { + + pub const FTOL: f64 = 2.220446049250313e-09; + +} mod statics {} diff --git a/ml/optim/src/specs.rs b/ml/optim/src/specs.rs index 09af9eb5..232a5544 100644 --- a/ml/optim/src/specs.rs +++ b/ml/optim/src/specs.rs @@ -16,6 +16,12 @@ pub trait Objective { fn objective(&self, x: &Array2, y: &Array1) -> Array1; } +pub trait PartialDerivative { + type Model; + + fn partial_derivative(&self, x: &Array2, y: &Array1) -> Array2; +} + pub trait Optimize { fn optimize(&self, params: &mut dyn Optimizable); } diff --git a/ml/optim/src/utils.rs b/ml/optim/src/utils.rs index 752dabaf..4b343177 100644 --- a/ml/optim/src/utils.rs +++ b/ml/optim/src/utils.rs @@ -2,3 +2,33 @@ Appellation: utils Contrib: FL03 */ +use ndarray::prelude::Array1; +use ndarray_stats::DeviationExt; + +pub fn minimize_inner(w: &mut Array1, fg: FG, epsilon: f64) -> anyhow::Result<(&mut Array1, f64, Array1)> +where + FG: Fn(&Array1) -> (f64, Array1), +{ + + let (mut fp, mut gp) = fg(&w); + + loop { + w.scaled_add(-epsilon, &gp); + let (f, g) = fg(&w); + + let expected_decrease = epsilon * norm_l2(&g); + let actual_decrease = fp - f; + if actual_decrease < expected_decrease * 0.5 { + return Err(anyhow::anyhow!("Not enough decrease")); + } + if actual_decrease < 2.220446049250313e-09 { + return Ok((w, f, g)); + } + fp = f; + gp = g; + } +} + +fn norm_l2(a_s: &Array1) -> f64 { + a_s.fold(0.0, |b, a| b + a * a) +} \ No newline at end of file diff --git a/ml/transformers/src/attention/multi/attention.rs b/ml/transformers/src/attention/multi/attention.rs index 753ea7bd..4f92de3f 100644 --- a/ml/transformers/src/attention/multi/attention.rs +++ b/ml/transformers/src/attention/multi/attention.rs @@ -7,7 +7,7 @@ use crate::attention::Weight; use crate::neural::layers::linear::LinearLayer; use crate::neural::prelude::Mask; use crate::ops::Split; -use ndarray::prelude::Array2; +use ndarray::prelude::{Array2, NdFloat}; use ndarray::{ScalarOperand, ShapeError}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; @@ -54,7 +54,7 @@ where impl MultiHeadAttention where - T: Float + ScalarOperand, + T: NdFloat + ScalarOperand, { pub fn attention(&self, data: &Array2, mask: &Mask) -> Result, ShapeError> { let weighted = data * self.weights(); From 88337b05e9de3ed771a0a9092174691273910a49 Mon Sep 17 00:00:00 2001 From: FL03 Date: Thu, 16 Nov 2023 08:19:48 -0600 Subject: [PATCH 051/118] update Signed-off-by: FL03 --- Cargo.toml | 1 + concision/Cargo.toml | 7 ++ concision/src/lib.rs | 4 + core/src/specs.rs | 102 ++------------------ math/Cargo.toml | 44 +++++++++ math/src/lib.rs | 20 ++++ math/src/primitives.rs | 11 +++ math/src/regress/linear.rs | 6 ++ math/src/regress/mod.rs | 12 +++ math/src/specs.rs | 90 ++++++++++++++++++ math/src/utils.rs | 23 +++++ math/tests/default.rs | 8 ++ ml/neural/src/layers/features.rs | 4 + ml/neural/src/layers/linear/regress.rs | 22 ++++- ml/neural/src/models/model.rs | 2 +- ml/neural/src/neurons/activate/mod.rs | 1 + ml/neural/src/neurons/mod.rs | 4 +- ml/neural/src/neurons/neuron.rs | 52 ++++++---- ml/neural/src/params/mod.rs | 32 +++++-- ml/neural/src/params/shapes.rs | 52 ++++++++++ ml/neural/src/params/weight.rs | 45 ++++++--- ml/optim/examples/descent.rs | 76 +++------------ ml/optim/src/grad/descent.rs | 127 +++++++------------------ ml/optim/src/grad/gradient.rs | 4 +- ml/optim/src/lib.rs | 6 +- ml/optim/src/norm/kinds.rs | 34 +++++++ ml/optim/src/norm/mod.rs | 71 ++++++++++++++ ml/optim/src/optimize/mod.rs | 23 +++++ ml/optim/src/optimize/optimizer.rs | 6 ++ ml/optim/src/optimizer/mod.rs | 1 - ml/optim/src/primitives.rs | 1 - ml/optim/src/specs.rs | 7 +- ml/optim/src/utils.rs | 28 +++--- 33 files changed, 611 insertions(+), 315 deletions(-) create mode 100644 math/Cargo.toml create mode 100644 math/src/lib.rs create mode 100644 math/src/primitives.rs create mode 100644 math/src/regress/linear.rs create mode 100644 math/src/regress/mod.rs create mode 100644 math/src/specs.rs create mode 100644 math/src/utils.rs create mode 100644 math/tests/default.rs create mode 100644 ml/neural/src/params/shapes.rs create mode 100644 ml/optim/src/norm/kinds.rs create mode 100644 ml/optim/src/norm/mod.rs create mode 100644 ml/optim/src/optimize/mod.rs create mode 100644 ml/optim/src/optimize/optimizer.rs delete mode 100644 ml/optim/src/optimizer/mod.rs diff --git a/Cargo.toml b/Cargo.toml index 27d9b960..e56dd7ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ members = [ "data", "derive", "macros", + "math", "ml/*", ] diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 9055457a..02d63ff7 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -15,6 +15,7 @@ version.workspace = true default = [ "core", "data", + "math", "ml", ] @@ -22,6 +23,7 @@ full = [ "core", "data", "derive", + "math", "ml", ] @@ -42,6 +44,10 @@ macros = [ "concision-macros" ] +math = [ + "concision-math" +] + ml = [ "neural", "nlp", @@ -78,6 +84,7 @@ concision-core = { features = [], optional = true, path = "../core", version = " concision-data = { features = [], optional = true, path = "../data", version = "0.1.12" } concision-derive = { features = [], optional = true, path = "../derive", version = "0.1.12" } concision-macros = { features = [], optional = true, path = "../macros", version = "0.1.12" } +concision-math = { features = [], optional = true, path = "../math", version = "0.1.12" } concision-neural = { features = [], optional = true, path = "../ml/neural", version = "0.1.12" } concision-nlp = { features = [], optional = true, path = "../ml/nlp", version = "0.1.12" } diff --git a/concision/src/lib.rs b/concision/src/lib.rs index 070877cd..c7ac51b8 100644 --- a/concision/src/lib.rs +++ b/concision/src/lib.rs @@ -15,6 +15,8 @@ pub use concision_data as data; pub use concision_derive::*; #[cfg(feature = "macros")] pub use concision_macros::*; +#[cfg(feature = "math")] +pub use concision_math as math; #[cfg(feature = "neural")] pub use concision_neural as neural; #[cfg(feature = "nlp")] @@ -33,6 +35,8 @@ pub mod prelude { pub use concision_derive::*; #[cfg(feature = "macros")] pub use concision_macros::*; + #[cfg(feature = "math")] + pub use concision_math::prelude::*; #[cfg(feature = "neural")] pub use concision_neural::prelude::*; #[cfg(feature = "nlp")] diff --git a/core/src/specs.rs b/core/src/specs.rs index a41c34cb..85eea3d3 100644 --- a/core/src/specs.rs +++ b/core/src/specs.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use ndarray::prelude::Array; -use ndarray::{Dimension, IntoDimension, NdFloat, ShapeError}; +use ndarray::{Dimension, IntoDimension}; // use ndarray::linalg::Dot; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Uniform}; @@ -12,8 +12,14 @@ use num::traits::NumOps; use num::{Float, Num, One, Zero}; use std::ops; +pub trait Borrowed: AsRef + AsMut {} + +impl Borrowed for S where S: AsRef + AsMut {} + pub trait BinaryNum: One + Zero {} +impl BinaryNum for T where T: One + Zero {} + pub trait NumOpsAssign: Num + NumOps + Sized + ops::AddAssign + ops::DivAssign + ops::MulAssign + ops::SubAssign { @@ -32,77 +38,6 @@ where } } -pub trait Product -where - T: Num, -{ - fn product(&self) -> T; -} - -impl Product for I -where - I: Clone + IntoIterator, - T: One + Num + ops::MulAssign, -{ - fn product(&self) -> T { - let mut res = T::one(); - for i in self.clone().into_iter() { - res *= i; - } - res - } -} - -trait Matmul -where - T: Num, - D: Dimension, -{ - fn matmul(&self, other: &Array) -> Result, ShapeError>; - - fn shape(&self) -> D; -} - -// impl Matmul for Array -// where -// T: Num + std::ops::Mul + std::ops::Add + Clone, -// D: Dimension, -// { -// fn matmul(&self, other: &Array) -> Result, ShapeError> { -// let self_shape = self.shape(); -// let other_shape = other.shape(); - -// if self_shape[self.ndim() - 1] != other_shape[self.ndim() - 2] { -// return Err(ShapeError::from_kind(ndarray::ErrorKind::IncompatibleShape)); -// } - -// let mut result = Array::zeros(self_shape); - -// let mut self_shape = self_shape.to_vec(); -// let self_last = self_shape.pop().unwrap(); -// let other_shape = other_shape.to_vec(); - -// let mut iter_self = self.iter(); -// let mut iter_other = other.iter(); - -// for mut row_result in result.outer_iter_mut() { -// for mut col_other in other.inner_iter() { -// let row_self = iter_self.clone(); -// let mut col_other = col_other.clone(); -// let dot = dot_product(&mut row_self, &mut col_other, self_last, &other_shape); -// row_result.assign(&dot); -// } -// iter_self.step_by(self_shape.last().unwrap().index()); -// } - -// Ok(result) -// } - -// fn shape(&self) -> D { -// self.raw_dim() -// } -// } - pub trait GenerateRandom where T: Float + SampleUniform, @@ -119,8 +54,8 @@ where fn uniform(axis: usize, dim: impl IntoDimension) -> Array { let dim = dim.into_dimension(); - let k = (T::one() / T::from(dim[axis]).unwrap()).sqrt(); - Array::random(dim, Uniform::new(-k, k)) + let dk = (T::one() / T::from(dim[axis]).unwrap()).sqrt(); + Self::uniform_between(dk, dim) } fn uniform_between(dk: T, dim: impl IntoDimension) -> Array { @@ -149,22 +84,3 @@ where Array::random(dim, Uniform::new(-k, k)) } } - -pub trait Mean {} - -pub trait Stats -where - T: NdFloat, -{ -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_product() { - let args = vec![2, 4, 6]; - assert_eq!(args.product(), 48); - } -} diff --git a/math/Cargo.toml b/math/Cargo.toml new file mode 100644 index 00000000..e4108012 --- /dev/null +++ b/math/Cargo.toml @@ -0,0 +1,44 @@ +[package] +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "concision-math" +readme.workspace = true +repository.workspace = true +version.workspace = true + +[features] +default = [] + +[lib] +bench = false +crate-type = ["cdylib", "rlib"] +doctest = false +test = true + +[build-dependencies] + +[dependencies] +anyhow.workspace = true +ndarray.workspace = true +ndarray-rand.workspace = true +ndarray-stats.workspace = true +num.workspace = true +serde.workspace = true +serde_json.workspace = true +smart-default.workspace = true +strum.workspace = true + +[dev-dependencies] + +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] + +[target.wasm32-unknown-unknown] + +[target.wasm32-wasi] diff --git a/math/src/lib.rs b/math/src/lib.rs new file mode 100644 index 00000000..9d0fb858 --- /dev/null +++ b/math/src/lib.rs @@ -0,0 +1,20 @@ +/* + Appellation: math + Contrib: FL03 +*/ +//! # Concision Math +pub use self::{primitives::*, specs::*, utils::*}; + +pub(crate) mod primitives; +pub(crate) mod specs; +pub(crate) mod utils; + +pub mod regress; + +pub mod prelude { + pub use crate::regress::*; + + pub use crate::primitives::*; + pub use crate::specs::*; + pub use crate::utils::*; +} diff --git a/math/src/primitives.rs b/math/src/primitives.rs new file mode 100644 index 00000000..859023bb --- /dev/null +++ b/math/src/primitives.rs @@ -0,0 +1,11 @@ +/* + Appellation: primitives + Contrib: FL03 +*/ +pub use self::{constants::*, statics::*, types::*}; + +mod constants {} + +mod statics {} + +mod types {} diff --git a/math/src/regress/linear.rs b/math/src/regress/linear.rs new file mode 100644 index 00000000..da279fa5 --- /dev/null +++ b/math/src/regress/linear.rs @@ -0,0 +1,6 @@ +/* + Appellation: linear + Contrib: FL03 +*/ + +pub struct LinearRegression {} diff --git a/math/src/regress/mod.rs b/math/src/regress/mod.rs new file mode 100644 index 00000000..5d4de7ed --- /dev/null +++ b/math/src/regress/mod.rs @@ -0,0 +1,12 @@ +/* + Appellation: epochs + Contrib: FL03 +*/ +pub use self::{linear::*, utils::*}; + +pub(crate) mod linear; + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests {} diff --git a/math/src/specs.rs b/math/src/specs.rs new file mode 100644 index 00000000..617284af --- /dev/null +++ b/math/src/specs.rs @@ -0,0 +1,90 @@ +/* + Appellation: specs + Contrib: FL03 +*/ +use ndarray::prelude::Array; +use ndarray::{Dimension, ShapeError}; +use num::{Num, One}; +use std::ops; + +pub trait Product +where + T: Num, +{ + fn product(&self) -> T; +} + +impl Product for I +where + I: Clone + IntoIterator, + T: One + Num + ops::MulAssign, +{ + fn product(&self) -> T { + let mut res = T::one(); + for i in self.clone().into_iter() { + res *= i; + } + res + } +} + +trait Matmul +where + T: Num, + D: Dimension, +{ + fn matmul(&self, other: &Array) -> Result, ShapeError>; + + fn shape(&self) -> D; +} + +// impl Matmul for Array +// where +// T: Num + std::ops::Mul + std::ops::Add + Clone, +// D: Dimension, +// { +// fn matmul(&self, other: &Array) -> Result, ShapeError> { +// let self_shape = self.shape(); +// let other_shape = other.shape(); + +// if self_shape[self.ndim() - 1] != other_shape[self.ndim() - 2] { +// return Err(ShapeError::from_kind(ndarray::ErrorKind::IncompatibleShape)); +// } + +// let mut result = Array::zeros(self_shape); + +// let mut self_shape = self_shape.to_vec(); +// let self_last = self_shape.pop().unwrap(); +// let other_shape = other_shape.to_vec(); + +// let mut iter_self = self.iter(); +// let mut iter_other = other.iter(); + +// for mut row_result in result.outer_iter_mut() { +// for mut col_other in other.inner_iter() { +// let row_self = iter_self.clone(); +// let mut col_other = col_other.clone(); +// let dot = dot_product(&mut row_self, &mut col_other, self_last, &other_shape); +// row_result.assign(&dot); +// } +// iter_self.step_by(self_shape.last().unwrap().index()); +// } + +// Ok(result) +// } + +// fn shape(&self) -> D { +// self.raw_dim() +// } +// } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_product() { + let args = vec![2, 4, 6]; + assert_eq!(args.product(), 48); + } +} diff --git a/math/src/utils.rs b/math/src/utils.rs new file mode 100644 index 00000000..c2e898e1 --- /dev/null +++ b/math/src/utils.rs @@ -0,0 +1,23 @@ +/* + Appellation: utils + Contrib: FL03 +*/ +use ndarray::linalg::Dot; +use ndarray::prelude::{Array, NdFloat}; +use ndarray::Dimension; +use num::FromPrimitive; + +pub fn covariance(ddof: T, x: &Array, y: &Array) -> anyhow::Result> +where + D: Dimension, + T: Default + FromPrimitive + NdFloat, + Array: Dot, Output = Array>, +{ + let x_mean = x.mean().unwrap_or_default(); + let y_mean = y.mean().unwrap_or_default(); + let xs = x - x_mean; + let ys = y - y_mean; + let cov = xs.dot(&ys.t().to_owned()); + let scale = T::one() / (T::from(x.len()).unwrap() - ddof); + Ok(cov * scale) +} diff --git a/math/tests/default.rs b/math/tests/default.rs new file mode 100644 index 00000000..0cac1eb5 --- /dev/null +++ b/math/tests/default.rs @@ -0,0 +1,8 @@ +#[cfg(test)] +#[test] +fn compiles() { + let f = |x: usize, y: usize| x + y; + + assert_eq!(f(10, 10), 20); + assert_ne!(f(1, 1), 3); +} diff --git a/ml/neural/src/layers/features.rs b/ml/neural/src/layers/features.rs index d876fe2b..dad4f0c8 100644 --- a/ml/neural/src/layers/features.rs +++ b/ml/neural/src/layers/features.rs @@ -18,6 +18,10 @@ impl Features { Self { inputs, outputs } } + pub fn neuron(inputs: usize) -> Self { + Self::new(inputs, 1) + } + pub fn uniform_scale(&self) -> T { (T::one() / T::from(self.inputs).unwrap()).sqrt() } diff --git a/ml/neural/src/layers/linear/regress.rs b/ml/neural/src/layers/linear/regress.rs index 0a9c549c..43fd9ed5 100644 --- a/ml/neural/src/layers/linear/regress.rs +++ b/ml/neural/src/layers/linear/regress.rs @@ -136,11 +136,12 @@ where data.dot(&self.weights().t()) + *self.bias() } - - pub fn apply_gradient(&mut self, gamma: T, gradient: impl Fn(&Array1) -> (T, Array1)) -> T { - let (cost, grad) = gradient(self.weights()); + pub fn apply_gradient(&mut self, gamma: T, gradient: G) + where + G: Fn(&Array1) -> Array1, + { + let grad = gradient(self.weights()); self.weights_mut().scaled_add(-gamma, &grad); - cost } pub fn update_with_gradient(&mut self, gamma: T, gradient: &Array1) { @@ -155,7 +156,18 @@ where type Output = Array1; fn forward(&self, data: &Array2) -> Self::Output { - data.dot(&self.weights().t().to_owned()) + self.bias().clone() + data.dot(&self.weights().t().to_owned()) + *self.bias() + } +} + +impl Forward> for Linear +where + T: NdFloat, +{ + type Output = T; + + fn forward(&self, data: &Array1) -> Self::Output { + data.dot(&self.weights().t().to_owned()) + *self.bias() } } diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs index 7a759a87..144daa4a 100644 --- a/ml/neural/src/models/model.rs +++ b/ml/neural/src/models/model.rs @@ -7,4 +7,4 @@ use ndarray::prelude::Array1; pub struct Model { pub bias: T, pub weights: Array1, -} \ No newline at end of file +} diff --git a/ml/neural/src/neurons/activate/mod.rs b/ml/neural/src/neurons/activate/mod.rs index 3ec13e5c..3e2fe344 100644 --- a/ml/neural/src/neurons/activate/mod.rs +++ b/ml/neural/src/neurons/activate/mod.rs @@ -13,6 +13,7 @@ pub(crate) mod nonlinear; pub type ActivationFn = fn(T) -> T; +#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct LinearActivation; impl LinearActivation { diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index 3763cbd9..bdb25b8b 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -14,7 +14,7 @@ pub(crate) mod utils {} #[cfg(test)] mod tests { - use super::activate::{softmax, Activate}; + use super::activate::{softmax, Activate, Softmax}; use super::*; // use lazy_static::lazy_static; use ndarray::{array, Array1}; @@ -37,7 +37,7 @@ mod tests { let data = array![[10.0, 10.0, 6.0, 1.0, 8.0]]; let weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; let neuron = Neuron::new(5) - .with_rho(softmax) + .with_rho(Softmax::default()) .with_weights(weights.clone()); let exp = softmax(data.clone().dot(&weights) + bias); diff --git a/ml/neural/src/neurons/neuron.rs b/ml/neural/src/neurons/neuron.rs index b3a28f74..7aec6619 100644 --- a/ml/neural/src/neurons/neuron.rs +++ b/ml/neural/src/neurons/neuron.rs @@ -2,31 +2,32 @@ Appellation: neuron Contrib: FL03 */ -use super::activate::{Activate, ActivationFn}; -use crate::core::GenerateRandom; +use super::activate::{Activate, ActivationFn, LinearActivation}; use crate::prelude::Forward; +use crate::{core::GenerateRandom, layers::L}; use ndarray::prelude::{Array1, Array2}; +pub trait ArtificialNeuron { + type Rho: Activate; +} + /// Artificial Neuron #[derive(Clone, Debug, PartialEq)] -pub struct Neuron { - activation: ActivationFn>, +pub struct Neuron +where + Rho: Activate>, +{ + activation: Rho, bias: f64, features: usize, weights: Array1, } -impl Neuron { - pub fn new(features: usize) -> Self { - Self { - activation: |x| x, - bias: 0.0, - features, - weights: Array1::zeros(features), - } - } - - pub fn with_rho(mut self, rho: ActivationFn>) -> Self { +impl Neuron +where + Rho: Activate>, +{ + pub fn with_rho(mut self, rho: Rho) -> Self { self.activation = rho; self } @@ -41,11 +42,12 @@ impl Neuron { } pub fn process(&self, args: &Array2) -> Array1 { - self.rho()(args.dot(&self.weights.t()) + self.bias()) + self.rho() + .activate(args.dot(&self.weights.t()) + self.bias()) } - pub fn rho(&self) -> ActivationFn> { - self.activation + pub fn rho(&self) -> &Rho { + &self.activation } pub fn weights(&self) -> &Array1 { @@ -75,6 +77,20 @@ impl Neuron { } } +impl Neuron +where + Rho: Activate> + Default, +{ + pub fn new(features: usize) -> Self { + Self { + activation: Rho::default(), + bias: 0.0, + features, + weights: Array1::zeros(features), + } + } +} + // impl Forward> for Neuron { // type Output = f64; diff --git a/ml/neural/src/params/mod.rs b/ml/neural/src/params/mod.rs index 75f49173..521e355c 100644 --- a/ml/neural/src/params/mod.rs +++ b/ml/neural/src/params/mod.rs @@ -6,18 +6,27 @@ //! //! ## Overview //! -pub use self::{bias::*, utils::*, weight::*}; +pub use self::{bias::*, shapes::*, utils::*, weight::*}; pub(crate) mod bias; +pub(crate) mod shapes; pub(crate) mod weight; +use crate::core::prelude::Borrowed; + use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Array2, Ix2}; +use ndarray::prelude::{Array, Array2, Ix2, NdFloat}; use ndarray::Dimension; use num::Float; -pub enum ParameterShapes { - Thick { features: usize, outputs: usize }, +pub trait WeightTensor +where + Array: Dot, Output = Array>, + D: Dimension, + T: NdFloat, +{ + fn weights(&self) -> &Array; + fn weights_mut(&mut self) -> &mut Array; } pub trait Parameter {} @@ -30,14 +39,19 @@ where fn bias_mut(&mut self) -> &mut Bias; } -pub trait W +impl WeightTensor for Array where - Self: Dot>, + Array: Borrowed> + Dot, Output = Array>, D: Dimension, - T: Float, + T: NdFloat, { - fn weights(&self) -> &Self; - fn weights_mut(&mut self) -> &Self; + fn weights(&self) -> &Array { + self.as_ref() + } + + fn weights_mut(&mut self) -> &mut Array { + self.as_mut() + } } pub trait Weighted diff --git a/ml/neural/src/params/shapes.rs b/ml/neural/src/params/shapes.rs new file mode 100644 index 00000000..14a3f3ac --- /dev/null +++ b/ml/neural/src/params/shapes.rs @@ -0,0 +1,52 @@ +/* + Appellation: shapes + Contrib: FL03 +*/ +use ndarray::{Dimension, IntoDimension}; +use serde::{Deserialize, Serialize}; +use strum::{EnumIs, EnumIter, EnumVariantNames}; + +pub trait LayerFeatures { + fn inputs(&self) -> usize; + fn outputs(&self) -> usize; +} + +pub trait NeuronFeatures { + fn inputs(&self) -> usize; +} + +impl LayerFeatures for ParameterShapes { + fn inputs(&self) -> usize { + match self { + ParameterShapes::Layer { inputs, .. } => *inputs, + ParameterShapes::Neuron { inputs } => *inputs, + } + } + + fn outputs(&self) -> usize { + match self { + ParameterShapes::Layer { outputs, .. } => *outputs, + ParameterShapes::Neuron { .. } => 1, + } + } +} + +#[derive( + Clone, + Copy, + Debug, + Deserialize, + EnumIs, + EnumIter, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +pub enum ParameterShapes { + Layer { inputs: usize, outputs: usize }, + Neuron { inputs: usize }, +} diff --git a/ml/neural/src/params/weight.rs b/ml/neural/src/params/weight.rs index e8a41294..f1d65959 100644 --- a/ml/neural/src/params/weight.rs +++ b/ml/neural/src/params/weight.rs @@ -2,10 +2,9 @@ Appellation: weight Contrib: FL03 */ +use crate::core::prelude::GenerateRandom; use ndarray::prelude::Array2; use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::Uniform; -use ndarray_rand::RandomExt; use num::Float; use serde::{Deserialize, Serialize}; @@ -18,11 +17,24 @@ pub struct Weight { impl Weight where - T: Default, + T: Float, { - pub fn new(m: usize, n: usize) -> Self { - let weights = Array2::default((m, n)); - Self { weights } + pub fn new(inputs: usize, outputs: Option) -> Self { + Self { + weights: Array2::zeros((outputs.unwrap_or(1), inputs)), + } + } + /// Returns the shape of the weights. (outputs, inputs) + pub fn shape(&self) -> (usize, usize) { + self.weights.dim() + } + + pub fn inputs(&self) -> usize { + self.shape().1 + } + + pub fn outputs(&self) -> usize { + self.shape().0 } } @@ -30,10 +42,21 @@ impl Weight where T: Float + SampleUniform, { - pub fn uniform(m: usize, n: usize) -> Array2 { - let k = T::one() / T::from(m).unwrap(); - let dk = k.sqrt(); - let dist = Uniform::new(-dk, dk); - Array2::random((m, n), dist) + pub fn init(mut self) -> Self { + let dk = (T::one() / T::from(self.inputs()).unwrap()).sqrt(); + self.weights = Array2::uniform_between(dk, self.shape()); + self + } +} + +impl AsRef> for Weight { + fn as_ref(&self) -> &Array2 { + &self.weights + } +} + +impl AsMut> for Weight { + fn as_mut(&mut self) -> &mut Array2 { + &mut self.weights } } diff --git a/ml/optim/examples/descent.rs b/ml/optim/examples/descent.rs index b3547283..ba2a69d2 100644 --- a/ml/optim/examples/descent.rs +++ b/ml/optim/examples/descent.rs @@ -1,10 +1,7 @@ -use concision_neural::layers::linear; -use concision_neural::prelude::{Features, Linear, Neuron}; +use concision_neural::prelude::{Features, Linear}; use concision_neural::prop::Forward; -use concision_optim::prelude::{ - gradient_descent, gradient_descent_node, gradient_descent_step, GradientDescent, -}; -use ndarray::prelude::{Array, s}; +use concision_optim::prelude::{gradient_descent, GradientDescent}; +use ndarray::prelude::{Array, Array1}; fn main() -> anyhow::Result<()> { let (samples, inputs) = (20, 5); @@ -12,26 +9,18 @@ fn main() -> anyhow::Result<()> { let features = Features::new(inputs, outputs); - let n = samples * inputs; + let _n = samples * inputs; - let (epochs, gamma) = (10, 0.5); + let (epochs, gamma) = (500, 0.01); // basic_descent(epochs, features, gamma)?; sample_descent(epochs, features, gamma)?; - // sample_node(epochs, features.outputs(), gamma)?; - // sample_steps(epochs, features.outputs(), gamma)?; Ok(()) } pub fn basic_descent(epochs: usize, features: Features, gamma: f64) -> anyhow::Result<()> { - let (samples, inputs) = (20, features.inputs()); - let n = samples * inputs; - - let x = Array::linspace(1., n as f64, n) - .into_shape((samples, inputs)) - .unwrap(); let mut model = Linear::new(features.inputs()).init_weight(); println!( @@ -54,59 +43,20 @@ pub fn sample_descent(epochs: usize, features: Features, gamma: f64) -> anyhow:: .unwrap(); let model = Linear::new(features.inputs()).init_weight(); - println!( - "Initial Prediction: {:?}", &y - model.forward(&x) + "Targets:\n\n{:?}\nPredictions:\n\n{:?}\n", + &y, + model.forward(&x) ); let mut grad = GradientDescent::new(gamma, model); - + let mut losses = Array1::zeros(epochs); for e in 0..epochs { - let cost = grad.descent(&x, &y); - println!("Step ({}) Cost {:?}", e, cost); + let cost = grad.logit(&x, &y)?; + losses[e] = cost; } - println!("Model:\n\nWeights:\n{:?}", grad.model().weights()); - Ok(()) -} - -pub fn sample_node(epochs: usize, features: usize, gamma: f64) -> anyhow::Result<()> { - let mut node = Neuron::new(features).init_weights(); - let (samples, inputs) = (20, features); - let n = samples * inputs; + println!("Losses:\n\n{:?}\n", &losses); - // Generate some example data - let x = Array::linspace(1., n as f64, n) - .into_shape((samples, inputs)) - .unwrap(); - let y = Array::linspace(1., n as f64, samples) - .into_shape(samples) - .unwrap(); - - for e in 0..epochs { - let cost = gradient_descent_node(gamma, &mut node, &x, &y); - println!("Step ({}) Cost {:?}", e, cost); - } - Ok(()) -} - -// pub fn sample_descent(gamma: f64, inputs: usize, outputs: usize, ) - -pub fn sample_steps(epochs: usize, features: usize, gamma: f64) -> anyhow::Result<()> { - let mut model = linear::Linear::new(features); - let (samples, inputs) = (20, features); - let n = samples * inputs; - - // Generate some example data - let x = Array::linspace(1., n as f64, n) - .into_shape((samples, inputs)) - .unwrap(); - let y = Array::linspace(1., n as f64, samples) - .into_shape(samples) - .unwrap(); - - for e in 0..epochs { - let cost = gradient_descent_step(&x, &y, &mut model, gamma); - println!("Step ({}) Cost {:?}", e, cost); - } + println!("Trained:\n\n{:?}", grad.model().forward(&x)); Ok(()) } diff --git a/ml/optim/src/grad/descent.rs b/ml/optim/src/grad/descent.rs index 0b8c8294..d5c1c3bc 100644 --- a/ml/optim/src/grad/descent.rs +++ b/ml/optim/src/grad/descent.rs @@ -3,8 +3,9 @@ Contrib: FL03 */ use crate::neural::layers::linear::Linear; -use crate::neural::prelude::{mse, Forward, Neuron}; -use ndarray::prelude::{Array1, Array2, Axis}; +use crate::neural::prelude::{mse, Forward}; +use crate::prelude::Normalize; +use ndarray::prelude::{Array1, Array2}; use ndarray_stats::DeviationExt; pub fn cost(target: &Array1, prediction: &Array1) -> f64 { @@ -14,15 +15,12 @@ pub fn cost(target: &Array1, prediction: &Array1) -> f64 { .unwrap_or_default() } - pub fn grad(data: &Array2, target: &Array1, prediction: &Array1) -> Array1 { let error = prediction - target; let scale = -2.0 / data.len() as f64; scale * data.t().dot(&error) } - - pub fn gradient_descent( weights: &mut Array1, epochs: usize, @@ -37,46 +35,6 @@ pub fn gradient_descent( losses } -pub fn gradient_descent_node( - gamma: f64, - node: &mut Neuron, - x: &Array2, - y: &Array1, -) -> f64 { - let pred = node.forward(x); - let error = pred.clone() - y; - let scale = -2.0 / x.len() as f64; - let wg = scale * x.t().dot(&error); - node.apply_weight_gradient(gamma, &wg); - node.forward(x).mean_sq_err(y).unwrap_or_default() -} - -pub fn gradient_descent_step( - x: &Array2, - y: &Array1, - model: &mut Linear, - learning_rate: f64, -) -> f64 { - let (samples, _inputs) = x.dim(); - // forward the provided data to the model - let predictions = model.predict(x); // fully connected - println!("Predictions (dim): {:?}", &predictions.shape()); - // calculate the error - let errors = &predictions - y; - println!("Errors (dim): {:?}", &errors.shape()); - // calculate a scaling factor - let scale = -2.0 / samples as f64; - let _bias = scale * &errors; - // calculate the gradient of the weights - let weights = scale * x.t().dot(&errors); - // let wg = (scale * x.t().dot(&errors)).sum_axis(Axis(1)); - - println!("Weights (dim): {:?}", &weights.shape()); - model.update_with_gradient(learning_rate, &weights.t().to_owned()); - // model.apply_gradient(&wg, learning_rate); - predictions.mean_sq_err(y).unwrap_or_default() -} - pub type BaseObjective = fn(&Array1) -> Array1; #[derive(Clone)] @@ -124,32 +82,42 @@ impl GradientDescent { self } - pub fn descent(&mut self, data: &Array2, targets: &Array1) -> anyhow::Result { - let lr = self.gamma; - let pred = self.model.forward(data); - let scale = -1.0 / data.len() as f64; - // let errors = targets - &pred; - // println!("Errors (dim): {:?}", &errors.shape()); + pub fn logit(&mut self, data: &Array2, targets: &Array1) -> anyhow::Result { let gradient = |p: &Array1| { let pred = data.dot(p); - let dist = targets.l2_dist(&pred).expect("L2 distance"); + let error = targets - &pred; + let scale = -1.0 / (2.0 * data.len() as f64); + let grad = scale * error.dot(data); - let errors = targets - &pred; - (dist, p.clone()) + &grad / grad.l2() }; - let c = self.model.apply_gradient(self.gamma, &gradient); - - println!("Dist: {:?}", self.model().weights() - gradient(self.model().weights()).1 * self.gamma() ); + self.model.apply_gradient(self.gamma, &gradient); + let loss = targets.mean_sq_err(&self.model.forward(data))?; Ok(loss) } + pub fn descent(&mut self, data: &Array2, targets: &Array1) -> anyhow::Result { + let gradient = |p: &Array1| { + let pred = data.dot(p); + let error = targets - pred; + let scale = -2.0 / data.len() as f64; + let grad = scale * data.t().dot(&error); + + &grad / grad.l2() + }; + self.model.apply_gradient(self.gamma, &gradient); + + // let loss = targets.mean_sq_err(&self.model.forward(data))?; + let loss = targets.mean_sq_err(&self.model.forward(data))?; + Ok(loss) + } pub fn gradient(&self, x: &Array2, y: &Array1) -> Array1 { let (samples, _inputs) = x.dim(); - let predictions = x.dot(&self.model.weights().t()) + *self.model.bias(); // fully connected - let errors = y - &predictions; + let pred = x.dot(&self.model.weights().t()) + *self.model.bias(); // fully connected + let errors = y - &pred; let scale = -2.0 / samples as f64; scale * x.t().dot(&errors) } @@ -172,51 +140,30 @@ impl GradientDescent { #[cfg(test)] mod tests { use super::*; - use ndarray::prelude::Array; + use ndarray::prelude::{Array, Array1, Array2}; - #[test] - fn test_descent() { - let (samples, inputs) = (20, 5); - let outputs = 3; + fn sample_data(samples: usize, inputs: usize) -> (Array2, Array1) { let n = samples * inputs; - - let (_epochs, gamma) = (1, 0.01); - // Generate some example data let x = Array::linspace(1., n as f64, n) .into_shape((samples, inputs)) .unwrap(); - let y = Array::linspace(1., n as f64, outputs) - .into_shape(outputs) + let y = Array::linspace(1., samples as f64, samples) + .into_shape(samples) .unwrap(); - - let model = Linear::new(outputs).init_weight(); - let mut grad = GradientDescent::new(gamma, model); - - let _s = grad.step(&x, &y); - - // sgd(&x, &y, &mut model, learning_rate, epochs, batch_size); + (x, y) } #[test] - fn test_gradient_descent() { + fn test_descent() { let (samples, inputs) = (20, 5); - let outputs = 5; - let n = samples * inputs; let (_epochs, gamma) = (1, 0.01); // Generate some example data - let x = Array::linspace(1., n as f64, n) - .into_shape((samples, inputs)) - .unwrap(); - let y = Array::linspace(1., n as f64, samples) - .into_shape(samples) - .unwrap(); + let (x, y) = sample_data(samples, inputs); - // let mut model = LinearLayer::::new_biased(inputs, outputs); - let mut model = Linear::new(outputs); - - let _grad = gradient_descent_step(&x, &y, &mut model, gamma); + let model = Linear::new(inputs).init_weight(); + let mut grad = GradientDescent::new(gamma, model); - // sgd(&x, &y, &mut model, learning_rate, epochs, batch_size); + let _s = grad.descent(&x, &y); } } diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs index e183505f..2cac6e63 100644 --- a/ml/optim/src/grad/gradient.rs +++ b/ml/optim/src/grad/gradient.rs @@ -9,9 +9,7 @@ pub type BiasGradient = Array1; pub type WeightGradient = Array2; -pub struct Grad { - -} +pub struct Grad {} pub struct Gradients { pub bias: BiasGradient, diff --git a/ml/optim/src/lib.rs b/ml/optim/src/lib.rs index 4a8b2031..1bef12b9 100644 --- a/ml/optim/src/lib.rs +++ b/ml/optim/src/lib.rs @@ -14,10 +14,14 @@ pub(crate) use concision_neural as neural; pub mod cost; pub mod grad; -pub mod optimizer; +pub mod norm; +pub mod optimize; pub mod prelude { + pub use crate::cost::*; pub use crate::grad::*; + pub use crate::norm::*; + pub use crate::optimize::*; pub use crate::primitives::*; pub use crate::specs::*; diff --git a/ml/optim/src/norm/kinds.rs b/ml/optim/src/norm/kinds.rs new file mode 100644 index 00000000..2b1d3cad --- /dev/null +++ b/ml/optim/src/norm/kinds.rs @@ -0,0 +1,34 @@ +/* + Appellation: kinds + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum Norms { + L0, + L1, + #[default] + L2, +} diff --git a/ml/optim/src/norm/mod.rs b/ml/optim/src/norm/mod.rs new file mode 100644 index 00000000..54336762 --- /dev/null +++ b/ml/optim/src/norm/mod.rs @@ -0,0 +1,71 @@ +/* + Appellation: norm + Contrib: FL03 +*/ +//! # norm +//! +pub use self::{kinds::*, utils::*}; + +pub(crate) mod kinds; + +use ndarray::prelude::{Array, NdFloat}; +use ndarray::Dimension; + +pub trait Normalize { + fn l0(&self) -> T; + + fn l1(&self) -> T; + + fn l2(&self) -> T; +} + +impl Normalize for Array +where + D: Dimension, + T: NdFloat, +{ + fn l0(&self) -> T { + utils::l0_norm(self) + } + + fn l1(&self) -> T { + utils::l1_norm(self) + } + + fn l2(&self) -> T { + utils::l2_norm(self) + } +} + +pub(crate) mod utils { + use ndarray::prelude::{Array, NdFloat}; + use ndarray::Dimension; + use ndarray_stats::QuantileExt; + + pub fn l0_norm(args: &Array) -> T + where + D: Dimension, + T: NdFloat, + { + *args.max().expect("No max value") + } + + pub fn l1_norm(args: &Array) -> T + where + D: Dimension, + T: NdFloat, + { + args.mapv(|xs| xs.abs()).sum() + } + + pub fn l2_norm(args: &Array) -> T + where + D: Dimension, + T: NdFloat, + { + args.mapv(|xs| xs.powi(2)).sum().sqrt() + } +} + +#[cfg(test)] +mod tests {} diff --git a/ml/optim/src/optimize/mod.rs b/ml/optim/src/optimize/mod.rs new file mode 100644 index 00000000..03482965 --- /dev/null +++ b/ml/optim/src/optimize/mod.rs @@ -0,0 +1,23 @@ +/* + Appellation: optimize + Contrib: FL03 +*/ +//! # optimize +//! +pub use self::{optimizer::*, utils::*}; + +pub(crate) mod optimizer; + +pub trait Optimize { + fn optimize(&self) -> Self; +} + +pub trait Optim: Iterator { + fn params(&self) -> &T; + fn optimize(&self) -> Self; +} + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests {} diff --git a/ml/optim/src/optimize/optimizer.rs b/ml/optim/src/optimize/optimizer.rs new file mode 100644 index 00000000..311b3e85 --- /dev/null +++ b/ml/optim/src/optimize/optimizer.rs @@ -0,0 +1,6 @@ +/* + Appellation: optimizer + Contrib: FL03 +*/ + +pub struct Optimizer {} diff --git a/ml/optim/src/optimizer/mod.rs b/ml/optim/src/optimizer/mod.rs deleted file mode 100644 index 8b137891..00000000 --- a/ml/optim/src/optimizer/mod.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/ml/optim/src/primitives.rs b/ml/optim/src/primitives.rs index 0ecc612c..01b073a1 100644 --- a/ml/optim/src/primitives.rs +++ b/ml/optim/src/primitives.rs @@ -7,7 +7,6 @@ pub use self::{constants::*, statics::*, types::*}; mod constants { pub const FTOL: f64 = 2.220446049250313e-09; - } mod statics {} diff --git a/ml/optim/src/specs.rs b/ml/optim/src/specs.rs index 232a5544..9198b4b5 100644 --- a/ml/optim/src/specs.rs +++ b/ml/optim/src/specs.rs @@ -5,6 +5,7 @@ use ndarray::prelude::{Array1, Array2}; pub trait Gradient { + type Params; type Model; fn gradient(&self, x: &Array2, y: &Array1) -> T; @@ -22,8 +23,6 @@ pub trait PartialDerivative { fn partial_derivative(&self, x: &Array2, y: &Array1) -> Array2; } -pub trait Optimize { - fn optimize(&self, params: &mut dyn Optimizable); +pub trait Minimize { + fn minimize(&self, scale: T) -> Self; } - -pub trait Optimizable {} diff --git a/ml/optim/src/utils.rs b/ml/optim/src/utils.rs index 4b343177..17e3f332 100644 --- a/ml/optim/src/utils.rs +++ b/ml/optim/src/utils.rs @@ -2,26 +2,28 @@ Appellation: utils Contrib: FL03 */ +use crate::prelude::FTOL; use ndarray::prelude::Array1; -use ndarray_stats::DeviationExt; -pub fn minimize_inner(w: &mut Array1, fg: FG, epsilon: f64) -> anyhow::Result<(&mut Array1, f64, Array1)> +pub fn minimize_inner( + w: &mut Array1, + fg: F, + epsilon: f64, +) -> anyhow::Result<(&mut Array1, f64, Array1)> where - FG: Fn(&Array1) -> (f64, Array1), + F: Fn(&Array1) -> (f64, Array1), { - - let (mut fp, mut gp) = fg(&w); + let (mut fp, mut gp) = fg(&w); // (cost, gradient) loop { w.scaled_add(-epsilon, &gp); let (f, g) = fg(&w); - let expected_decrease = epsilon * norm_l2(&g); - let actual_decrease = fp - f; - if actual_decrease < expected_decrease * 0.5 { - return Err(anyhow::anyhow!("Not enough decrease")); - } - if actual_decrease < 2.220446049250313e-09 { + let exp = epsilon * norm_l2(&g); + let delta = fp - f; // actual descrease; last - current + if delta < exp * 0.5 { + return Err(anyhow::anyhow!("Not enough decrease")); + } else if delta < FTOL { return Ok((w, f, g)); } fp = f; @@ -29,6 +31,6 @@ where } } -fn norm_l2(a_s: &Array1) -> f64 { +pub fn norm_l2(a_s: &Array1) -> f64 { a_s.fold(0.0, |b, a| b + a * a) -} \ No newline at end of file +} From 7fa6b87116cc1b9979f2ab0b2e3053ded8e9b03a Mon Sep 17 00:00:00 2001 From: FL03 Date: Thu, 16 Nov 2023 13:19:50 -0600 Subject: [PATCH 052/118] update Signed-off-by: FL03 --- Cargo.toml | 1 + bin/cli/Cargo.toml | 16 ++ bin/cli/src/main.rs | 3 + bin/concise/Cargo.toml | 34 ++++ bin/concise/src/main.rs | 3 + core/src/primitives.rs | 3 + core/src/specs.rs | 9 +- core/src/utils.rs | 23 ++- data/Cargo.toml | 1 + data/src/datasets/dataset.rs | 38 +++++ data/src/datasets/mod.rs | 10 ++ data/src/lib.rs | 2 + data/src/specs.rs | 19 +++ math/src/specs.rs | 8 + ml/neural/src/layers/features.rs | 26 +--- ml/neural/src/layers/layer.rs | 4 +- ml/neural/src/layers/linear/layer.rs | 57 +++---- ml/neural/src/layers/linear/mod.rs | 33 ++-- ml/neural/src/layers/linear/regress.rs | 16 +- ml/neural/src/layers/sublayer.rs | 20 ++- ml/neural/src/models/features.rs | 115 ++++++++++++++ ml/neural/src/models/mod.rs | 12 +- ml/neural/src/models/model.rs | 63 +++++++- ml/neural/src/models/params.rs | 132 ++++++++++++++++ ml/neural/src/neurons/mod.rs | 3 +- ml/neural/src/neurons/neuron.rs | 145 ++++++++++++------ ml/neural/src/nn/network.rs | 4 +- ml/neural/src/ops/mod.rs | 1 + ml/neural/src/ops/norm.rs | 69 +++++++-- ml/neural/src/params/mod.rs | 11 ++ ml/neural/src/params/shapes.rs | 44 +++++- ml/neural/src/primitives.rs | 2 +- ml/optim/examples/descent.rs | 2 +- ml/optim/examples/norm.rs | 32 ++++ ml/optim/examples/sgd.rs | 2 +- ml/optim/src/grad/descent.rs | 52 +------ ml/optim/src/grad/gradient.rs | 62 ++++++-- ml/optim/src/grad/sgd.rs | 4 +- ml/optim/src/norm/kinds.rs | 44 +++++- ml/optim/src/norm/mod.rs | 29 +++- ml/optim/src/norm/normalizer.rs | 40 +++++ .../src/attention/multi/attention.rs | 6 +- ml/transformers/src/ffn/network.rs | 4 +- 43 files changed, 932 insertions(+), 272 deletions(-) create mode 100644 bin/cli/Cargo.toml create mode 100644 bin/cli/src/main.rs create mode 100644 bin/concise/Cargo.toml create mode 100644 bin/concise/src/main.rs create mode 100644 data/src/datasets/dataset.rs create mode 100644 data/src/datasets/mod.rs create mode 100644 ml/neural/src/models/features.rs create mode 100644 ml/neural/src/models/params.rs create mode 100644 ml/optim/examples/norm.rs create mode 100644 ml/optim/src/norm/normalizer.rs diff --git a/Cargo.toml b/Cargo.toml index e56dd7ea..116fe71e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ default-members = [ ] members = [ + "bin/*", "concision", "core", "data", diff --git a/bin/cli/Cargo.toml b/bin/cli/Cargo.toml new file mode 100644 index 00000000..ea94c7a1 --- /dev/null +++ b/bin/cli/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "cli" +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +version.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/bin/cli/src/main.rs b/bin/cli/src/main.rs new file mode 100644 index 00000000..e7a11a96 --- /dev/null +++ b/bin/cli/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/bin/concise/Cargo.toml b/bin/concise/Cargo.toml new file mode 100644 index 00000000..e3d97a42 --- /dev/null +++ b/bin/concise/Cargo.toml @@ -0,0 +1,34 @@ +[package] +authors.workspace = true +categories.workspace = true +default-run = "concise" +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "concise" +readme.workspace = true +repository.workspace = true +version.workspace = true + +[[bin]] +name = "concise" +path = "src/main.rs" + +[build-dependencies] + +[dependencies] +anyhow.workspace = true +concision = { features = ["full"], path = "../../concision", version = "0.1.12" } +ndarray.workspace = true + +[dev-dependencies] + +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] + +[target.wasm32-unknown-unknown] + +[target.wasm32-wasi] \ No newline at end of file diff --git a/bin/concise/src/main.rs b/bin/concise/src/main.rs new file mode 100644 index 00000000..e7a11a96 --- /dev/null +++ b/bin/concise/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/core/src/primitives.rs b/core/src/primitives.rs index 64c09104..cf61ef2d 100644 --- a/core/src/primitives.rs +++ b/core/src/primitives.rs @@ -16,4 +16,7 @@ mod types { pub type BoxError = Box; /// pub type BoxResult = std::result::Result; + + /// + pub type ShapeResult = std::result::Result; } diff --git a/core/src/specs.rs b/core/src/specs.rs index 85eea3d3..d4aa3e3e 100644 --- a/core/src/specs.rs +++ b/core/src/specs.rs @@ -8,9 +8,7 @@ use ndarray::{Dimension, IntoDimension}; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Uniform}; use ndarray_rand::RandomExt; -use num::traits::NumOps; -use num::{Float, Num, One, Zero}; -use std::ops; +use num::{Float, One, Zero}; pub trait Borrowed: AsRef + AsMut {} @@ -20,11 +18,6 @@ pub trait BinaryNum: One + Zero {} impl BinaryNum for T where T: One + Zero {} -pub trait NumOpsAssign: - Num + NumOps + Sized + ops::AddAssign + ops::DivAssign + ops::MulAssign + ops::SubAssign -{ -} - pub trait Pair { fn pair(&self) -> (A, B); } diff --git a/core/src/utils.rs b/core/src/utils.rs index 71d1c06e..b72e6324 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -2,10 +2,10 @@ Appellation: utils Contrib: FL03 */ -use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Axis, NdFloat}; -use ndarray::{concatenate, Dimension, RemoveAxis}; -use num::FromPrimitive; + +use ndarray::prelude::{Array, Axis, Dimension}; +use ndarray::{concatenate, IntoDimension, RemoveAxis, ShapeError}; +use num::Float; pub fn concat_iter(axis: usize, iter: impl IntoIterator>) -> Array where @@ -20,19 +20,14 @@ where out } -pub fn covariance(ddof: T, x: &Array, y: &Array) -> anyhow::Result> +pub fn linarr(dim: impl IntoDimension) -> Result, ShapeError> where D: Dimension, - T: Default + FromPrimitive + NdFloat, - Array: Dot, Output = Array>, + T: Float, { - let x_mean = x.mean().unwrap_or_default(); - let y_mean = y.mean().unwrap_or_default(); - let xs = x - x_mean; - let ys = y - y_mean; - let cov = xs.dot(&ys.t().to_owned()); - let scale = T::one() / (T::from(x.len()).unwrap() - ddof); - Ok(cov * scale) + let dim = dim.into_dimension(); + let n = dim.as_array_view().product(); + Array::linspace(T::one(), T::from(n).unwrap(), n).into_shape(dim) } pub fn now() -> u128 { diff --git a/data/Cargo.toml b/data/Cargo.toml index df4ae40c..e78970c2 100644 --- a/data/Cargo.toml +++ b/data/Cargo.toml @@ -25,6 +25,7 @@ test = true [dependencies] anyhow.workspace = true +linfa = { features = [], version = "0.7" } ndarray.workspace = true num.workspace = true serde.workspace = true diff --git a/data/src/datasets/dataset.rs b/data/src/datasets/dataset.rs new file mode 100644 index 00000000..acff96d8 --- /dev/null +++ b/data/src/datasets/dataset.rs @@ -0,0 +1,38 @@ +/* + Appellation: dataset + Contrib: FL03 +*/ +use ndarray::prelude::{Array, Array2, Ix2}; +use ndarray::Dimension; +use num::Float; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub struct Dataset +where + D: Dimension, +{ + data: Array2, + targets: Array, +} + +impl Dataset +where + D: Dimension, + T: Float, +{ + pub fn new(data: Array2, targets: Array) -> Self { + Self { data, targets } + } +} + +impl std::fmt::Display for Dataset +where + D: Dimension + Serialize, + T: Serialize, +{ + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", serde_json::to_string(self).unwrap()) + } +} diff --git a/data/src/datasets/mod.rs b/data/src/datasets/mod.rs new file mode 100644 index 00000000..f88ec742 --- /dev/null +++ b/data/src/datasets/mod.rs @@ -0,0 +1,10 @@ +/* + Appellation: datasets + Contrib: FL03 +*/ +//! # Dataset +pub use self::{dataset::*, utils::*}; + +pub(crate) mod dataset; + +pub(crate) mod utils {} diff --git a/data/src/lib.rs b/data/src/lib.rs index f736b907..c7833060 100644 --- a/data/src/lib.rs +++ b/data/src/lib.rs @@ -9,11 +9,13 @@ pub(crate) mod primitives; pub(crate) mod specs; pub(crate) mod utils; +pub mod datasets; pub mod df; pub mod flows; pub mod tensors; pub mod prelude { + pub use crate::datasets::*; pub use crate::df::*; pub use crate::flows::*; pub use crate::tensors::*; diff --git a/data/src/specs.rs b/data/src/specs.rs index 1d8faa71..8b000db7 100644 --- a/data/src/specs.rs +++ b/data/src/specs.rs @@ -2,3 +2,22 @@ Appellation: specs Contrib: FL03 */ + +pub trait Records { + fn features(&self) -> usize; + + fn samples(&self) -> usize; +} + +impl Records for S +where + S: AsRef<(usize, usize)>, +{ + fn features(&self) -> usize { + self.as_ref().1 + } + + fn samples(&self) -> usize { + self.as_ref().0 + } +} diff --git a/math/src/specs.rs b/math/src/specs.rs index 617284af..d69eb691 100644 --- a/math/src/specs.rs +++ b/math/src/specs.rs @@ -7,6 +7,14 @@ use ndarray::{Dimension, ShapeError}; use num::{Num, One}; use std::ops; +pub trait NumOpsAssign: + Sized + ops::AddAssign + ops::DivAssign + ops::MulAssign + ops::SubAssign +{ +} + +impl NumOpsAssign for T where T: ops::AddAssign + ops::DivAssign + ops::MulAssign + ops::SubAssign +{} + pub trait Product where T: Num, diff --git a/ml/neural/src/layers/features.rs b/ml/neural/src/layers/features.rs index dad4f0c8..236cb41d 100644 --- a/ml/neural/src/layers/features.rs +++ b/ml/neural/src/layers/features.rs @@ -23,7 +23,7 @@ impl Features { } pub fn uniform_scale(&self) -> T { - (T::one() / T::from(self.inputs).unwrap()).sqrt() + (T::one() / T::from(self.inputs()).unwrap()).sqrt() } pub fn inputs(&self) -> usize { @@ -34,24 +34,6 @@ impl Features { self.outputs } - pub fn set_inputs(&mut self, inputs: usize) { - self.inputs = inputs; - } - - pub fn set_outputs(&mut self, outputs: usize) { - self.outputs = outputs; - } - - pub fn with_inputs(mut self, inputs: usize) -> Self { - self.inputs = inputs; - self - } - - pub fn with_outputs(mut self, outputs: usize) -> Self { - self.outputs = outputs; - self - } - pub fn in_by_out(&self) -> (usize, usize) { (self.inputs, self.outputs) } @@ -113,3 +95,9 @@ impl From<(usize, usize)> for Features { Self { inputs, outputs } } } + +impl From for Features { + fn from(inputs: usize) -> Self { + Self { inputs, outputs: 1 } + } +} diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 62b97458..e7b0959d 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -2,9 +2,9 @@ Appellation: layer Contrib: FL03 */ -use super::{Features, LayerType}; +use super::LayerType; use crate::params::Bias; -use crate::prop::Forward; +use crate::prelude::{Features, Forward}; use ndarray::prelude::Array2; use ndarray::ScalarOperand; diff --git a/ml/neural/src/layers/linear/layer.rs b/ml/neural/src/layers/linear/layer.rs index 0cdc7eb3..5fca892b 100644 --- a/ml/neural/src/layers/linear/layer.rs +++ b/ml/neural/src/layers/linear/layer.rs @@ -3,8 +3,7 @@ Contrib: FL03 */ use crate::core::prelude::GenerateRandom; -use crate::layers::Features; -use crate::prelude::{Bias, Forward}; +use crate::prelude::{Features, Forward}; use ndarray::linalg::Dot; use ndarray::prelude::{Array, Array1, Array2, NdFloat}; @@ -16,7 +15,7 @@ use std::ops::Add; #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] pub struct LinearLayer { - bias: Bias, + bias: Array1, pub features: Features, weights: Array2, } @@ -25,19 +24,26 @@ impl LinearLayer where T: Float, { - pub fn new(features: Features) -> Self { + pub fn new(inputs: usize, outputs: usize) -> Self { Self { - bias: Array1::zeros(features.outputs()).into(), + bias: Array1::zeros(outputs), + features: Features::new(inputs, outputs), + weights: Array2::zeros((inputs, outputs)), + } + } + pub fn from_features(features: Features) -> Self { + Self { + bias: Array1::zeros(features.outputs()), features, weights: Array2::zeros(features.out_by_in()), } } - pub fn bias(&self) -> &Bias { + pub fn bias(&self) -> &Array1 { &self.bias } - pub fn bias_mut(&mut self) -> &mut Bias { + pub fn bias_mut(&mut self) -> &mut Array1 { &mut self.bias } @@ -57,7 +63,7 @@ where &mut self.weights } - pub fn set_bias(&mut self, bias: Bias) { + pub fn set_bias(&mut self, bias: Array1) { self.bias = bias; } @@ -80,30 +86,16 @@ where T: Float + SampleUniform, { pub fn init_bias(mut self) -> Self { - let (inputs, outputs) = self.features().in_by_out(); - let dk = (T::one() / T::from(inputs).unwrap()).sqrt(); - self.bias = ndarray::Array1::uniform_between(dk, outputs).into(); + let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); + self.bias = ndarray::Array1::uniform_between(dk, self.features().outputs()); self } pub fn init_weight(mut self) -> Self { - let (inputs, outputs) = self.features().in_by_out(); - let dk = (T::one() / T::from(inputs).unwrap()).sqrt(); - self.bias = ndarray::Array1::uniform_between(dk, outputs).into(); - self.weights = Array2::uniform_between(dk, (outputs, inputs)); + let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); + self.weights = Array2::uniform_between(dk, self.features().out_by_in()); self } - - pub fn new_biased(inputs: usize, outputs: usize) -> Self { - let features = Features::new(inputs, outputs); - let weights = Array2::uniform(1, (outputs, inputs)); - let bias = Bias::biased(outputs); - Self { - bias, - features, - weights, - } - } } impl LinearLayer @@ -114,14 +106,7 @@ where where T: 'static, { - self.linear(data) - } - - pub fn linear(&self, data: &Array2) -> Array2 - where - T: 'static, - { - data.dot(&self.weights.t()) + &self.bias + self.forward(data) } pub fn update_with_gradient(&mut self, gradient: &Array2, lr: T) { @@ -140,8 +125,8 @@ where D: Dimension, S: Dimension, T: NdFloat, - Array: Add, Output = Array> + Dot, Output = Array>, - Array: Add, Output = Array>, + Array: Add, Output = Array> + Dot, Output = Array>, + Array: Add, Output = Array>, { type Output = Array; diff --git a/ml/neural/src/layers/linear/mod.rs b/ml/neural/src/layers/linear/mod.rs index 457932ab..78f4a6a5 100644 --- a/ml/neural/src/layers/linear/mod.rs +++ b/ml/neural/src/layers/linear/mod.rs @@ -8,27 +8,13 @@ pub use self::{layer::*, regress::*, utils::*}; pub(crate) mod layer; pub(crate) mod regress; -use crate::params::{Biased, Weighted}; use ndarray::prelude::Array2; -use ndarray::ScalarOperand; use num::Float; -pub trait LinearTransformation -where - T: Float, -{ - fn linear(&self, data: &Array2) -> Array2; -} +// pub trait Lin where T: Float { -impl LinearTransformation for S -where - S: Biased + Weighted, - T: Float + ScalarOperand + 'static, -{ - fn linear(&self, data: &Array2) -> Array2 { - data.dot(&self.weights().t()) + self.bias() - } -} +// fn forward(&self, data: &Array2) -> Array2; +// } pub(crate) mod utils { use ndarray::prelude::{Array1, Array2}; @@ -46,13 +32,16 @@ pub(crate) mod utils { #[cfg(test)] mod tests { use super::*; + use crate::prelude::{Features, Forward}; + use ndarray::prelude::Array2; #[test] fn test_linear_layer() { - let (inputs, outputs) = (2, 2); - let data = Array2::::ones((inputs, outputs)); - let layer = LinearLayer::new_biased(inputs, outputs); - let linear = layer.linear(&data); - assert_eq!(linear.dim(), (inputs, outputs)); + let (samples, inputs, outputs) = (20, 2, 2); + let features = Features::new(inputs, outputs); + let data = Array2::::ones((20, features.inputs())); + let layer = LinearLayer::from_features(features).init_weight(); + let pred = layer.forward(&data); + assert_eq!(pred.dim(), (samples, outputs)); } } diff --git a/ml/neural/src/layers/linear/regress.rs b/ml/neural/src/layers/linear/regress.rs index 43fd9ed5..ba4f9141 100644 --- a/ml/neural/src/layers/linear/regress.rs +++ b/ml/neural/src/layers/linear/regress.rs @@ -3,6 +3,7 @@ Contrib: FL03 */ use crate::core::prelude::GenerateRandom; + use crate::prelude::Forward; use ndarray::prelude::{Array1, Array2, Axis, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; @@ -10,17 +11,6 @@ use ndarray_stats::CorrelationExt; use num::{Float, FromPrimitive}; use rand::Rng; -pub enum Params { - Layer { - bias: Array1, // (outputs,) - weights: Array2, // (outputs, inputs) - }, - Node { - bias: f64, - weights: Array1, // (inputs,) - }, -} - #[derive(Clone, Debug, PartialEq)] pub struct Linear where @@ -140,8 +130,10 @@ where where G: Fn(&Array1) -> Array1, { - let grad = gradient(self.weights()); + let mut grad = gradient(self.weights()); + grad /= grad.mapv(|ws| ws.powi(2)).sum().sqrt(); self.weights_mut().scaled_add(-gamma, &grad); + self.weights /= self.weights().mapv(|ws| ws.powi(2)).sum().sqrt(); } pub fn update_with_gradient(&mut self, gamma: T, gradient: &Array1) { diff --git a/ml/neural/src/layers/sublayer.rs b/ml/neural/src/layers/sublayer.rs index bd88240c..368a140a 100644 --- a/ml/neural/src/layers/sublayer.rs +++ b/ml/neural/src/layers/sublayer.rs @@ -4,10 +4,9 @@ */ use super::Layer; use crate::ops::LayerNorm; -use crate::prop::Forward; +use crate::prelude::Forward; -use ndarray::prelude::Array2; -use ndarray::ScalarOperand; +use ndarray::prelude::{Array2, NdFloat}; use num::{Float, FromPrimitive}; use serde::{Deserialize, Serialize}; @@ -17,15 +16,20 @@ pub struct Sublayer { norm: LayerNorm, } -impl Sublayer { +impl Sublayer +where + T: Float, +{ pub fn new(layer: Layer, norm: LayerNorm) -> Self { Self { layer, norm } } +} - pub fn forward(&self, data: &Array2) -> Array2 - where - T: FromPrimitive + ScalarOperand, - { +impl Sublayer +where + T: FromPrimitive + NdFloat, +{ + pub fn forward(&self, data: &Array2) -> Array2 { let norm = self.norm.forward(data); let layer = data + self.layer.forward(&norm); layer diff --git a/ml/neural/src/models/features.rs b/ml/neural/src/models/features.rs new file mode 100644 index 00000000..dad4f0c8 --- /dev/null +++ b/ml/neural/src/models/features.rs @@ -0,0 +1,115 @@ +/* + Appellation: features + Contrib: FL03 +*/ +use ndarray::IntoDimension; +use serde::{Deserialize, Serialize}; + +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] +pub struct Features { + pub inputs: usize, + pub outputs: usize, +} + +impl Features { + pub fn new(inputs: usize, outputs: usize) -> Self { + Self { inputs, outputs } + } + + pub fn neuron(inputs: usize) -> Self { + Self::new(inputs, 1) + } + + pub fn uniform_scale(&self) -> T { + (T::one() / T::from(self.inputs).unwrap()).sqrt() + } + + pub fn inputs(&self) -> usize { + self.inputs + } + + pub fn outputs(&self) -> usize { + self.outputs + } + + pub fn set_inputs(&mut self, inputs: usize) { + self.inputs = inputs; + } + + pub fn set_outputs(&mut self, outputs: usize) { + self.outputs = outputs; + } + + pub fn with_inputs(mut self, inputs: usize) -> Self { + self.inputs = inputs; + self + } + + pub fn with_outputs(mut self, outputs: usize) -> Self { + self.outputs = outputs; + self + } + + pub fn in_by_out(&self) -> (usize, usize) { + (self.inputs, self.outputs) + } + + pub fn out_by_in(&self) -> (usize, usize) { + (self.outputs, self.inputs) + } +} + +impl std::fmt::Display for Features { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "({}, {})", self.inputs, self.outputs) + } +} + +impl IntoDimension for Features { + type Dim = ndarray::IxDyn; + + fn into_dimension(self) -> Self::Dim { + ndarray::IxDyn(&[self.inputs, self.outputs]) + } +} + +impl From for ndarray::Ix2 { + fn from(features: Features) -> Self { + ndarray::Ix2(features.inputs, features.outputs) + } +} + +impl From for ndarray::IxDyn { + fn from(features: Features) -> Self { + ndarray::IxDyn(&[features.inputs, features.outputs]) + } +} + +impl From for [usize; 2] { + fn from(features: Features) -> Self { + [features.inputs, features.outputs] + } +} + +impl From<[usize; 2]> for Features { + fn from(features: [usize; 2]) -> Self { + Self { + inputs: features[0], + outputs: features[1], + } + } +} + +impl From for (usize, usize) { + fn from(features: Features) -> Self { + (features.inputs, features.outputs) + } +} + +impl From<(usize, usize)> for Features { + fn from((inputs, outputs): (usize, usize)) -> Self { + Self { inputs, outputs } + } +} diff --git a/ml/neural/src/models/mod.rs b/ml/neural/src/models/mod.rs index d009b3a6..1effcd87 100644 --- a/ml/neural/src/models/mod.rs +++ b/ml/neural/src/models/mod.rs @@ -4,12 +4,18 @@ */ //! # Model //! -pub use self::{model::*, utils::*}; +pub use self::{features::*, model::*, params::*, utils::*}; +pub(crate) mod features; pub(crate) mod model; +pub(crate) mod params; -pub trait Module { - fn add_module(&mut self, module: impl Module); +pub trait Module { + fn add_module(&mut self, module: impl Module); + + fn params(&self) -> &ModelParams; + + fn params_mut(&mut self) -> &mut ModelParams; } pub(crate) mod utils {} diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs index 144daa4a..6b7ea18c 100644 --- a/ml/neural/src/models/model.rs +++ b/ml/neural/src/models/model.rs @@ -2,9 +2,64 @@ Appellation: model Contrib: FL03 */ -use ndarray::prelude::Array1; +use super::{Features, ModelParams, Parameterized}; +use ndarray::prelude::{Array2, NdFloat}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use num::Float; -pub struct Model { - pub bias: T, - pub weights: Array1, +pub struct Model { + pub features: Features, + params: ModelParams, +} + +impl Model +where + T: Float, +{ + pub fn new(features: Features) -> Self { + Self { + features, + params: ModelParams::new(features), + } + } + + pub fn features(&self) -> &Features { + &self.features + } + + pub fn features_mut(&mut self) -> &mut Features { + &mut self.features + } +} + +impl Model +where + T: NdFloat, +{ + pub fn linear(&self, args: &Array2) -> Array2 { + args.dot(&self.params().weights().t()) + self.params().bias() + } +} + +impl Model +where + T: Float + SampleUniform, +{ + pub fn init(mut self, biased: bool) -> Self { + self.params = self.params.init(biased); + self + } +} + +impl Parameterized for Model +where + T: Float, +{ + fn params(&self) -> &ModelParams { + &self.params + } + + fn params_mut(&mut self) -> &mut ModelParams { + &mut self.params + } } diff --git a/ml/neural/src/models/params.rs b/ml/neural/src/models/params.rs new file mode 100644 index 00000000..dddff08c --- /dev/null +++ b/ml/neural/src/models/params.rs @@ -0,0 +1,132 @@ +/* + Appellation: model + Contrib: FL03 +*/ +use super::Features; +use crate::core::prelude::GenerateRandom; +use ndarray::prelude::{Array1, Array2}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use num::Float; + +pub trait Parameterized +where + T: Float, +{ + fn bias(&self) -> &Array1 { + self.params().bias() + } + + fn bias_mut(&mut self) -> &mut Array1 { + self.params_mut().bias_mut() + } + + fn params(&self) -> &ModelParams; + + fn params_mut(&mut self) -> &mut ModelParams; + + fn weights(&self) -> &Array2 { + self.params().weights() + } + + fn weights_mut(&mut self) -> &mut Array2 { + self.params_mut().weights_mut() + } + + fn set_bias(&mut self, bias: Array1) { + self.params_mut().set_bias(bias); + } + + fn set_weights(&mut self, weights: Array2) { + self.params_mut().set_weights(weights); + } +} + +pub struct ModelParams { + bias: Array1, + pub features: Features, + weights: Array2, +} + +impl ModelParams +where + T: Float, +{ + pub fn new(features: Features) -> Self { + Self { + bias: Array1::zeros(features.outputs()), + features, + weights: Array2::zeros(features.out_by_in()), + } + } + + pub fn reset(&mut self) { + self.bias = Array1::zeros(self.features.outputs()); + self.weights = Array2::zeros(self.features.out_by_in()); + } + + pub fn bias(&self) -> &Array1 { + &self.bias + } + + pub fn bias_mut(&mut self) -> &mut Array1 { + &mut self.bias + } + + pub fn features(&self) -> &Features { + &self.features + } + + pub fn features_mut(&mut self) -> &mut Features { + &mut self.features + } + + pub fn weights(&self) -> &Array2 { + &self.weights + } + + pub fn weights_mut(&mut self) -> &mut Array2 { + &mut self.weights + } + + pub fn set_bias(&mut self, bias: Array1) { + self.bias = bias; + } + + pub fn set_weights(&mut self, weights: Array2) { + self.weights = weights; + } + + pub fn with_bias(mut self, bias: Array1) -> Self { + self.bias = bias; + self + } + + pub fn with_weights(mut self, weights: Array2) -> Self { + self.weights = weights; + self + } +} + +impl ModelParams +where + T: Float + SampleUniform, +{ + pub fn init(mut self, biased: bool) -> Self { + if biased { + self = self.init_bias(); + } + self.init_weight() + } + + pub fn init_bias(mut self) -> Self { + let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); + self.bias = Array1::uniform_between(dk, self.features().outputs()); + self + } + + pub fn init_weight(mut self) -> Self { + let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); + self.weights = Array2::uniform_between(dk, self.features().out_by_in()); + self + } +} diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index bdb25b8b..20920e70 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -16,6 +16,7 @@ pub(crate) mod utils {} mod tests { use super::activate::{softmax, Activate, Softmax}; use super::*; + use crate::prelude::Forward; // use lazy_static::lazy_static; use ndarray::{array, Array1}; @@ -42,7 +43,7 @@ mod tests { let exp = softmax(data.clone().dot(&weights) + bias); - assert_eq!(exp, neuron.process(&data)); + assert_eq!(exp, neuron.forward(&data)); } // #[test] diff --git a/ml/neural/src/neurons/neuron.rs b/ml/neural/src/neurons/neuron.rs index 7aec6619..caf4c63d 100644 --- a/ml/neural/src/neurons/neuron.rs +++ b/ml/neural/src/neurons/neuron.rs @@ -2,95 +2,144 @@ Appellation: neuron Contrib: FL03 */ -use super::activate::{Activate, ActivationFn, LinearActivation}; +use super::activate::{Activate, LinearActivation}; +use crate::core::GenerateRandom; use crate::prelude::Forward; -use crate::{core::GenerateRandom, layers::L}; -use ndarray::prelude::{Array1, Array2}; +use ndarray::prelude::{Array1, Array2, NdFloat}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use num::Float; +use rand::Rng; -pub trait ArtificialNeuron { - type Rho: Activate; +pub trait ArtificialNeuron +where + T: NdFloat, +{ + type Rho: Activate>; + + fn bias(&self) -> T; + + fn forward(&self, args: &Array2) -> Array1 { + self.rho() + .activate(args.dot(&self.weights().t()) + self.bias()) + } + + fn rho(&self) -> &Self::Rho; + + fn weights(&self) -> &Array1; } /// Artificial Neuron #[derive(Clone, Debug, PartialEq)] -pub struct Neuron +pub struct Neuron where - Rho: Activate>, + A: Activate>, { - activation: Rho, - bias: f64, + activation: A, + bias: T, features: usize, - weights: Array1, + weights: Array1, } -impl Neuron +impl Neuron where - Rho: Activate>, + A: Activate>, + T: Float, { - pub fn with_rho(mut self, rho: Rho) -> Self { - self.activation = rho; - self - } - - pub fn init_weights(mut self) -> Self { - self.weights = Array1::uniform(0, self.features); - self - } - - pub fn bias(&self) -> f64 { - self.bias + pub fn bias(&self) -> &T { + &self.bias } - pub fn process(&self, args: &Array2) -> Array1 { - self.rho() - .activate(args.dot(&self.weights.t()) + self.bias()) - } - - pub fn rho(&self) -> &Rho { + pub fn rho(&self) -> &A { &self.activation } - pub fn weights(&self) -> &Array1 { + pub fn weights(&self) -> &Array1 { &self.weights } - pub fn set_bias(&mut self, bias: f64) { + pub fn weights_mut(&mut self) -> &mut Array1 { + &mut self.weights + } + + pub fn set_bias(&mut self, bias: T) { self.bias = bias; } - pub fn set_weights(&mut self, weights: Array1) { + pub fn set_weights(&mut self, weights: Array1) { self.weights = weights; } - pub fn with_bias(mut self, bias: f64) -> Self { + pub fn with_bias(mut self, bias: T) -> Self { self.bias = bias; self } - pub fn with_weights(mut self, weights: Array1) -> Self { - self.weights = weights; + pub fn with_rho(mut self, rho: A) -> Self { + self.activation = rho; self } - pub fn apply_weight_gradient(&mut self, gamma: f64, gradient: &Array1) { - self.weights = &self.weights - gamma * gradient; + pub fn with_weights(mut self, weights: Array1) -> Self { + self.weights = weights; + self } } -impl Neuron +impl Neuron where - Rho: Activate> + Default, + T: NdFloat, + A: Activate> + Default, { pub fn new(features: usize) -> Self { Self { - activation: Rho::default(), - bias: 0.0, + activation: A::default(), + bias: T::zero(), features, weights: Array1::zeros(features), } } } +impl Neuron +where + T: NdFloat, + A: Activate>, +{ + pub fn apply_gradient(&mut self, gamma: T, gradient: G) + where + G: Fn(&Array1) -> Array1, + { + let grad = gradient(&self.weights); + self.weights_mut().scaled_add(-gamma, &grad); + } +} + +impl Neuron +where + T: Float + SampleUniform, + A: Activate> + Default, +{ + pub fn init(mut self, biased: bool) -> Self { + if biased { + self = self.init_bias(); + } + self.init_weight() + } + + pub fn init_bias(mut self) -> Self { + let dk = (T::one() / T::from(self.features).unwrap()).sqrt(); + self.bias = rand::thread_rng().gen_range(-dk..dk); + self + } + + pub fn init_weight(mut self) -> Self { + let features = self.features; + let dk = (T::one() / T::from(features).unwrap()).sqrt(); + self.weights = Array1::uniform_between(dk, features); + self + } +} + // impl Forward> for Neuron { // type Output = f64; @@ -100,11 +149,15 @@ where // } -impl Forward> for Neuron { - type Output = Array1; +impl Forward> for Neuron +where + T: NdFloat, + A: Activate> + Default, +{ + type Output = Array1; - fn forward(&self, args: &Array2) -> Self::Output { - let linstep = args.dot(&self.weights().t().to_owned()) + self.bias; + fn forward(&self, args: &Array2) -> Self::Output { + let linstep = args.dot(&self.weights().t()) + self.bias; self.rho().activate(linstep) } } diff --git a/ml/neural/src/nn/network.rs b/ml/neural/src/nn/network.rs index 16df0f3a..c0515db5 100644 --- a/ml/neural/src/nn/network.rs +++ b/ml/neural/src/nn/network.rs @@ -6,5 +6,7 @@ use crate::layers::Layer; use num::Float; pub struct NeuralNetwork { - pub layers: Vec>, + pub input: Layer, + pub hidden: Vec>, + pub output: Layer, } diff --git a/ml/neural/src/ops/mod.rs b/ml/neural/src/ops/mod.rs index b7a90f4d..6a21e539 100644 --- a/ml/neural/src/ops/mod.rs +++ b/ml/neural/src/ops/mod.rs @@ -12,6 +12,7 @@ pub(crate) mod utils {} #[cfg(test)] mod tests { use super::*; + use crate::prelude::Forward; use computare::prelude::RoundTo; use ndarray::prelude::{array, Array, Ix2}; diff --git a/ml/neural/src/ops/norm.rs b/ml/neural/src/ops/norm.rs index 815044f8..7fb25369 100644 --- a/ml/neural/src/ops/norm.rs +++ b/ml/neural/src/ops/norm.rs @@ -2,18 +2,26 @@ Appellation: norm Contrib: FL03 */ -use ndarray::prelude::{Array1, Array2}; -use ndarray::{Axis, ScalarOperand}; +use crate::prelude::Forward; +use ndarray::prelude::{Array, Array1, NdFloat}; +use ndarray::Dimension; use num::{Float, FromPrimitive}; use serde::{Deserialize, Serialize}; +use std::ops::{Add, Mul}; #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -pub struct LayerNorm { +pub struct LayerNorm +where + T: Float, +{ alpha: Array1, beta: Array1, } -impl LayerNorm { +impl LayerNorm +where + T: Float, +{ pub fn new(features: usize) -> Self { Self { alpha: Array1::ones(features), @@ -21,18 +29,59 @@ impl LayerNorm { } } - pub fn forward(&self, x: &Array2) -> Array2 + pub fn alpha(&self) -> &Array1 { + &self.alpha + } + + pub fn alpha_mut(&mut self) -> &mut Array1 { + &mut self.alpha + } + + pub fn beta(&self) -> &Array1 { + &self.beta + } + + pub fn beta_mut(&mut self) -> &mut Array1 { + &mut self.beta + } +} + +impl LayerNorm +where + T: FromPrimitive + NdFloat, +{ + pub fn norm_and_scale(&self, x: &Array) -> Array where - T: FromPrimitive + ScalarOperand, + D: Dimension, + Array: Add, Output = Array> + Mul, Output = Array>, { let epsilon = T::from(1e-6).unwrap(); // Calculate the mean and standard deviation of the activations along the feature axis. - let mean = x.mean_axis(Axis(1)).unwrap(); - let std = x.std_axis(Axis(1), T::one()); + let mean = x.mean().unwrap_or_else(T::zero); + // Normalize the activations. + let norm = (x - mean) / (x.std(T::one()) + epsilon); + + // Scale and shift the normalized activations with learnable parameters alpha and beta. + norm * self.alpha().clone() + self.beta().clone() + } +} + +impl Forward> for LayerNorm +where + D: Dimension, + T: FromPrimitive + NdFloat, + Array: Add, Output = Array> + Mul, Output = Array>, +{ + type Output = Array; + + fn forward(&self, x: &Array) -> Self::Output { + let epsilon = T::from(1e-6).unwrap(); + // Calculate the mean and standard deviation of the activations along the feature axis. + let mean = x.mean().unwrap_or_else(T::zero); // Normalize the activations. - let norm = (x - &mean) / (&std + epsilon); + let norm = (x - mean) / (x.std(T::one()) + epsilon); // Scale and shift the normalized activations with learnable parameters alpha and beta. - &norm * &self.alpha + &self.beta + norm * self.alpha().clone() + self.beta().clone() } } diff --git a/ml/neural/src/params/mod.rs b/ml/neural/src/params/mod.rs index 521e355c..ead1c0ab 100644 --- a/ml/neural/src/params/mod.rs +++ b/ml/neural/src/params/mod.rs @@ -31,6 +31,17 @@ where pub trait Parameter {} +pub trait Params +where + D: Dimension, + T: Float, +{ + fn bias(&self) -> &Array; + fn bias_mut(&mut self) -> &mut Array; + fn weights(&self) -> &Array; + fn weights_mut(&mut self) -> &mut Array; +} + pub trait Biased where T: Float, diff --git a/ml/neural/src/params/shapes.rs b/ml/neural/src/params/shapes.rs index 14a3f3ac..bd67e16c 100644 --- a/ml/neural/src/params/shapes.rs +++ b/ml/neural/src/params/shapes.rs @@ -2,19 +2,14 @@ Appellation: shapes Contrib: FL03 */ -use ndarray::{Dimension, IntoDimension}; use serde::{Deserialize, Serialize}; -use strum::{EnumIs, EnumIter, EnumVariantNames}; +use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; pub trait LayerFeatures { fn inputs(&self) -> usize; fn outputs(&self) -> usize; } -pub trait NeuronFeatures { - fn inputs(&self) -> usize; -} - impl LayerFeatures for ParameterShapes { fn inputs(&self) -> usize { match self { @@ -50,3 +45,40 @@ pub enum ParameterShapes { Layer { inputs: usize, outputs: usize }, Neuron { inputs: usize }, } + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum Parameters { + Bias, + #[default] + Weights, +} + +impl Parameters { + pub fn bias() -> Self { + Self::Bias + } + + pub fn weights() -> Self { + Self::Weights + } +} diff --git a/ml/neural/src/primitives.rs b/ml/neural/src/primitives.rs index 88844d0e..23346bb7 100644 --- a/ml/neural/src/primitives.rs +++ b/ml/neural/src/primitives.rs @@ -16,5 +16,5 @@ pub(crate) mod types { pub type LayerBias = ndarray::Array1; - pub type WeightTensor = ndarray::Array2; + pub type LayerWeight = ndarray::Array2; } diff --git a/ml/optim/examples/descent.rs b/ml/optim/examples/descent.rs index ba2a69d2..98d29aa0 100644 --- a/ml/optim/examples/descent.rs +++ b/ml/optim/examples/descent.rs @@ -11,7 +11,7 @@ fn main() -> anyhow::Result<()> { let _n = samples * inputs; - let (epochs, gamma) = (500, 0.01); + let (epochs, gamma) = (500, 0.001); // basic_descent(epochs, features, gamma)?; diff --git a/ml/optim/examples/norm.rs b/ml/optim/examples/norm.rs new file mode 100644 index 00000000..54384fb3 --- /dev/null +++ b/ml/optim/examples/norm.rs @@ -0,0 +1,32 @@ +use concision_neural::prelude::Features; +use concision_optim::prelude::Norm; +use ndarray::prelude::Array; + +fn main() -> anyhow::Result<()> { + let (samples, inputs) = (20, 5); + let outputs = 1; + + let features = Features::new(inputs, outputs); + + // basic_descent(epochs, features, gamma)?; + + sample_norm(features, samples)?; + + Ok(()) +} + +pub fn sample_norm(features: Features, samples: usize) -> anyhow::Result<()> { + let n = samples * features.inputs(); + let args = Array::linspace(1., n as f64, n) + .into_shape((samples, features.inputs())) + .unwrap(); + + println!( + "Norms:\n\nL0: {:?}\nL1: {:?}\nL2: {:?}\n", + &args.l0(), + &args.l1(), + &args.l2() + ); + + Ok(()) +} diff --git a/ml/optim/examples/sgd.rs b/ml/optim/examples/sgd.rs index d12837c0..06d78c21 100644 --- a/ml/optim/examples/sgd.rs +++ b/ml/optim/examples/sgd.rs @@ -19,7 +19,7 @@ fn main() -> anyhow::Result<()> { .unwrap() + 1.0; - let model = LinearLayer::::new_biased(inputs, outputs); + let model = LinearLayer::::new(inputs, outputs); let mut sgd = StochasticGradientDescent::new(batch_size, epochs, gamma, model); let losses = sgd.sgd(&x, &y); diff --git a/ml/optim/src/grad/descent.rs b/ml/optim/src/grad/descent.rs index d5c1c3bc..94631ada 100644 --- a/ml/optim/src/grad/descent.rs +++ b/ml/optim/src/grad/descent.rs @@ -3,24 +3,11 @@ Contrib: FL03 */ use crate::neural::layers::linear::Linear; -use crate::neural::prelude::{mse, Forward}; -use crate::prelude::Normalize; +use crate::neural::prelude::Forward; +use crate::prelude::Norm; use ndarray::prelude::{Array1, Array2}; use ndarray_stats::DeviationExt; -pub fn cost(target: &Array1, prediction: &Array1) -> f64 { - (target - prediction) - .map(|x| x.powi(2)) - .mean() - .unwrap_or_default() -} - -pub fn grad(data: &Array2, target: &Array1, prediction: &Array1) -> Array1 { - let error = prediction - target; - let scale = -2.0 / data.len() as f64; - scale * data.t().dot(&error) -} - pub fn gradient_descent( weights: &mut Array1, epochs: usize, @@ -35,8 +22,6 @@ pub fn gradient_descent( losses } -pub type BaseObjective = fn(&Array1) -> Array1; - #[derive(Clone)] pub struct GradientDescent { pub gamma: f64, @@ -83,13 +68,13 @@ impl GradientDescent { } pub fn logit(&mut self, data: &Array2, targets: &Array1) -> anyhow::Result { + // let pred = self.model.forward(data); let gradient = |p: &Array1| { - let pred = data.dot(p); - let error = targets - &pred; + let error = targets - &data.dot(&(p / p.l2())); let scale = -1.0 / (2.0 * data.len() as f64); let grad = scale * error.dot(data); - &grad / grad.l2() + grad }; self.model.apply_gradient(self.gamma, &gradient); @@ -97,7 +82,7 @@ impl GradientDescent { Ok(loss) } - pub fn descent(&mut self, data: &Array2, targets: &Array1) -> anyhow::Result { + pub fn step(&mut self, data: &Array2, targets: &Array1) -> anyhow::Result { let gradient = |p: &Array1| { let pred = data.dot(p); let error = targets - pred; @@ -112,29 +97,6 @@ impl GradientDescent { let loss = targets.mean_sq_err(&self.model.forward(data))?; Ok(loss) } - - pub fn gradient(&self, x: &Array2, y: &Array1) -> Array1 { - let (samples, _inputs) = x.dim(); - - let pred = x.dot(&self.model.weights().t()) + *self.model.bias(); // fully connected - let errors = y - &pred; - let scale = -2.0 / samples as f64; - scale * x.t().dot(&errors) - } - - pub fn step(&mut self, x: &Array2, y: &Array1) -> f64 { - let (samples, _inputs) = x.dim(); - - let predictions = x.dot(&self.model.weights().t()) + *self.model.bias(); // fully connected - let errors = y - &predictions; - let scale = -2.0 / samples as f64; - let _bias = scale * &errors; - let weights = scale * x.t().dot(&errors); - self.model - .update_with_gradient(self.gamma, &weights.t().to_owned()); - - mse(&self.model.forward(x), y).unwrap_or_default() - } } #[cfg(test)] @@ -164,6 +126,6 @@ mod tests { let model = Linear::new(inputs).init_weight(); let mut grad = GradientDescent::new(gamma, model); - let _s = grad.descent(&x, &y); + let _s = grad.step(&x, &y); } } diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs index 2cac6e63..abd89ec7 100644 --- a/ml/optim/src/grad/gradient.rs +++ b/ml/optim/src/grad/gradient.rs @@ -2,33 +2,63 @@ Appellation: grad Contrib: FL03 */ - +use crate::neural::models::ModelParams; use ndarray::prelude::{Array1, Array2}; +use num::Float; + +pub struct GradStep +where + T: Float, +{ + gamma: T, + params: ModelParams, +} -pub type BiasGradient = Array1; +impl GradStep +where + T: Float, +{ + pub fn new(gamma: T, params: ModelParams) -> Self { + Self { gamma, params } + } -pub type WeightGradient = Array2; + pub fn gamma(&self) -> T { + self.gamma + } -pub struct Grad {} + pub fn gamma_mut(&mut self) -> &mut T { + &mut self.gamma + } -pub struct Gradients { - pub bias: BiasGradient, - pub weights: WeightGradient, -} + pub fn params(&self) -> &ModelParams { + &self.params + } -impl Gradients { - pub fn new(bias: BiasGradient, weights: WeightGradient) -> Self { - Self { bias, weights } + pub fn params_mut(&mut self) -> &mut ModelParams { + &mut self.params } - pub fn zeros(inputs: usize, outputs: usize) -> Self { - Self { - bias: Array1::zeros(outputs), - weights: Array2::zeros((inputs, outputs)), - } + pub fn set_gamma(&mut self, gamma: T) { + self.gamma = gamma; + } + + pub fn set_params(&mut self, params: ModelParams) { + self.params = params; + } + + pub fn with_gamma(mut self, gamma: T) -> Self { + self.gamma = gamma; + self + } + + pub fn with_params(mut self, params: ModelParams) -> Self { + self.params = params; + self } } +impl GradStep where T: Float {} + #[cfg(test)] mod tests { diff --git a/ml/optim/src/grad/sgd.rs b/ml/optim/src/grad/sgd.rs index c26c8af4..862894a8 100644 --- a/ml/optim/src/grad/sgd.rs +++ b/ml/optim/src/grad/sgd.rs @@ -220,7 +220,7 @@ mod tests { let x = Array::linspace(1., 100., 100).into_shape(shape).unwrap(); let y = Array::linspace(1., 100., 5).into_shape(5).unwrap(); - let model = LinearLayer::::new_biased(inputs, 5); + let model = LinearLayer::::new(inputs, 5); let mut sgd = StochasticGradientDescent::new(batch_size, epochs, gamma, model); sgd.sgd(&x, &y); @@ -236,7 +236,7 @@ mod tests { let x = Array::linspace(1., 100., 100).into_shape(shape).unwrap(); let y = Array1::::uniform(0, 100); - let mut model = LinearLayer::::new_biased(inputs, 5); + let mut model = LinearLayer::::new(inputs, 5); // let grad = sgd(&x, &y, &mut model, epochs, gamma, batch_size); // assert!(grad.is_ok()); diff --git a/ml/optim/src/norm/kinds.rs b/ml/optim/src/norm/kinds.rs index 2b1d3cad..f27b1cb0 100644 --- a/ml/optim/src/norm/kinds.rs +++ b/ml/optim/src/norm/kinds.rs @@ -2,6 +2,9 @@ Appellation: kinds Contrib: FL03 */ +use super::Norm; +use ndarray::prelude::{Array, NdFloat}; +use ndarray::Dimension; use serde::{Deserialize, Serialize}; use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; @@ -27,8 +30,43 @@ use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] pub enum Norms { - L0, - L1, + L0 = 0, + L1 = 1, #[default] - L2, + L2 = 2, +} + +impl Norms { + pub fn l0() -> Self { + Self::L0 + } + + pub fn l1() -> Self { + Self::L1 + } + + pub fn l2() -> Self { + Self::L2 + } + + pub fn normalize(&self, args: &S) -> T + where + S: Norm, + { + use Norms::*; + + match *self { + L0 => args.l0(), + L1 => args.l1(), + L2 => args.l2(), + } + } + + pub fn norm_and_scale(&self, args: &Array) -> Array + where + D: Dimension, + T: NdFloat, + { + args / self.normalize(args) + } } diff --git a/ml/optim/src/norm/mod.rs b/ml/optim/src/norm/mod.rs index 54336762..5ee09d74 100644 --- a/ml/optim/src/norm/mod.rs +++ b/ml/optim/src/norm/mod.rs @@ -4,14 +4,22 @@ */ //! # norm //! -pub use self::{kinds::*, utils::*}; +pub use self::{kinds::*, normalizer::*, utils::*}; pub(crate) mod kinds; +pub(crate) mod normalizer; use ndarray::prelude::{Array, NdFloat}; use ndarray::Dimension; +use ndarray_stats::QuantileExt; pub trait Normalize { + type Output; + + fn norm(&self, args: &T) -> Self::Output; +} + +pub trait Norm { fn l0(&self) -> T; fn l1(&self) -> T; @@ -19,21 +27,21 @@ pub trait Normalize { fn l2(&self) -> T; } -impl Normalize for Array +impl Norm for Array where D: Dimension, T: NdFloat, { fn l0(&self) -> T { - utils::l0_norm(self) + *self.max().expect("No max value") } fn l1(&self) -> T { - utils::l1_norm(self) + self.mapv(|xs| xs.abs()).sum() } fn l2(&self) -> T { - utils::l2_norm(self) + self.mapv(|xs| xs.powi(2)).sum().sqrt() } } @@ -68,4 +76,13 @@ pub(crate) mod utils { } #[cfg(test)] -mod tests {} +mod tests { + use super::*; + + #[test] + fn test_l0_norm() { + let args = Array::linspace(1., 3., 3).into_shape(3).unwrap(); + + assert_eq!(l0_norm(&args), 3.); + } +} diff --git a/ml/optim/src/norm/normalizer.rs b/ml/optim/src/norm/normalizer.rs new file mode 100644 index 00000000..706fc5dc --- /dev/null +++ b/ml/optim/src/norm/normalizer.rs @@ -0,0 +1,40 @@ +/* + Appellation: normalizer + Contrib: FL03 +*/ +use super::{Norm, Norms}; +use ndarray::prelude::{Array, NdFloat}; +use ndarray::Dimension; +use serde::{Deserialize, Serialize}; + +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] +pub struct Normalizer { + pub mode: Norms, +} + +impl Normalizer { + pub fn new(mode: Norms) -> Self { + Self { mode } + } + + pub fn normalize(&self, args: &S) -> T + where + S: Norm, + { + match self.mode { + Norms::L0 => args.l0(), + Norms::L1 => args.l1(), + Norms::L2 => args.l2(), + } + } + + pub fn norm_and_scale(&self, args: &Array) -> Array + where + D: Dimension, + T: NdFloat, + { + args / self.normalize(args) + } +} diff --git a/ml/transformers/src/attention/multi/attention.rs b/ml/transformers/src/attention/multi/attention.rs index 4f92de3f..456b2768 100644 --- a/ml/transformers/src/attention/multi/attention.rs +++ b/ml/transformers/src/attention/multi/attention.rs @@ -5,7 +5,7 @@ use super::{multihead, MultiHeadParams}; use crate::attention::Weight; use crate::neural::layers::linear::LinearLayer; -use crate::neural::prelude::Mask; +use crate::neural::prelude::{Forward, Mask}; use crate::ops::Split; use ndarray::prelude::{Array2, NdFloat}; use ndarray::{ScalarOperand, ShapeError}; @@ -45,7 +45,7 @@ where let params = MultiHeadParams::new(heads, model); let weights = Weight::uniform((model, model)); Self { - linear: LinearLayer::new_biased(model, model), + linear: LinearLayer::new(model, model), params, weights, } @@ -60,7 +60,7 @@ where let weighted = data * self.weights(); let (q, k, v) = weighted.split(self.params().heads())?; let score = multihead(&q, &k, &v, mask)?; - let res = self.linear().linear(&score); + let res = self.linear().forward(&score); Ok(res) } } diff --git a/ml/transformers/src/ffn/network.rs b/ml/transformers/src/ffn/network.rs index db13324c..f6f060cf 100644 --- a/ml/transformers/src/ffn/network.rs +++ b/ml/transformers/src/ffn/network.rs @@ -19,7 +19,7 @@ pub struct FFN { impl FFN { pub fn new(model: usize, network: Option) -> Self { let params = FFNParams::new(model, network.unwrap_or(crate::NETWORK_SIZE)); - let layer = LinearLayer::new_biased(params.model, params.network); + let layer = LinearLayer::new(params.model, params.network); Self { input: layer.clone(), output: layer, @@ -33,6 +33,6 @@ impl Forward> for FFN { fn forward(&self, data: &Array2) -> Self::Output { self.output - .linear(&Activate::activate(&ReLU, self.input.linear(data))) + .forward(&Activate::activate(&ReLU, self.input.forward(data))) } } From 3d72f8e20f77094f66abcbd65914ba5dd7ea7db0 Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 17 Nov 2023 13:01:13 -0600 Subject: [PATCH 053/118] update Signed-off-by: FL03 --- concision/Cargo.toml | 1 + concision/src/lib.rs | 4 + core/src/errors/mod.rs | 2 + core/src/specs.rs | 14 +- core/tests/utils.rs | 13 ++ data/src/lib.rs | 2 + ml/ml/Cargo.toml | 67 ++++++++ ml/ml/benches/default.rs | 52 ++++++ ml/ml/examples/nn.rs | 8 + ml/ml/src/lib.rs | 29 ++++ ml/ml/tests/default.rs | 8 + .../{neurons => func}/activate/activator.rs | 17 +- .../src/{neurons => func}/activate/binary.rs | 11 +- .../src/{neurons => func}/activate/mod.rs | 16 +- .../{neurons => func}/activate/nonlinear.rs | 96 ++++++++--- ml/neural/src/{ => func}/loss/kinds.rs | 0 ml/neural/src/{ => func}/loss/mod.rs | 23 ++- ml/neural/src/{ => func}/loss/regress.rs | 6 +- ml/neural/src/func/mod.rs | 29 ++++ ml/neural/src/layers/features.rs | 18 +-- ml/neural/src/layers/kinds.rs | 81 ++++++++-- ml/neural/src/layers/layer.rs | 151 ++++++++++++------ ml/neural/src/layers/linear/mod.rs | 22 +-- ml/neural/src/layers/mod.rs | 24 ++- ml/neural/src/layers/params.rs | 107 +++++++++++++ ml/neural/src/layers/sublayer.rs | 17 +- ml/neural/src/lib.rs | 7 +- ml/neural/src/models/features.rs | 4 +- ml/neural/src/models/mod.rs | 2 + ml/neural/src/models/model.rs | 20 ++- ml/neural/src/models/params.rs | 92 ++++------- ml/neural/src/models/stack.rs | 32 ++++ ml/neural/src/neurons/mod.rs | 26 ++- ml/neural/src/neurons/neuron.rs | 24 +-- ml/neural/src/neurons/params.rs | 137 ++++++++++++++++ ml/neural/src/nn/deep.rs | 54 +++++++ ml/neural/src/nn/mod.rs | 8 +- ml/neural/src/nn/network.rs | 12 -- ml/neural/src/params/mod.rs | 108 +++++++++---- ml/neural/src/params/param.rs | 115 +++++++++++++ ml/optim/examples/descent.rs | 10 +- ml/optim/examples/norm.rs | 13 +- ml/optim/src/cost/mod.rs | 13 +- ml/optim/src/grad/descent.rs | 29 +--- ml/optim/src/grad/gradient.rs | 46 +++++- ml/optim/src/grad/mod.rs | 97 ++++++----- ml/optim/src/norm/mod.rs | 16 +- ml/optim/src/specs.rs | 47 +++++- ml/optim/src/utils.rs | 39 ++++- ml/transformers/src/attention/head.rs | 9 +- ml/transformers/src/attention/mod.rs | 13 +- ml/transformers/src/attention/multi/mod.rs | 13 +- ml/transformers/src/ffn/network.rs | 2 +- 53 files changed, 1411 insertions(+), 395 deletions(-) create mode 100644 core/tests/utils.rs create mode 100644 ml/ml/Cargo.toml create mode 100644 ml/ml/benches/default.rs create mode 100644 ml/ml/examples/nn.rs create mode 100644 ml/ml/src/lib.rs create mode 100644 ml/ml/tests/default.rs rename ml/neural/src/{neurons => func}/activate/activator.rs (65%) rename ml/neural/src/{neurons => func}/activate/binary.rs (67%) rename ml/neural/src/{neurons => func}/activate/mod.rs (73%) rename ml/neural/src/{neurons => func}/activate/nonlinear.rs (55%) rename ml/neural/src/{ => func}/loss/kinds.rs (100%) rename ml/neural/src/{ => func}/loss/mod.rs (79%) rename ml/neural/src/{ => func}/loss/regress.rs (96%) create mode 100644 ml/neural/src/func/mod.rs create mode 100644 ml/neural/src/layers/params.rs create mode 100644 ml/neural/src/models/stack.rs create mode 100644 ml/neural/src/neurons/params.rs create mode 100644 ml/neural/src/nn/deep.rs delete mode 100644 ml/neural/src/nn/network.rs create mode 100644 ml/neural/src/params/param.rs diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 02d63ff7..2bdfabb2 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -86,6 +86,7 @@ concision-derive = { features = [], optional = true, path = "../derive", version concision-macros = { features = [], optional = true, path = "../macros", version = "0.1.12" } concision-math = { features = [], optional = true, path = "../math", version = "0.1.12" } +# concision-ml = { features = [], optional = true, path = "../ml/ml", version = "0.1.12" } concision-neural = { features = [], optional = true, path = "../ml/neural", version = "0.1.12" } concision-nlp = { features = [], optional = true, path = "../ml/nlp", version = "0.1.12" } concision-optim = { features = [], optional = true, path = "../ml/optim", version = "0.1.12" } diff --git a/concision/src/lib.rs b/concision/src/lib.rs index c7ac51b8..aa93f127 100644 --- a/concision/src/lib.rs +++ b/concision/src/lib.rs @@ -17,6 +17,8 @@ pub use concision_derive::*; pub use concision_macros::*; #[cfg(feature = "math")] pub use concision_math as math; +// #[cfg(feature = "ml")] +// pub use concision_ml as ml; #[cfg(feature = "neural")] pub use concision_neural as neural; #[cfg(feature = "nlp")] @@ -37,6 +39,8 @@ pub mod prelude { pub use concision_macros::*; #[cfg(feature = "math")] pub use concision_math::prelude::*; + // #[cfg(feature = "ml")] + // pub use concision_ml::prelude::*; #[cfg(feature = "neural")] pub use concision_neural::prelude::*; #[cfg(feature = "nlp")] diff --git a/core/src/errors/mod.rs b/core/src/errors/mod.rs index 267dace8..35d99c5c 100644 --- a/core/src/errors/mod.rs +++ b/core/src/errors/mod.rs @@ -6,4 +6,6 @@ pub use self::{error::*, utils::*}; pub(crate) mod error; + + pub(crate) mod utils {} diff --git a/core/src/specs.rs b/core/src/specs.rs index d4aa3e3e..436f5d7e 100644 --- a/core/src/specs.rs +++ b/core/src/specs.rs @@ -2,14 +2,24 @@ Appellation: specs Contrib: FL03 */ -use ndarray::prelude::Array; -use ndarray::{Dimension, IntoDimension}; +use ndarray::prelude::{Array, Axis, Dimension}; +use ndarray::IntoDimension; // use ndarray::linalg::Dot; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Uniform}; use ndarray_rand::RandomExt; use num::{Float, One, Zero}; +pub trait IntoAxis { + fn into_axis(self) -> Axis; +} + +impl IntoAxis for usize { + fn into_axis(self) -> Axis { + Axis(self) + } +} + pub trait Borrowed: AsRef + AsMut {} impl Borrowed for S where S: AsRef + AsMut {} diff --git a/core/tests/utils.rs b/core/tests/utils.rs new file mode 100644 index 00000000..90bb11c3 --- /dev/null +++ b/core/tests/utils.rs @@ -0,0 +1,13 @@ +#[cfg(test)] +extern crate concision_core; + +use concision_core::prelude::now; + +#[test] +fn test_timestamp() { + let period = std::time::Duration::from_secs(1); + let ts = now(); + assert!(ts > 0); + std::thread::sleep(period); + assert!(now() > ts); +} diff --git a/data/src/lib.rs b/data/src/lib.rs index c7833060..bbf23117 100644 --- a/data/src/lib.rs +++ b/data/src/lib.rs @@ -15,6 +15,8 @@ pub mod flows; pub mod tensors; pub mod prelude { + pub use linfa::dataset::{Dataset, DatasetBase, DatasetView}; + pub use crate::datasets::*; pub use crate::df::*; pub use crate::flows::*; diff --git a/ml/ml/Cargo.toml b/ml/ml/Cargo.toml new file mode 100644 index 00000000..7a504bcc --- /dev/null +++ b/ml/ml/Cargo.toml @@ -0,0 +1,67 @@ +[package] +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "concision-ml" +readme.workspace = true +repository.workspace = true +version.workspace = true + +[features] +default = [ + "neural", + "optim", +] + +full = [ + "neural", + "nlp", + "optim", + "transformers", +] + +neural = [ + "concision-neural" +] + +nlp = [ + "concision-nlp" +] + +optim = [ + "concision-optim", +] + +transformers = [ + "concision-transformers" +] + +[lib] +bench = true +crate-type = ["cdylib", "rlib"] +doctest = false +test = true + +[build-dependencies] + +[dependencies] +concision-neural = { features = [], optional = true, path = "../neural", version = "0.1.12" } +concision-nlp = { features = [], optional = true, path = "../nlp", version = "0.1.12" } +concision-optim = { features = [], optional = true, path = "../optim", version = "0.1.12" } +concision-transformers = { features = [], optional = true, path = "../transformers", version = "0.1.12" } + +[dev-dependencies] +anyhow.workspace = true +ndarray.workspace = true + +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] + +[target.wasm32-unknown-unknown] + +[target.wasm32-wasi] \ No newline at end of file diff --git a/ml/ml/benches/default.rs b/ml/ml/benches/default.rs new file mode 100644 index 00000000..937f2387 --- /dev/null +++ b/ml/ml/benches/default.rs @@ -0,0 +1,52 @@ +// bench.rs +#![feature(test)] + +extern crate test; + +use std::mem::replace; +use test::Bencher; + +// bench: find the `BENCH_SIZE` first terms of the fibonacci sequence +static BENCH_SIZE: usize = 20; + +// recursive fibonacci +fn fibonacci(n: usize) -> u32 { + if n < 2 { + 1 + } else { + fibonacci(n - 1) + fibonacci(n - 2) + } +} + +// iterative fibonacci +struct Fibonacci { + curr: u32, + next: u32, +} + +impl Iterator for Fibonacci { + type Item = u32; + fn next(&mut self) -> Option { + let new_next = self.curr + self.next; + let new_curr = replace(&mut self.next, new_next); + + Some(replace(&mut self.curr, new_curr)) + } +} + +fn fibonacci_sequence() -> Fibonacci { + Fibonacci { curr: 1, next: 1 } +} + +// function to benchmark must be annotated with `#[bench]` +#[bench] +fn recursive_fibonacci(b: &mut Bencher) { + // exact code to benchmark must be passed as a closure to the iter + // method of Bencher + b.iter(|| (0..BENCH_SIZE).map(fibonacci).collect::>()) +} + +#[bench] +fn iterative_fibonacci(b: &mut Bencher) { + b.iter(|| fibonacci_sequence().take(BENCH_SIZE).collect::>()) +} diff --git a/ml/ml/examples/nn.rs b/ml/ml/examples/nn.rs new file mode 100644 index 00000000..94813f87 --- /dev/null +++ b/ml/ml/examples/nn.rs @@ -0,0 +1,8 @@ +extern crate concision_ml; + + +fn main() -> anyhow::Result<()> { + println!("Welcome to concision!"); + + Ok(()) +} diff --git a/ml/ml/src/lib.rs b/ml/ml/src/lib.rs new file mode 100644 index 00000000..93479eac --- /dev/null +++ b/ml/ml/src/lib.rs @@ -0,0 +1,29 @@ +/* + Appellation: concision-ml + Contrib: FL03 +*/ +//! # concision-ml +//! +//! Concision aims to be a complete machine learning library written in pure Rust. +//! + +#[cfg(feature = "neural")] +pub use concision_neural as neural; +#[cfg(feature = "nlp")] +pub use concision_nlp as nlp; +#[cfg(feature = "optim")] +pub use concision_optim as optim; +#[cfg(feature = "transformers")] +pub use concision_transformers as transformers; + +pub mod prelude { + + #[cfg(feature = "neural")] + pub use concision_neural::prelude::*; + #[cfg(feature = "nlp")] + pub use concision_nlp::prelude::*; + #[cfg(feature = "optim")] + pub use concision_optim::prelude::*; + #[cfg(feature = "transformers")] + pub use concision_transformers::prelude::*; +} diff --git a/ml/ml/tests/default.rs b/ml/ml/tests/default.rs new file mode 100644 index 00000000..0cac1eb5 --- /dev/null +++ b/ml/ml/tests/default.rs @@ -0,0 +1,8 @@ +#[cfg(test)] +#[test] +fn compiles() { + let f = |x: usize, y: usize| x + y; + + assert_eq!(f(10, 10), 20); + assert_ne!(f(1, 1), 3); +} diff --git a/ml/neural/src/neurons/activate/activator.rs b/ml/neural/src/func/activate/activator.rs similarity index 65% rename from ml/neural/src/neurons/activate/activator.rs rename to ml/neural/src/func/activate/activator.rs index 6a36d526..abf8c40e 100644 --- a/ml/neural/src/neurons/activate/activator.rs +++ b/ml/neural/src/func/activate/activator.rs @@ -2,22 +2,10 @@ Appellation: activator Contrib: FL03 */ -use super::{Activate, ActivationFn, ActivationMethod}; +use super::{Activate, LinearActivation}; use std::marker::PhantomData; -pub trait ActivationParams {} - -pub trait LinearActivation: ActivationMethod { - fn rho() -> ActivationFn { - |x| x - } - - fn linear(&self, x: T) -> T { - x - } -} - -pub struct Activator +pub struct Activator where A: Activate, { @@ -45,3 +33,4 @@ where self.method.activate(x) } } + diff --git a/ml/neural/src/neurons/activate/binary.rs b/ml/neural/src/func/activate/binary.rs similarity index 67% rename from ml/neural/src/neurons/activate/binary.rs rename to ml/neural/src/func/activate/binary.rs index 453ff1d0..44e84f22 100644 --- a/ml/neural/src/neurons/activate/binary.rs +++ b/ml/neural/src/func/activate/binary.rs @@ -2,15 +2,18 @@ Appellation: binary Contrib: FL03 */ -use crate::neurons::activate::Activate; -use ndarray::prelude::Array; -use ndarray::Dimension; +use super::Activate; +use ndarray::prelude::{Array, Dimension}; use num::{One, Zero}; +use serde::{Deserialize, Serialize}; +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] pub struct Heavyside; impl Heavyside { - pub fn heavyside(x: T) -> T + pub fn heavyside(x: T) -> T where T: One + PartialOrd + Zero, { diff --git a/ml/neural/src/neurons/activate/mod.rs b/ml/neural/src/func/activate/mod.rs similarity index 73% rename from ml/neural/src/neurons/activate/mod.rs rename to ml/neural/src/func/activate/mod.rs index 3e2fe344..1ec63aa3 100644 --- a/ml/neural/src/neurons/activate/mod.rs +++ b/ml/neural/src/func/activate/mod.rs @@ -45,4 +45,18 @@ where } } -pub(crate) mod utils {} +pub(crate) mod utils { + use num::{One, Zero}; + + pub fn linear_activation(x: &T) -> T where T: Clone { + x.clone() + } + + pub fn heavyside(x: &T) -> T where T: Clone + One + PartialOrd + Zero { + if x.clone() > T::zero() { + T::one() + } else { + T::zero() + } + } +} diff --git a/ml/neural/src/neurons/activate/nonlinear.rs b/ml/neural/src/func/activate/nonlinear.rs similarity index 55% rename from ml/neural/src/neurons/activate/nonlinear.rs rename to ml/neural/src/func/activate/nonlinear.rs index aae3c878..4c39d792 100644 --- a/ml/neural/src/neurons/activate/nonlinear.rs +++ b/ml/neural/src/func/activate/nonlinear.rs @@ -3,25 +3,44 @@ Contrib: FL03 */ use super::Activate; -use ndarray::prelude::{Array, Array1, Axis}; -use ndarray::{Dimension, RemoveAxis, ScalarOperand}; +use ndarray::prelude::{Array, Axis, Dimension, NdFloat}; +use ndarray::RemoveAxis; use num::{Float, Zero}; use serde::{Deserialize, Serialize}; -pub fn softmax(args: Array1) -> Array1 +pub fn softmax(args: Array) -> Array where + D: Dimension, T: Float, { let denom = args.mapv(|x| x.exp()).sum(); args.mapv(|x| x.exp() / denom) } +pub fn softmax_axis(args: Array, axis: Option) -> Array +where + D: Dimension + RemoveAxis, + T: NdFloat, +{ + let exp = args.mapv(|x| x.exp()); + if let Some(axis) = axis { + let denom = exp.sum_axis(Axis(axis)); + exp / denom + } else { + let denom = exp.sum(); + exp / denom + } +} + +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] pub struct ReLU; impl ReLU { - pub fn compute(x: T) -> T { - if x > T::zero() { - x + pub fn relu(args: T) -> T where T: PartialOrd + Zero { + if args > T::zero() { + args } else { T::zero() } @@ -34,15 +53,26 @@ where T: Float, { fn activate(&self, x: Array) -> Array { - x.mapv(|x| Self::compute(x)) + x.mapv(|x| Self::relu(x)) } } +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] pub struct Sigmoid; impl Sigmoid { - pub fn compute(x: T) -> T { - T::one() / (T::one() + (-x).exp()) + pub fn sigmoid(x: T) -> T where T: Float { + (T::one() + (-x).exp()).powi(-2) + } + + pub fn derivative(x: T) -> T where T: Float { + - (T::one() + (-x).exp()).powi(-2) * (-x).exp() + } + + pub fn gradient(args: &Array) -> Array where D: Dimension, T: Float { + args.mapv(|x| Self::derivative(x)) } } @@ -52,24 +82,11 @@ where T: Float, { fn activate(&self, x: Array) -> Array { - x.mapv(|x| Self::compute(x)) + x.mapv(|x| Self::sigmoid(x)) } } -pub fn softmax_axis(args: Array, axis: Option) -> Array -where - T: Float + ScalarOperand, - D: Dimension + RemoveAxis, -{ - let exp = args.mapv(|x| x.exp()); - if let Some(axis) = axis { - let denom = exp.sum_axis(Axis(axis)); - exp / denom - } else { - let denom = exp.sum(); - exp / denom - } -} + #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, @@ -82,12 +99,41 @@ impl Softmax { pub fn new(axis: Option) -> Self { Self { axis } } + + pub fn axis(&self) -> Option { + self.axis + } + + + pub fn softmax(args: Array) -> Array + where + D: Dimension, + T: Float, + { + let denom = args.mapv(|x| x.exp()).sum(); + args.mapv(|x| x.exp() / denom) + } + + pub fn softmax_axis(&self, args: Array,) -> Array + where + T: NdFloat, + D: Dimension + RemoveAxis, + { + let exp = args.mapv(|x| x.exp()); + if let Some(axis) = self.axis { + let denom = exp.sum_axis(Axis(axis)); + exp / denom + } else { + let denom = exp.sum(); + exp / denom + } + } } impl Activate> for Softmax where D: Dimension + RemoveAxis, - T: Float + ScalarOperand, + T: NdFloat, { fn activate(&self, x: Array) -> Array { let exp = x.mapv(|x| x.exp()); diff --git a/ml/neural/src/loss/kinds.rs b/ml/neural/src/func/loss/kinds.rs similarity index 100% rename from ml/neural/src/loss/kinds.rs rename to ml/neural/src/func/loss/kinds.rs diff --git a/ml/neural/src/loss/mod.rs b/ml/neural/src/func/loss/mod.rs similarity index 79% rename from ml/neural/src/loss/mod.rs rename to ml/neural/src/func/loss/mod.rs index a3c19635..a9c2cf48 100644 --- a/ml/neural/src/loss/mod.rs +++ b/ml/neural/src/func/loss/mod.rs @@ -15,12 +15,11 @@ pub(crate) mod kinds; pub mod regress; use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Array1}; -use ndarray::{Dimension, ScalarOperand}; +use ndarray::prelude::{Array, Array1, Array2, Dimension, NdFloat}; use num::{Float, FromPrimitive}; use std::ops; -pub trait Loss { +pub trait Loss where T: Float { fn loss(&self, pred: &Array, target: &Array1) -> T; } @@ -29,7 +28,21 @@ pub trait Loss { pub struct MSE; impl MSE { - pub fn partial_slope( + pub fn partial_slope( + data: &Array2, + target: &Array1, + bias: &Array1, + weights: &Array2, + ) -> T + where + T: FromPrimitive + NdFloat, + { + let pred = data.dot(&weights.t().to_owned()) + bias.clone(); + let error = target - &pred; + let w = data.t().dot(&error) * (-T::from(2).unwrap()); + w.mean().unwrap() + } + pub fn partial( data: &Array, bias: &Array1, slope: &Array, @@ -37,7 +50,7 @@ impl MSE { ) -> (T, T) where D: Dimension, - T: Float + FromPrimitive + ScalarOperand, + T: FromPrimitive + NdFloat, Array: Dot>, as Dot>>::Output: ops::Add, Output = Array> + ops::Sub, Output = Array> diff --git a/ml/neural/src/loss/regress.rs b/ml/neural/src/func/loss/regress.rs similarity index 96% rename from ml/neural/src/loss/regress.rs rename to ml/neural/src/func/loss/regress.rs index 1a59ba33..d9d05b87 100644 --- a/ml/neural/src/loss/regress.rs +++ b/ml/neural/src/func/loss/regress.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use super::Loss; -use ndarray::prelude::{Array, Array1}; +use ndarray::prelude::{Array, Array1, NdFloat}; use ndarray::Dimension; use num::Float; use std::ops; @@ -75,9 +75,11 @@ where pub struct MeanSquaredError; + + impl Loss for MeanSquaredError where - T: Float + ops::AddAssign + ops::DivAssign, + T: NdFloat, { fn loss(&self, pred: &Array, target: &Array1) -> T { let mut res = T::zero(); diff --git a/ml/neural/src/func/mod.rs b/ml/neural/src/func/mod.rs new file mode 100644 index 00000000..9798361d --- /dev/null +++ b/ml/neural/src/func/mod.rs @@ -0,0 +1,29 @@ +/* + Appellation: func + Contrib: FL03 +*/ +//! # Functional +//! +//! This module implements several functional aspects of the neural network. +//! +//! ## Activation +//! +//! The activation functions are implemented as structs that implement the `Fn` trait. +//! +//! ## Loss +//! +//! The loss functions are implemented as structs that implement the `Fn` trait. +pub use self::utils::*; + +pub mod activate; +pub mod loss; + + +pub(crate) mod utils{ +} + +#[cfg(test)] +mod tests { + // use super::*; + +} diff --git a/ml/neural/src/layers/features.rs b/ml/neural/src/layers/features.rs index 236cb41d..a915e7ae 100644 --- a/ml/neural/src/layers/features.rs +++ b/ml/neural/src/layers/features.rs @@ -50,48 +50,48 @@ impl std::fmt::Display for Features { } impl IntoDimension for Features { - type Dim = ndarray::IxDyn; + type Dim = ndarray::Ix2; fn into_dimension(self) -> Self::Dim { - ndarray::IxDyn(&[self.inputs, self.outputs]) + ndarray::Ix2(self.outputs, self.inputs) } } impl From for ndarray::Ix2 { fn from(features: Features) -> Self { - ndarray::Ix2(features.inputs, features.outputs) + ndarray::Ix2(features.outputs, features.inputs) } } impl From for ndarray::IxDyn { fn from(features: Features) -> Self { - ndarray::IxDyn(&[features.inputs, features.outputs]) + ndarray::IxDyn(&[features.outputs, features.inputs]) } } impl From for [usize; 2] { fn from(features: Features) -> Self { - [features.inputs, features.outputs] + [features.outputs, features.inputs] } } impl From<[usize; 2]> for Features { fn from(features: [usize; 2]) -> Self { Self { - inputs: features[0], - outputs: features[1], + inputs: features[1], + outputs: features[0], } } } impl From for (usize, usize) { fn from(features: Features) -> Self { - (features.inputs, features.outputs) + (features.outputs, features.inputs) } } impl From<(usize, usize)> for Features { - fn from((inputs, outputs): (usize, usize)) -> Self { + fn from((outputs, inputs): (usize, usize)) -> Self { Self { inputs, outputs } } } diff --git a/ml/neural/src/layers/kinds.rs b/ml/neural/src/layers/kinds.rs index c5c18ffe..e643e45d 100644 --- a/ml/neural/src/layers/kinds.rs +++ b/ml/neural/src/layers/kinds.rs @@ -33,19 +33,82 @@ pub enum LayerType { Output, } +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] pub struct Position { - pub index: usize, + pub idx: usize, pub kind: LayerType, } impl Position { - pub fn new(index: usize, kind: LayerType) -> Self { - Self { index, kind } - } - pub fn new_input() -> Self { - Self { - index: 0, - kind: LayerType::Input, - } + pub fn new(idx: usize, kind: LayerType) -> Self { + Self { idx, kind } + } + + pub fn input() -> Self { + Self::new(0, LayerType::Input) + } + + pub fn hidden(idx: usize) -> Self { + Self::new(idx, LayerType::Hidden(idx)) + } + + pub fn output(idx: usize) -> Self { + Self::new(idx, LayerType::Output) + } + + pub fn is_input(&self) -> bool { + self.kind().is_input() + } + + pub fn is_hidden(&self) -> bool { + self.kind().is_hidden() + } + + pub fn is_output(&self) -> bool { + self.kind().is_output() + } + + pub fn kind(&self) -> &LayerType { + &self.kind + } + + pub fn position(&self) -> usize { + self.idx + } +} + +impl AsRef for Position { + fn as_ref(&self) -> &usize { + &self.idx + } +} + +impl AsRef for Position { + fn as_ref(&self) -> &LayerType { + &self.kind + } +} + +impl AsMut for Position { + fn as_mut(&mut self) -> &mut usize { + &mut self.idx + } +} + +impl AsMut for Position { + fn as_mut(&mut self) -> &mut LayerType { + &mut self.kind + } +} + +impl From for usize { + fn from(pos: Position) -> Self { + pos.idx + } +} + +impl From for LayerType { + fn from(pos: Position) -> Self { + pos.kind } } diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index e7b0959d..0f585c10 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -1,89 +1,134 @@ /* - Appellation: layer + Appellation: model Contrib: FL03 */ -use super::LayerType; -use crate::params::Bias; -use crate::prelude::{Features, Forward}; - -use ndarray::prelude::Array2; -use ndarray::ScalarOperand; +use super::{Features, LayerType, LayerParams, Position}; +use crate::prelude::{Activate, Forward, LinearActivation, Parameterized, Params}; +use ndarray::prelude::{Array2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -pub struct Layer { - bias: Bias, - features: Features, - layer: LayerType, - weights: Array2, -} -impl Layer { - pub fn new(inputs: usize, outputs: usize, layer: LayerType) -> Self - where - T: SampleUniform, - { - let features = Features::new(inputs, outputs); - let weights = Array2::ones((features.inputs(), features.outputs())); +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct Layer +where + A: Activate>, + T: Float, +{ + activator: A, + pub features: Features, + params: LayerParams, + position: Position, +} +impl Layer +where + A: Activate> + Default, + T: Float, +{ + pub fn new(features: Features, position: Position) -> Self { Self { - bias: Bias::default(), + activator: A::default(), features, - layer, - weights, + params: LayerParams::new(features), + position, } } +} - pub fn bias(&self) -> &Bias { - &self.bias +impl Layer +where + A: Activate>, + T: Float, +{ + pub fn activator(&self) -> &A { + &self.activator } - pub fn layer(&self) -> &LayerType { - &self.layer + pub fn kind(&self) -> &LayerType { + self.position().kind() } - pub fn features(&self) -> Features { - self.features + pub fn position(&self) -> &Position { + &self.position } +} - pub fn set_layer(&mut self, layer: LayerType) { - self.layer = layer; +impl Layer +where + A: Activate>, + T: NdFloat, +{ + pub fn linear(&self, args: &Array2) -> Array2 { + args.dot(&self.params.weights().t()) + self.params.bias() } +} - pub fn weights(&self) -> &Array2 { - &self.weights +impl Layer +where + A: Activate>, + T: Float + SampleUniform, +{ + pub fn init(mut self, biased: bool) -> Self { + self.params = self.params.init(biased); + self } } -impl Layer +impl Forward> for Layer where - T: Float + SampleUniform, + A: Activate>, + T: NdFloat, { - pub fn biased(inputs: usize, outputs: usize, layer: LayerType) -> Self - where - T: SampleUniform, - { - let features = Features::new(inputs, outputs); - let weights = Array2::ones((features.inputs(), features.outputs())); + type Output = Array2; - Self { - bias: Bias::biased(outputs), - features, - layer, - weights, - } + fn forward(&self, args: &Array2) -> Self::Output { + self.activator.activate(self.linear(args)) } } -impl Forward> for Layer +impl Parameterized for Layer where - T: Float + ScalarOperand + 'static, + A: Activate>, + T: Float, { - type Output = Array2; + type Features = Features; + type Params = LayerParams; + + fn features(&self) -> &Features { + &self.features + } - fn forward(&self, data: &Array2) -> Self::Output { - data.dot(&self.weights().t()) + self.bias() + fn features_mut(&mut self) -> &mut Features { + &mut self.features + } + + fn params(&self) -> &LayerParams { + &self.params + } + + fn params_mut(&mut self) -> &mut LayerParams { + &mut self.params + } +} + +impl PartialOrd for Layer +where + A: Activate> + PartialEq, + T: Float, +{ + fn partial_cmp(&self, other: &Self) -> Option { + self.position.partial_cmp(&other.position) + } +} + +impl From for Layer +where + A: Activate> + Default, + T: Float, +{ + fn from(features: Features) -> Self { + Self::new(features, Position::input()) } } diff --git a/ml/neural/src/layers/linear/mod.rs b/ml/neural/src/layers/linear/mod.rs index 78f4a6a5..8a50af31 100644 --- a/ml/neural/src/layers/linear/mod.rs +++ b/ml/neural/src/layers/linear/mod.rs @@ -8,25 +8,25 @@ pub use self::{layer::*, regress::*, utils::*}; pub(crate) mod layer; pub(crate) mod regress; -use ndarray::prelude::Array2; -use num::Float; - -// pub trait Lin where T: Float { - -// fn forward(&self, data: &Array2) -> Array2; -// } pub(crate) mod utils { - use ndarray::prelude::{Array1, Array2}; - use num::Float; + use ndarray::prelude::{Array1, Array2, NdFloat}; - pub fn linear_transformation( + pub fn linear_transformation( data: &Array2, bias: &Array1, weights: &Array2, - ) -> Array2 { + ) -> Array2 where T: NdFloat { data.dot(&weights.t()) + bias } + + pub fn linear_node( + data: &Array2, + bias: &T, + weights: &Array1, + ) -> Array1 where T: NdFloat { + data.dot(&weights.t()) + bias.clone() + } } #[cfg(test)] diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index c70cfeca..71b0414a 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -3,16 +3,17 @@ Contrib: FL03 */ //! # Layers -pub use self::{features::*, kinds::*, layer::*, sublayer::*, utils::*}; +pub use self::{features::*, kinds::*, layer::*, params::*, sublayer::*, utils::*}; pub(crate) mod features; pub(crate) mod kinds; pub(crate) mod layer; +pub(crate) mod params; pub(crate) mod sublayer; pub mod linear; -use crate::neurons::activate::Activate; +use crate::func::activate::Activate; use ndarray::prelude::{Array1, Array2}; use num::Float; @@ -42,8 +43,23 @@ pub(crate) mod utils {} #[cfg(test)] mod tests { - // use super::*; + use super::*; + use crate::core::prelude::linarr; + use crate::func::activate::Softmax; + use crate::prelude::Forward; + use ndarray::prelude::Ix2; #[test] - fn test_layer() {} + fn test_layer() { + let (samples, inputs, outputs) = (20, 5, 3); + let features = Features::new(inputs, outputs); + + let args = linarr::((samples, inputs)).unwrap(); + + let layer = Layer::::from(features).init(true); + + let pred = layer.forward(&args); + + assert_eq!(pred.dim(), (samples, outputs)); + } } diff --git a/ml/neural/src/layers/params.rs b/ml/neural/src/layers/params.rs new file mode 100644 index 00000000..533a1f2d --- /dev/null +++ b/ml/neural/src/layers/params.rs @@ -0,0 +1,107 @@ +/* + Appellation: params + Contrib: FL03 +*/ +use super::Features; +use crate::prelude::Params; +use crate::core::prelude::GenerateRandom; +use ndarray::prelude::{Array1, Array2, Ix2}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use num::Float; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct LayerParams { + bias: Array1, + pub features: Features, + weights: Array2, +} + +impl LayerParams +where + T: Float, +{ + pub fn new(features: Features) -> Self { + Self { + bias: Array1::zeros(features.outputs()), + features, + weights: Array2::zeros(features.out_by_in()), + } + } + + pub fn reset(&mut self) { + self.bias = Array1::zeros(self.features.outputs()); + self.weights = Array2::zeros(self.features.out_by_in()); + } + + pub fn features(&self) -> &Features { + &self.features + } + + pub fn features_mut(&mut self) -> &mut Features { + &mut self.features + } + + pub fn with_bias(mut self, bias: Array1) -> Self { + self.bias = bias; + self + } + + pub fn with_weights(mut self, weights: Array2) -> Self { + self.weights = weights; + self + } +} + +impl LayerParams +where + T: Float + SampleUniform, +{ + pub fn init(mut self, biased: bool) -> Self { + if biased { + self = self.init_bias(); + } + self.init_weight() + } + + pub fn init_bias(mut self) -> Self { + let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); + self.bias = Array1::uniform_between(dk, self.features().outputs()); + self + } + + pub fn init_weight(mut self) -> Self { + let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); + self.weights = Array2::uniform_between(dk, self.features().out_by_in()); + self + } +} + +impl Params for LayerParams +where + T: Float, +{ + fn bias(&self) -> &Array1 { + &self.bias + } + + fn bias_mut(&mut self) -> &mut Array1 { + &mut self.bias + } + + fn weights(&self) -> &Array2 { + &self.weights + } + + fn weights_mut(&mut self) -> &mut Array2 { + &mut self.weights + } + + fn set_bias(&mut self, bias: Array1) { + self.bias = bias; + } + + fn set_weights(&mut self, weights: Array2) { + self.weights = weights; + } +} diff --git a/ml/neural/src/layers/sublayer.rs b/ml/neural/src/layers/sublayer.rs index 368a140a..4563e8d0 100644 --- a/ml/neural/src/layers/sublayer.rs +++ b/ml/neural/src/layers/sublayer.rs @@ -3,6 +3,7 @@ Contrib: FL03 */ use super::Layer; +use crate::func::activate::{Activate, LinearActivation}; use crate::ops::LayerNorm; use crate::prelude::Forward; @@ -11,25 +12,29 @@ use num::{Float, FromPrimitive}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct Sublayer { - layer: Layer, +pub struct Sublayer where A: Activate>, T: Float { + layer: Layer, norm: LayerNorm, } -impl Sublayer +impl Sublayer where + A: Activate>, T: Float, { - pub fn new(layer: Layer, norm: LayerNorm) -> Self { + pub fn new(layer: Layer, norm: LayerNorm) -> Self { Self { layer, norm } } } -impl Sublayer +impl Forward> for Sublayer where + A: Activate>, T: FromPrimitive + NdFloat, { - pub fn forward(&self, data: &Array2) -> Array2 { + type Output = Array2; + + fn forward(&self, data: &Array2) -> Self::Output { let norm = self.norm.forward(data); let layer = data + self.layer.forward(&norm); layer diff --git a/ml/neural/src/lib.rs b/ml/neural/src/lib.rs index 6bba9f13..06eb494a 100644 --- a/ml/neural/src/lib.rs +++ b/ml/neural/src/lib.rs @@ -14,8 +14,8 @@ pub(crate) mod specs; pub(crate) mod utils; pub mod arch; +pub mod func; pub mod layers; -pub mod loss; pub mod masks; pub mod models; pub mod neurons; @@ -28,12 +28,11 @@ pub(crate) use concision_core as core; pub mod prelude { pub use crate::arch::*; + pub use crate::func::{activate::*, loss::*}; pub use crate::layers::linear::*; pub use crate::layers::*; - pub use crate::loss::*; pub use crate::masks::*; - pub use crate::neurons::activate::*; - pub use crate::neurons::{Neuron, Node}; + pub use crate::neurons::*; pub use crate::nn::*; pub use crate::ops::*; pub use crate::params::*; diff --git a/ml/neural/src/models/features.rs b/ml/neural/src/models/features.rs index dad4f0c8..52c2a07c 100644 --- a/ml/neural/src/models/features.rs +++ b/ml/neural/src/models/features.rs @@ -68,10 +68,10 @@ impl std::fmt::Display for Features { } impl IntoDimension for Features { - type Dim = ndarray::IxDyn; + type Dim = ndarray::Ix2; fn into_dimension(self) -> Self::Dim { - ndarray::IxDyn(&[self.inputs, self.outputs]) + ndarray::Ix2(self.outputs, self.inputs) } } diff --git a/ml/neural/src/models/mod.rs b/ml/neural/src/models/mod.rs index 1effcd87..3d1981cb 100644 --- a/ml/neural/src/models/mod.rs +++ b/ml/neural/src/models/mod.rs @@ -10,6 +10,8 @@ pub(crate) mod features; pub(crate) mod model; pub(crate) mod params; +pub mod stack; + pub trait Module { fn add_module(&mut self, module: impl Module); diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs index 6b7ea18c..2ce91ed2 100644 --- a/ml/neural/src/models/model.rs +++ b/ml/neural/src/models/model.rs @@ -2,14 +2,16 @@ Appellation: model Contrib: FL03 */ -use super::{Features, ModelParams, Parameterized}; +use super::{Features, ModelParams,}; +use crate::prelude::{Biased, Parameterized, Weighted}; use ndarray::prelude::{Array2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; pub struct Model { pub features: Features, - params: ModelParams, + children: Vec>, + params: ModelParams } impl Model @@ -19,6 +21,7 @@ where pub fn new(features: Features) -> Self { Self { features, + children: Vec::new(), params: ModelParams::new(features), } } @@ -37,7 +40,7 @@ where T: NdFloat, { pub fn linear(&self, args: &Array2) -> Array2 { - args.dot(&self.params().weights().t()) + self.params().bias() + args.dot(&self.weights().t()) + self.bias() } } @@ -55,6 +58,17 @@ impl Parameterized for Model where T: Float, { + type Features = Features; + type Params = ModelParams; + + fn features(&self) -> &Features { + &self.features + } + + fn features_mut(&mut self) -> &mut Features { + &mut self.features + } + fn params(&self) -> &ModelParams { &self.params } diff --git a/ml/neural/src/models/params.rs b/ml/neural/src/models/params.rs index dddff08c..5521eb68 100644 --- a/ml/neural/src/models/params.rs +++ b/ml/neural/src/models/params.rs @@ -4,43 +4,13 @@ */ use super::Features; use crate::core::prelude::GenerateRandom; -use ndarray::prelude::{Array1, Array2}; +use crate::prelude::Params; +use ndarray::prelude::{Array1, Array2, Ix2}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; +use serde::{Deserialize, Serialize}; -pub trait Parameterized -where - T: Float, -{ - fn bias(&self) -> &Array1 { - self.params().bias() - } - - fn bias_mut(&mut self) -> &mut Array1 { - self.params_mut().bias_mut() - } - - fn params(&self) -> &ModelParams; - - fn params_mut(&mut self) -> &mut ModelParams; - - fn weights(&self) -> &Array2 { - self.params().weights() - } - - fn weights_mut(&mut self) -> &mut Array2 { - self.params_mut().weights_mut() - } - - fn set_bias(&mut self, bias: Array1) { - self.params_mut().set_bias(bias); - } - - fn set_weights(&mut self, weights: Array2) { - self.params_mut().set_weights(weights); - } -} - +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct ModelParams { bias: Array1, pub features: Features, @@ -64,14 +34,6 @@ where self.weights = Array2::zeros(self.features.out_by_in()); } - pub fn bias(&self) -> &Array1 { - &self.bias - } - - pub fn bias_mut(&mut self) -> &mut Array1 { - &mut self.bias - } - pub fn features(&self) -> &Features { &self.features } @@ -80,22 +42,6 @@ where &mut self.features } - pub fn weights(&self) -> &Array2 { - &self.weights - } - - pub fn weights_mut(&mut self) -> &mut Array2 { - &mut self.weights - } - - pub fn set_bias(&mut self, bias: Array1) { - self.bias = bias; - } - - pub fn set_weights(&mut self, weights: Array2) { - self.weights = weights; - } - pub fn with_bias(mut self, bias: Array1) -> Self { self.bias = bias; self @@ -130,3 +76,33 @@ where self } } + + +impl Params for ModelParams +where + T: Float, +{ + fn bias(&self) -> &Array1 { + &self.bias + } + + fn bias_mut(&mut self) -> &mut Array1 { + &mut self.bias + } + + fn weights(&self) -> &Array2 { + &self.weights + } + + fn weights_mut(&mut self) -> &mut Array2 { + &mut self.weights + } + + fn set_bias(&mut self, bias: Array1) { + self.bias = bias; + } + + fn set_weights(&mut self, weights: Array2) { + self.weights = weights; + } +} \ No newline at end of file diff --git a/ml/neural/src/models/stack.rs b/ml/neural/src/models/stack.rs new file mode 100644 index 00000000..3bb37b5b --- /dev/null +++ b/ml/neural/src/models/stack.rs @@ -0,0 +1,32 @@ +/* + Appellation: model + Contrib: FL03 +*/ +use crate::layers::Layer; +use crate::prelude::{Activate, LinearActivation,}; +use ndarray::prelude::Array2; +use num::Float; +use serde::{Deserialize, Serialize}; + + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct Stack +where + A: Activate>, + T: Float, +{ + layers: Vec>, +} + +impl Stack +where + A: Activate> + Default, + T: Float, +{ + pub fn new() -> Self { + Self { + layers: Vec::new(), + } + } +} + diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index 20920e70..f0a825ea 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -3,19 +3,39 @@ Contrib: FL03 */ //! # neurons -pub use self::{neuron::*, node::*, utils::*}; +pub use self::{neuron::*, node::*, params::*, utils::*}; pub(crate) mod neuron; pub(crate) mod node; +pub(crate) mod params; -pub mod activate; +use crate::func::activate::Activate; +use ndarray::prelude::{Array1, Array2, NdFloat}; + +pub trait ArtificialNeuron +where + T: NdFloat, +{ + type Rho: Activate>; + + fn bias(&self) -> T; + + fn forward(&self, args: &Array2) -> Array1 { + self.rho() + .activate(args.dot(&self.weights().t()) + self.bias()) + } + + fn rho(&self) -> &Self::Rho; + + fn weights(&self) -> &Array1; +} pub(crate) mod utils {} #[cfg(test)] mod tests { - use super::activate::{softmax, Activate, Softmax}; use super::*; + use crate::func::activate::{Activate, Softmax, softmax}; use crate::prelude::Forward; // use lazy_static::lazy_static; use ndarray::{array, Array1}; diff --git a/ml/neural/src/neurons/neuron.rs b/ml/neural/src/neurons/neuron.rs index caf4c63d..4becc79b 100644 --- a/ml/neural/src/neurons/neuron.rs +++ b/ml/neural/src/neurons/neuron.rs @@ -2,7 +2,7 @@ Appellation: neuron Contrib: FL03 */ -use super::activate::{Activate, LinearActivation}; +use crate::func::activate::{Activate, LinearActivation}; use crate::core::GenerateRandom; use crate::prelude::Forward; use ndarray::prelude::{Array1, Array2, NdFloat}; @@ -10,24 +10,6 @@ use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use rand::Rng; -pub trait ArtificialNeuron -where - T: NdFloat, -{ - type Rho: Activate>; - - fn bias(&self) -> T; - - fn forward(&self, args: &Array2) -> Array1 { - self.rho() - .activate(args.dot(&self.weights().t()) + self.bias()) - } - - fn rho(&self) -> &Self::Rho; - - fn weights(&self) -> &Array1; -} - /// Artificial Neuron #[derive(Clone, Debug, PartialEq)] pub struct Neuron @@ -117,7 +99,7 @@ where impl Neuron where T: Float + SampleUniform, - A: Activate> + Default, + A: Activate>, { pub fn init(mut self, biased: bool) -> Self { if biased { @@ -152,7 +134,7 @@ where impl Forward> for Neuron where T: NdFloat, - A: Activate> + Default, + A: Activate>, { type Output = Array1; diff --git a/ml/neural/src/neurons/params.rs b/ml/neural/src/neurons/params.rs new file mode 100644 index 00000000..403428fe --- /dev/null +++ b/ml/neural/src/neurons/params.rs @@ -0,0 +1,137 @@ +/* + Appellation: params + Contrib: FL03 +*/ +use crate::core::prelude::GenerateRandom; + +use crate::prelude::Params; +use ndarray::prelude::{Array0, Array1, Array2, Ix1, NdFloat}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use num::{Float, FromPrimitive}; + +#[derive(Clone, Debug, PartialEq)] +pub struct NeuronParams +where + T: Float, +{ + bias: Array0, + pub features: usize, + weights: Array1, +} + +impl NeuronParams +where + T: Float, +{ + pub fn new(features: usize) -> Self { + Self { + bias: Array0::zeros(()), + features, + weights: Array1::zeros(features), + } + } + + pub fn features(&self) -> usize { + self.features + } + + pub fn features_mut(&mut self) -> &mut usize { + &mut self.features + } + + pub fn set_features(&mut self, features: usize) { + self.features = features; + } + + + pub fn with_bias(mut self, bias: Array0) -> Self { + self.bias = bias.into(); + self + } + + pub fn with_features(mut self, features: usize) -> Self { + self.features = features; + self + } + + pub fn with_weights(mut self, weights: Array1) -> Self { + self.weights = weights; + self + } +} + +impl NeuronParams +where + T: Float + SampleUniform, +{ + pub fn init(mut self, biased: bool) -> Self { + if biased { + self = self.init_bias(); + } + self.init_weight() + } + + pub fn init_bias(mut self) -> Self { + let dk = (T::one() / T::from(self.features).unwrap()).sqrt(); + self.bias = Array0::uniform_between(dk, ()); + self + } + + pub fn init_weight(mut self) -> Self { + let features = self.features; + let dk = (T::one() / T::from(features).unwrap()).sqrt(); + self.weights = Array1::uniform_between(dk, features); + self + } +} + +impl NeuronParams +where + T: FromPrimitive + NdFloat, +{ + pub fn apply_gradient(&mut self, gamma: T, gradient: G) + where + G: Fn(&Array1) -> Array1, + { + let mut grad = gradient(self.weights()); + grad /= grad.mapv(|ws| ws.powi(2)).sum().sqrt(); + self.weights_mut().scaled_add(-gamma, &grad); + self.weights /= self.weights().mapv(|ws| ws.powi(2)).sum().sqrt(); + } + + pub fn linear(&self, data: &Array2) -> Array1 { + data.dot(&self.weights().t()) + self.bias() + } +} + +impl Params for NeuronParams +where + T: Float, +{ + + fn bias(&self) -> &Array0 { + &self.bias + } + + fn bias_mut(&mut self) -> &mut Array0 { + &mut self.bias + } + + fn weights(&self) -> &Array1 { + &self.weights + } + + fn weights_mut(&mut self) -> &mut Array1 { + &mut self.weights + } + + fn set_bias(&mut self, bias: Array0) { + self.bias = bias; + } + + fn set_weights(&mut self, weights: Array1) { + self.weights = weights; + + } +} + diff --git a/ml/neural/src/nn/deep.rs b/ml/neural/src/nn/deep.rs new file mode 100644 index 00000000..8eced8a0 --- /dev/null +++ b/ml/neural/src/nn/deep.rs @@ -0,0 +1,54 @@ +/* + Appellation: network + Contrib: FL03 +*/ +use crate::func::activate::{Activate, LinearActivation}; +use crate::prelude::{Forward, Layer, Parameterized}; + +use ndarray::prelude::{Array2, NdFloat}; +use num::Float; + +pub struct DeepNetwork where T: Float, I: Activate>, H: Activate>, O: Activate> { + pub input: Layer, + pub hidden: Vec>, + pub output: Layer, +} + +impl DeepNetwork where T: Float, I: Activate>, H: Activate>, O: Activate> { + pub fn new(input: Layer, hidden: Vec>, output: Layer) -> Self { + Self { + input, + hidden, + output, + } + } + + pub fn validate_dims(&self) -> bool { + let mut dim = true; + for (i, layer) in self.hidden.iter().enumerate() { + if i== 0 { + dim = self.input.features().outputs() == self.hidden[i].features().inputs() + } + else if i == self.hidden.len() - 1 { + dim = dim && layer.features().outputs() == self.output.features().inputs(); + } else { + dim = dim && layer.features().outputs() == self.hidden[i + 1].features().inputs(); + } + } + dim + } +} + +impl Forward> for DeepNetwork where T: NdFloat, I: Activate>, H: Activate>, O: Activate> { + type Output = Array2; + + fn forward(&self, args: &Array2) -> Self::Output { + let mut out = self.input.forward(args); + for layer in &self.hidden { + out = layer.forward(&out); + } + self.output.forward(&out) + } +} + + diff --git a/ml/neural/src/nn/mod.rs b/ml/neural/src/nn/mod.rs index 1a3c4ec3..b2608fbb 100644 --- a/ml/neural/src/nn/mod.rs +++ b/ml/neural/src/nn/mod.rs @@ -3,9 +3,9 @@ Contrib: FL03 */ //! # Neural Network -pub use self::{network::*, utils::*}; +pub use self::{deep::*, utils::*}; -pub(crate) mod network; +pub(crate) mod deep; use crate::layers::Layer; use crate::Trainable; @@ -17,6 +17,10 @@ pub trait NeuralNet: Trainable { } fn layers(&self) -> &[Layer]; + + fn input_layer(&self) -> &Layer { + &self.layers()[0] + } } pub(crate) mod utils {} diff --git a/ml/neural/src/nn/network.rs b/ml/neural/src/nn/network.rs deleted file mode 100644 index c0515db5..00000000 --- a/ml/neural/src/nn/network.rs +++ /dev/null @@ -1,12 +0,0 @@ -/* - Appellation: network - Contrib: FL03 -*/ -use crate::layers::Layer; -use num::Float; - -pub struct NeuralNetwork { - pub input: Layer, - pub hidden: Vec>, - pub output: Layer, -} diff --git a/ml/neural/src/params/mod.rs b/ml/neural/src/params/mod.rs index ead1c0ab..d458d655 100644 --- a/ml/neural/src/params/mod.rs +++ b/ml/neural/src/params/mod.rs @@ -6,85 +6,129 @@ //! //! ## Overview //! -pub use self::{bias::*, shapes::*, utils::*, weight::*}; +pub use self::{bias::*, param::*, shapes::*, utils::*, weight::*}; pub(crate) mod bias; +pub(crate) mod param; pub(crate) mod shapes; pub(crate) mod weight; -use crate::core::prelude::Borrowed; - +use ndarray::IntoDimension; use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Array2, Ix2, NdFloat}; -use ndarray::Dimension; +use ndarray::prelude::{Array, Dimension, Ix2,}; use num::Float; -pub trait WeightTensor + + +pub trait Biased where - Array: Dot, Output = Array>, D: Dimension, - T: NdFloat, + T: Float, + Self: Weighted { + /// Returns an owned reference to the bias of the layer. + fn bias(&self) -> &Array; + /// Returns a mutable reference to the bias of the layer. + fn bias_mut(&mut self) -> &mut Array; +} + + + +pub trait Weighted +where + D: Dimension, + T: Float, +{ + /// Returns an owned reference to the weights of the layer. fn weights(&self) -> &Array; + /// Returns a mutable reference to the weights of the layer. fn weights_mut(&mut self) -> &mut Array; } -pub trait Parameter {} +pub trait WeightedExt: Weighted +where + Array: Dot, Output = Array>, + D: Dimension, + T: Float, +{ + +} pub trait Params where D: Dimension, T: Float, { + + /// Returns an owned reference to the bias of the layer. fn bias(&self) -> &Array; + /// Returns a mutable reference to the bias of the layer. fn bias_mut(&mut self) -> &mut Array; + /// Returns an owned reference to the weights of the layer. fn weights(&self) -> &Array; + /// Returns a mutable reference to the weights of the layer. fn weights_mut(&mut self) -> &mut Array; + /// Sets the bias of the layer. + fn set_bias(&mut self, bias: Array); + /// Sets the weights of the layer. + fn set_weights(&mut self, weights: Array); } -pub trait Biased -where - T: Float, -{ - fn bias(&self) -> &Bias; - fn bias_mut(&mut self) -> &mut Bias; -} -impl WeightTensor for Array +pub trait Parameterized where - Array: Borrowed> + Dot, Output = Array>, D: Dimension, - T: NdFloat, + T: Float, { - fn weights(&self) -> &Array { - self.as_ref() + type Features: IntoDimension; + type Params: Params; + + fn features(&self) -> &Self::Features; + + fn features_mut(&mut self) -> &mut Self::Features; + + fn params(&self) -> &Self::Params; + + fn params_mut(&mut self) -> &mut Self::Params; + + fn set_bias(&mut self, bias: Array) { + self.params_mut().set_bias(bias); } - fn weights_mut(&mut self) -> &mut Array { - self.as_mut() + fn set_weights(&mut self, weights: Array) { + self.params_mut().set_weights(weights); } } -pub trait Weighted +impl Biased for P where + D: Dimension, + P: Parameterized, T: Float, + ::Smaller: Dimension, +

>::Params: 'static, { - fn weights(&self) -> &Array2; + fn bias(&self) -> &Array { + self.params().bias() + } - fn weights_mut(&mut self) -> &mut Array2; + fn bias_mut(&mut self) -> &mut Array { + self.params_mut().bias_mut() + } } -impl Weighted for S +impl Weighted for P where - S: AsMut> + AsRef>, + P: Parameterized, + D: Dimension, T: Float, { - fn weights(&self) -> &Array2 { - self.as_ref() + fn weights(&self) -> &Array { + self.params().weights() } - fn weights_mut(&mut self) -> &mut Array2 { - self.as_mut() + fn weights_mut(&mut self) -> &mut Array { + self.params_mut().weights_mut() } } diff --git a/ml/neural/src/params/param.rs b/ml/neural/src/params/param.rs new file mode 100644 index 00000000..c2605604 --- /dev/null +++ b/ml/neural/src/params/param.rs @@ -0,0 +1,115 @@ +/* + Appellation: model + Contrib: FL03 +*/ +use crate::layers::Features; +use crate::core::prelude::GenerateRandom; +use ndarray::prelude::{Array1, Array2}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use num::Float; +use serde::{Deserialize, Serialize}; + +pub trait ParamFeatures { + fn inputs(&self) -> usize; + + fn outputs(&self) -> usize; + + fn in_by_out(&self) -> (usize, usize) { + (self.inputs(), self.outputs()) + } + + fn out_by_in(&self) -> (usize, usize) { + (self.outputs(), self.inputs()) + } +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct ParamGroup where T: Float { + bias: Array1, + pub features: Features, + weights: Array2, +} + +impl ParamGroup +where + T: Float, +{ + pub fn new(features: Features) -> Self { + Self { + bias: Array1::zeros(features.outputs()), + features, + weights: Array2::zeros(features.out_by_in()), + } + } + + pub fn reset(&mut self) { + self.bias = Array1::zeros(self.features.outputs()); + self.weights = Array2::zeros(self.features.out_by_in()); + } + + pub fn bias(&self) -> &Array1 { + &self.bias + } + + pub fn bias_mut(&mut self) -> &mut Array1 { + &mut self.bias + } + + pub fn features(&self) -> &Features { + &self.features + } + + pub fn features_mut(&mut self) -> &mut Features { + &mut self.features + } + + pub fn weights(&self) -> &Array2 { + &self.weights + } + + pub fn weights_mut(&mut self) -> &mut Array2 { + &mut self.weights + } + + pub fn set_bias(&mut self, bias: Array1) { + self.bias = bias; + } + + pub fn set_weights(&mut self, weights: Array2) { + self.weights = weights; + } + + pub fn with_bias(mut self, bias: Array1) -> Self { + self.bias = bias; + self + } + + pub fn with_weights(mut self, weights: Array2) -> Self { + self.weights = weights; + self + } +} + +impl ParamGroup +where + T: Float + SampleUniform, +{ + pub fn init(mut self, biased: bool) -> Self { + if biased { + self = self.init_bias(); + } + self.init_weight() + } + + pub fn init_bias(mut self) -> Self { + let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); + self.bias = Array1::uniform_between(dk, self.features().outputs()); + self + } + + pub fn init_weight(mut self) -> Self { + let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); + self.weights = Array2::uniform_between(dk, self.features().out_by_in()); + self + } +} diff --git a/ml/optim/examples/descent.rs b/ml/optim/examples/descent.rs index 98d29aa0..f53c4f8d 100644 --- a/ml/optim/examples/descent.rs +++ b/ml/optim/examples/descent.rs @@ -1,4 +1,4 @@ -use concision_neural::prelude::{Features, Linear}; +use concision_neural::prelude::{Features, Linear, Sigmoid}; use concision_neural::prop::Forward; use concision_optim::prelude::{gradient_descent, GradientDescent}; use ndarray::prelude::{Array, Array1}; @@ -13,9 +13,9 @@ fn main() -> anyhow::Result<()> { let (epochs, gamma) = (500, 0.001); - // basic_descent(epochs, features, gamma)?; + basic_descent(epochs, features, gamma)?; - sample_descent(epochs, features, gamma)?; + // sample_descent(epochs, features, gamma)?; Ok(()) } @@ -25,7 +25,7 @@ pub fn basic_descent(epochs: usize, features: Features, gamma: f64) -> anyhow::R println!( "{:?}", - gradient_descent(model.weights_mut(), epochs, gamma, |a| a.clone()) + gradient_descent(model.weights_mut(), epochs, gamma, Sigmoid::gradient) ); Ok(()) } @@ -52,7 +52,7 @@ pub fn sample_descent(epochs: usize, features: Features, gamma: f64) -> anyhow:: let mut grad = GradientDescent::new(gamma, model); let mut losses = Array1::zeros(epochs); for e in 0..epochs { - let cost = grad.logit(&x, &y)?; + let cost = grad.step(&x, &y)?; losses[e] = cost; } println!("Losses:\n\n{:?}\n", &losses); diff --git a/ml/optim/examples/norm.rs b/ml/optim/examples/norm.rs index 54384fb3..065b5884 100644 --- a/ml/optim/examples/norm.rs +++ b/ml/optim/examples/norm.rs @@ -3,8 +3,8 @@ use concision_optim::prelude::Norm; use ndarray::prelude::Array; fn main() -> anyhow::Result<()> { - let (samples, inputs) = (20, 5); - let outputs = 1; + let (samples, inputs) = (20, 3); + let outputs = 8; let features = Features::new(inputs, outputs); @@ -27,6 +27,15 @@ pub fn sample_norm(features: Features, samples: usize) -> anyhow::Result<()> { &args.l1(), &args.l2() ); + let args = Array::linspace(1., features.inputs() as f64, features.inputs()) + .into_shape(features.inputs()) + .unwrap(); + println!( + "Norms:\n\nL0: {:?}\nL1: {:?}\nL2: {:?}\n", + &args.l0(), + &args.l1(), + &args.l2() + ); Ok(()) } diff --git a/ml/optim/src/cost/mod.rs b/ml/optim/src/cost/mod.rs index bba10846..f5c919ab 100644 --- a/ml/optim/src/cost/mod.rs +++ b/ml/optim/src/cost/mod.rs @@ -16,7 +16,7 @@ pub trait Cost where T: Float, { - fn cost(&self, pred: &T, target: &T) -> T; + fn cost(&self, other: &Self) -> T; } pub trait CostArr @@ -32,14 +32,17 @@ pub(crate) mod utils { use ndarray::prelude::Array; use ndarray::Dimension; - use num::Float; + use num::{Float, FromPrimitive}; - pub fn mse<'a, T, D>(pred: &Array, target: &Array) -> Array + pub fn mse<'a, T, D>(pred: &Array, target: &Array) -> T where - T: Float, + T: Float + FromPrimitive, D: Dimension, { - (pred - target).mapv(|x| x.powi(2)) + (pred - target) + .mapv(|x| x.powi(2)) + .mean() + .unwrap_or_else(T::zero) } pub fn mae<'a, T, D>(pred: &Array, target: &Array) -> Array diff --git a/ml/optim/src/grad/descent.rs b/ml/optim/src/grad/descent.rs index 94631ada..47b2a60c 100644 --- a/ml/optim/src/grad/descent.rs +++ b/ml/optim/src/grad/descent.rs @@ -5,23 +5,9 @@ use crate::neural::layers::linear::Linear; use crate::neural::prelude::Forward; use crate::prelude::Norm; -use ndarray::prelude::{Array1, Array2}; +use ndarray::prelude::{Array1, Array2,}; use ndarray_stats::DeviationExt; -pub fn gradient_descent( - weights: &mut Array1, - epochs: usize, - gamma: f64, - partial: impl Fn(&Array1) -> Array1, -) -> Array1 { - let mut losses = Array1::zeros(epochs); - for e in 0..epochs { - *weights = weights.clone() - gamma * partial(&weights.clone()); - losses[e] = weights.mean().unwrap_or_default(); - } - losses -} - #[derive(Clone)] pub struct GradientDescent { pub gamma: f64, @@ -67,14 +53,14 @@ impl GradientDescent { self } - pub fn logit(&mut self, data: &Array2, targets: &Array1) -> anyhow::Result { + pub fn gradient(&mut self, data: &Array2, targets: &Array1) -> anyhow::Result { // let pred = self.model.forward(data); let gradient = |p: &Array1| { let error = targets - &data.dot(&(p / p.l2())); let scale = -1.0 / (2.0 * data.len() as f64); let grad = scale * error.dot(data); - grad + &grad / grad.l2() }; self.model.apply_gradient(self.gamma, &gradient); @@ -83,17 +69,16 @@ impl GradientDescent { } pub fn step(&mut self, data: &Array2, targets: &Array1) -> anyhow::Result { + // let pred = self.model.forward(data); let gradient = |p: &Array1| { - let pred = data.dot(p); - let error = targets - pred; - let scale = -2.0 / data.len() as f64; - let grad = scale * data.t().dot(&error); + let error = targets - &data.dot(&(p / p.l2())); + let scale = -1.0 / (2.0 * data.len() as f64); + let grad = scale * error.dot(data); &grad / grad.l2() }; self.model.apply_gradient(self.gamma, &gradient); - // let loss = targets.mean_sq_err(&self.model.forward(data))?; let loss = targets.mean_sq_err(&self.model.forward(data))?; Ok(loss) } diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs index abd89ec7..27818c44 100644 --- a/ml/optim/src/grad/gradient.rs +++ b/ml/optim/src/grad/gradient.rs @@ -3,14 +3,58 @@ Contrib: FL03 */ use crate::neural::models::ModelParams; -use ndarray::prelude::{Array1, Array2}; +use ndarray::prelude::{Array1, Array2, NdFloat}; use num::Float; +pub struct Grad +where + T: Float, +{ + gamma: T, + params: Vec>, + objective: fn(&Array2) -> Array2, +} + +impl Grad where T: Float { + pub fn gamma(&self) -> T { + self.gamma + } + + pub fn gamma_mut(&mut self) -> &mut T { + &mut self.gamma + } + + pub fn objective(&self) -> fn(&Array2) -> Array2 { + self.objective + } + + pub fn params(&self) -> &Vec> { + &self.params + } + + pub fn params_mut(&mut self) -> &mut Vec> { + &mut self.params + } + + +} + +impl Grad where T: NdFloat { + + pub fn step(&mut self, x: &Array2, y: &Array2) -> anyhow::Result { + let mut cost = T::zero(); + Ok(cost) + } + + +} + pub struct GradStep where T: Float, { gamma: T, + params: ModelParams, } diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index d787e653..fec658c0 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -10,53 +10,74 @@ pub(crate) mod gradient; pub mod sgd; -use num::Float; +pub struct BatchParams { + pub batch_size: usize, +} -pub trait Descent { - type Params; +pub struct DescentParams { + pub batch_size: usize, + pub epochs: usize, + pub gamma: f64, // learning rate - fn descent(&self) -> Vec; + pub lambda: f64, // decay rate + pub mu: f64, // momentum rate + pub nesterov: bool, + pub tau: f64, // momentum damper } -pub trait LearningRate -where - T: Float, -{ - fn gamma(&self) -> T; -} +pub(crate) mod utils { + use ndarray::prelude::{Array, Array1, Dimension, NdFloat}; + use num::FromPrimitive; -pub trait Momentum -where - T: Float, -{ - fn mu(&self) -> T; // Momentum Rate -} + pub fn gradient_descent( + params: &mut Array, + epochs: usize, + gamma: T, + partial: impl Fn(&Array) -> Array, + ) -> Array1 where D: Dimension, T: FromPrimitive + NdFloat { + let mut losses = Array1::zeros(epochs); + for e in 0..epochs { + let grad = partial(params); + params.scaled_add(-gamma, &grad); + losses[e] = params.mean().unwrap_or_else(T::zero); + } + losses + } -pub trait Nesterov: Momentum { - fn nestrov(&self) -> bool; + pub fn gradient_descent_step( + params: &mut Array, + gamma: T, + partial: impl Fn(&Array) -> Array, + ) -> T where D: Dimension, T: FromPrimitive + NdFloat { + let grad = partial(params); + params.scaled_add(-gamma, &grad); + params.mean().unwrap_or_else(T::zero) + } } -pub trait Decay -where - T: Float, -{ - fn lambda(&self) -> T; // Decay Rate -} +#[cfg(test)] +mod tests { -pub trait Dampener -where - T: Float, -{ - fn tau(&self) -> T; // Momentum Damper -} + use super::*; + use crate::neural::prelude::{Features, Layer, LinearActivation, Weighted}; + use ndarray::prelude::Array2; -pub struct DescentParams { - pub batch_size: usize, - pub epochs: usize, - pub gamma: f64, // learning rate -} + fn test_grad(args: &Array2) -> Array2 { + args.clone() + } -pub(crate) mod utils {} + #[test] + fn descent() { + let (_samples, inputs) = (20, 5); + let outputs = 1; -#[cfg(test)] -mod tests {} + let (epochs, gamma) = (10, 0.001); + + let features = Features::new(inputs, outputs); + + let mut model = Layer::::from(features).init(true); + + let losses = gradient_descent(&mut model.weights_mut(), epochs, gamma, test_grad); + assert_eq!(losses.len(), epochs); + } +} diff --git a/ml/optim/src/norm/mod.rs b/ml/optim/src/norm/mod.rs index 5ee09d74..348696f7 100644 --- a/ml/optim/src/norm/mod.rs +++ b/ml/optim/src/norm/mod.rs @@ -9,9 +9,10 @@ pub use self::{kinds::*, normalizer::*, utils::*}; pub(crate) mod kinds; pub(crate) mod normalizer; -use ndarray::prelude::{Array, NdFloat}; +use ndarray::prelude::Array; use ndarray::Dimension; use ndarray_stats::QuantileExt; +use num::Float; pub trait Normalize { type Output; @@ -30,7 +31,7 @@ pub trait Norm { impl Norm for Array where D: Dimension, - T: NdFloat, + T: Float, { fn l0(&self) -> T { *self.max().expect("No max value") @@ -41,19 +42,20 @@ where } fn l2(&self) -> T { - self.mapv(|xs| xs.powi(2)).sum().sqrt() + self.fold(T::zero(), |b, a| b + a.powi(2)).sqrt() } } pub(crate) mod utils { - use ndarray::prelude::{Array, NdFloat}; + use ndarray::prelude::Array; use ndarray::Dimension; use ndarray_stats::QuantileExt; + use num::Float; pub fn l0_norm(args: &Array) -> T where D: Dimension, - T: NdFloat, + T: Float, { *args.max().expect("No max value") } @@ -61,7 +63,7 @@ pub(crate) mod utils { pub fn l1_norm(args: &Array) -> T where D: Dimension, - T: NdFloat, + T: Float, { args.mapv(|xs| xs.abs()).sum() } @@ -69,7 +71,7 @@ pub(crate) mod utils { pub fn l2_norm(args: &Array) -> T where D: Dimension, - T: NdFloat, + T: Float, { args.mapv(|xs| xs.powi(2)).sum().sqrt() } diff --git a/ml/optim/src/specs.rs b/ml/optim/src/specs.rs index 9198b4b5..ed8c9113 100644 --- a/ml/optim/src/specs.rs +++ b/ml/optim/src/specs.rs @@ -2,13 +2,15 @@ Appellation: specs Contrib: FL03 */ -use ndarray::prelude::{Array1, Array2}; +use ndarray::prelude::{Array, Array1, Array2, Dimension}; +use num::Float; -pub trait Gradient { - type Params; - type Model; +pub trait Gradient where T: Float { + fn partial(&self, x: T) -> T; - fn gradient(&self, x: &Array2, y: &Array1) -> T; + fn gradient(&self, args: &Array) -> Array where D: Dimension { + args.mapv(|xs| self.partial(xs)) + } } pub trait Objective { @@ -18,11 +20,42 @@ pub trait Objective { } pub trait PartialDerivative { - type Model; + type Args; - fn partial_derivative(&self, x: &Array2, y: &Array1) -> Array2; + fn partial_derivative(&self, args: Self::Args) -> T; } pub trait Minimize { fn minimize(&self, scale: T) -> Self; } + + +pub trait LearningRate +where + T: Float, +{ + fn gamma(&self) -> T; +} + +pub trait Momentum +where + T: Float, +{ + fn mu(&self) -> T; // Momentum Rate + + fn nestrov(&self) -> bool; +} + +pub trait Decay +where + T: Float, +{ + fn lambda(&self) -> T; // Decay Rate +} + +pub trait Dampener +where + T: Float, +{ + fn tau(&self) -> T; // Momentum Damper +} diff --git a/ml/optim/src/utils.rs b/ml/optim/src/utils.rs index 17e3f332..65072c90 100644 --- a/ml/optim/src/utils.rs +++ b/ml/optim/src/utils.rs @@ -3,7 +3,8 @@ Contrib: FL03 */ use crate::prelude::FTOL; -use ndarray::prelude::Array1; +use ndarray::prelude::{Array, Array1, Dimension}; +use num::Float; pub fn minimize_inner( w: &mut Array1, @@ -31,6 +32,38 @@ where } } -pub fn norm_l2(a_s: &Array1) -> f64 { - a_s.fold(0.0, |b, a| b + a * a) +pub fn minimize( + w: &mut Array1, + fg: F, + epsilon: f64, + max_iter: usize, +) -> anyhow::Result<(&mut Array1, f64, Array1)> +where + F: Fn(&Array1) -> (f64, Array1), +{ + let (mut fp, mut gp) = fg(&w); // (cost, gradient) + + for _ in 0..max_iter { + w.scaled_add(-epsilon, &gp); + let (f, g) = fg(&w); + + let exp = epsilon * norm_l2(&g); + let delta = fp - f; // actual descrease; last - current + if delta < exp * 0.5 { + return Err(anyhow::anyhow!("Not enough decrease")); + } else if delta < FTOL { + return Ok((w, f, g)); + } + fp = f; + gp = g; + } + Ok((w, fp, gp)) +} + +pub fn norm_l2(arr: &Array) -> T +where + D: Dimension, + T: Float, +{ + arr.fold(T::zero(), |b, a| b + a.powi(2)) } diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index 6a10204d..c579da0f 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -4,9 +4,8 @@ */ use super::params::{HeadShape, QKV}; use super::{Head, Weight}; -use crate::neural::neurons::activate::{Activate, Softmax}; -use ndarray::prelude::Array2; -use ndarray::ScalarOperand; +use crate::neural::func::activate::{Activate, Softmax}; +use ndarray::prelude::{Array2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; @@ -60,7 +59,7 @@ where } } -impl AttentionHead { +impl AttentionHead where T: NdFloat { pub fn attention(&mut self, data: &Array2) -> Array2 { // multiply the data by the wieghted query, key, and value matrices, respectively let weighted = data * self.weights(); @@ -68,7 +67,7 @@ impl AttentionHead { // compute the attention score let inner = (q.dot(&k.t()) + self.mask.clone()) * self.scale(); - Activate::activate(&Softmax::default(), inner).dot(&v) + Softmax::default().activate(inner).dot(&v) } } diff --git a/ml/transformers/src/attention/mod.rs b/ml/transformers/src/attention/mod.rs index 0e6289a1..da05cc6c 100644 --- a/ml/transformers/src/attention/mod.rs +++ b/ml/transformers/src/attention/mod.rs @@ -19,8 +19,7 @@ pub mod params; use crate::core::prelude::BoxResult; use crate::prelude::BaseDim; -use ndarray::prelude::{Array, Array2, Ix2}; -use ndarray::{Dimension, ScalarOperand}; +use ndarray::prelude::{Array, Array2, Dimension, Ix2, NdFloat}; use num::Float; use std::ops::Mul; @@ -29,10 +28,8 @@ pub type InputArray = Array; pub type AttentionArray = Array; -pub trait Attention { +pub trait Attention { fn attention(&self, data: &Array2) -> BoxResult> - where - T: ScalarOperand, { // let (seq, model) = data.dim(); @@ -85,11 +82,9 @@ pub trait Weights: Mul, Output = Self> { pub(crate) mod utils { use crate::neural::prelude::{Activate, Softmax}; - use ndarray::prelude::Array2; - use ndarray::ScalarOperand; - use num::Float; + use ndarray::prelude::{Array2, NdFloat}; - pub fn attention( + pub fn attention( query: &Array2, key: &Array2, value: &Array2, diff --git a/ml/transformers/src/attention/multi/mod.rs b/ml/transformers/src/attention/multi/mod.rs index ab36daad..334cc385 100644 --- a/ml/transformers/src/attention/multi/mod.rs +++ b/ml/transformers/src/attention/multi/mod.rs @@ -11,13 +11,11 @@ use crate::attention::Weight; use crate::core::prelude::BoxResult; use crate::neural::prelude::Mask; use crate::ops::Split; -use ndarray::prelude::Array2; -use ndarray::ScalarOperand; -use num::Float; +use ndarray::prelude::{Array2, NdFloat}; pub trait MultiHead where - T: Float + ScalarOperand, + T: NdFloat, { fn attention(&mut self, data: &Array2, mask: &Mask) -> BoxResult> { let weighted = data * self.weights(); @@ -35,9 +33,8 @@ pub(crate) mod utils { use crate::attention::attention; use crate::neural::prelude::Mask; use crate::ops::Merge; - use ndarray::prelude::{s, Array2, Array3, Array4}; - use ndarray::{ScalarOperand, ShapeError}; - use num::Float; + use ndarray::prelude::{s, Array2, Array3, Array4, NdFloat}; + use ndarray::ShapeError; pub fn batched_multihead( query: &Array4, @@ -67,7 +64,7 @@ pub(crate) mod utils { mask: &Mask, ) -> Result, ShapeError> where - T: Float + ScalarOperand, + T: NdFloat, { let (heads, _, _) = query.dim(); let mut score = Array3::::zeros(query.dim()); diff --git a/ml/transformers/src/ffn/network.rs b/ml/transformers/src/ffn/network.rs index f6f060cf..aeccec92 100644 --- a/ml/transformers/src/ffn/network.rs +++ b/ml/transformers/src/ffn/network.rs @@ -4,7 +4,7 @@ */ use super::FFNParams; use crate::neural::layers::linear::LinearLayer; -use crate::neural::neurons::activate::{Activate, ReLU}; +use crate::neural::func::activate::{Activate, ReLU}; use crate::neural::prelude::Forward; use ndarray::prelude::Array2; use serde::{Deserialize, Serialize}; From 0d76c6fb8fc8309a0d5c0a6779d718ea186bc064 Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 17 Nov 2023 14:31:27 -0600 Subject: [PATCH 054/118] Refactor model and optimizer modules Signed-off-by: FL03 --- core/src/errors/mod.rs | 2 - ml/ml/examples/nn.rs | 1 - ml/ml/src/lib.rs | 2 +- ml/neural/src/func/activate/activator.rs | 29 ++---- ml/neural/src/func/activate/mod.rs | 42 +++++---- ml/neural/src/func/activate/nonlinear.rs | 28 ++++-- ml/neural/src/func/loss/mod.rs | 5 +- ml/neural/src/func/loss/regress.rs | 2 - ml/neural/src/func/mod.rs | 15 ++- ml/neural/src/layers/kinds.rs | 4 +- ml/neural/src/layers/layer.rs | 3 +- ml/neural/src/layers/linear/mod.rs | 15 +-- ml/neural/src/layers/params.rs | 2 +- ml/neural/src/layers/sublayer.rs | 8 +- ml/neural/src/models/features.rs | 115 ----------------------- ml/neural/src/models/mod.rs | 3 +- ml/neural/src/models/model.rs | 73 +------------- ml/neural/src/models/params.rs | 6 +- ml/neural/src/models/stack.rs | 8 +- ml/neural/src/neurons/mod.rs | 2 +- ml/neural/src/neurons/neuron.rs | 2 +- ml/neural/src/neurons/params.rs | 4 - ml/neural/src/nn/deep.rs | 31 ++++-- ml/neural/src/params/mod.rs | 15 +-- ml/neural/src/params/param.rs | 7 +- ml/optim/examples/descent.rs | 8 +- ml/optim/src/grad/descent.rs | 29 +++--- ml/optim/src/grad/gradient.rs | 83 +++------------- ml/optim/src/grad/mod.rs | 27 +++--- ml/optim/src/optimize/optimizer.rs | 6 +- ml/optim/src/specs.rs | 11 ++- ml/transformers/src/attention/head.rs | 5 +- ml/transformers/src/attention/mod.rs | 3 +- ml/transformers/src/ffn/network.rs | 2 +- 34 files changed, 194 insertions(+), 404 deletions(-) delete mode 100644 ml/neural/src/models/features.rs diff --git a/core/src/errors/mod.rs b/core/src/errors/mod.rs index 35d99c5c..267dace8 100644 --- a/core/src/errors/mod.rs +++ b/core/src/errors/mod.rs @@ -6,6 +6,4 @@ pub use self::{error::*, utils::*}; pub(crate) mod error; - - pub(crate) mod utils {} diff --git a/ml/ml/examples/nn.rs b/ml/ml/examples/nn.rs index 94813f87..f9d1e06a 100644 --- a/ml/ml/examples/nn.rs +++ b/ml/ml/examples/nn.rs @@ -1,6 +1,5 @@ extern crate concision_ml; - fn main() -> anyhow::Result<()> { println!("Welcome to concision!"); diff --git a/ml/ml/src/lib.rs b/ml/ml/src/lib.rs index 93479eac..5e3d349a 100644 --- a/ml/ml/src/lib.rs +++ b/ml/ml/src/lib.rs @@ -17,7 +17,7 @@ pub use concision_optim as optim; pub use concision_transformers as transformers; pub mod prelude { - + #[cfg(feature = "neural")] pub use concision_neural::prelude::*; #[cfg(feature = "nlp")] diff --git a/ml/neural/src/func/activate/activator.rs b/ml/neural/src/func/activate/activator.rs index abf8c40e..609589c2 100644 --- a/ml/neural/src/func/activate/activator.rs +++ b/ml/neural/src/func/activate/activator.rs @@ -2,35 +2,20 @@ Appellation: activator Contrib: FL03 */ -use super::{Activate, LinearActivation}; -use std::marker::PhantomData; +use super::Activate; -pub struct Activator -where - A: Activate, -{ - method: A, - _args: PhantomData, +pub struct Activator { + method: Box>, } -impl Activator -where - A: Activate, -{ - pub fn new(method: A) -> Self { - Activator { - method, - _args: PhantomData, - } +impl Activator { + pub fn new(method: Box>) -> Self { + Self { method } } } -impl Activate for Activator -where - A: Activate, -{ +impl Activate for Activator { fn activate(&self, x: T) -> T { self.method.activate(x) } } - diff --git a/ml/neural/src/func/activate/mod.rs b/ml/neural/src/func/activate/mod.rs index 1ec63aa3..20a4039e 100644 --- a/ml/neural/src/func/activate/mod.rs +++ b/ml/neural/src/func/activate/mod.rs @@ -13,20 +13,7 @@ pub(crate) mod nonlinear; pub type ActivationFn = fn(T) -> T; -#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct LinearActivation; - -impl LinearActivation { - pub fn method() -> ActivationFn { - |x| x - } -} - -impl Activate for LinearActivation { - fn activate(&self, x: T) -> T { - Self::method()(x) - } -} +pub type BoxedActivation = Box>; pub trait ActivationMethod { fn method_name(&self) -> &str; @@ -45,14 +32,35 @@ where } } +#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct LinearActivation; + +impl LinearActivation { + pub fn method() -> ActivationFn { + |x| x + } +} + +impl Activate for LinearActivation { + fn activate(&self, x: T) -> T { + Self::method()(x) + } +} + pub(crate) mod utils { use num::{One, Zero}; - - pub fn linear_activation(x: &T) -> T where T: Clone { + + pub fn linear_activation(x: &T) -> T + where + T: Clone, + { x.clone() } - pub fn heavyside(x: &T) -> T where T: Clone + One + PartialOrd + Zero { + pub fn heavyside(x: &T) -> T + where + T: Clone + One + PartialOrd + Zero, + { if x.clone() > T::zero() { T::one() } else { diff --git a/ml/neural/src/func/activate/nonlinear.rs b/ml/neural/src/func/activate/nonlinear.rs index 4c39d792..28ed3e9f 100644 --- a/ml/neural/src/func/activate/nonlinear.rs +++ b/ml/neural/src/func/activate/nonlinear.rs @@ -38,7 +38,10 @@ where pub struct ReLU; impl ReLU { - pub fn relu(args: T) -> T where T: PartialOrd + Zero { + pub fn relu(args: T) -> T + where + T: PartialOrd + Zero, + { if args > T::zero() { args } else { @@ -63,15 +66,25 @@ where pub struct Sigmoid; impl Sigmoid { - pub fn sigmoid(x: T) -> T where T: Float { + pub fn sigmoid(x: T) -> T + where + T: Float, + { (T::one() + (-x).exp()).powi(-2) } - pub fn derivative(x: T) -> T where T: Float { - - (T::one() + (-x).exp()).powi(-2) * (-x).exp() + pub fn derivative(x: T) -> T + where + T: Float, + { + -(T::one() + (-x).exp()).powi(-2) * (-x).exp() } - pub fn gradient(args: &Array) -> Array where D: Dimension, T: Float { + pub fn gradient(args: &Array) -> Array + where + D: Dimension, + T: Float, + { args.mapv(|x| Self::derivative(x)) } } @@ -86,8 +99,6 @@ where } } - - #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] @@ -104,7 +115,6 @@ impl Softmax { self.axis } - pub fn softmax(args: Array) -> Array where D: Dimension, @@ -114,7 +124,7 @@ impl Softmax { args.mapv(|x| x.exp() / denom) } - pub fn softmax_axis(&self, args: Array,) -> Array + pub fn softmax_axis(&self, args: Array) -> Array where T: NdFloat, D: Dimension + RemoveAxis, diff --git a/ml/neural/src/func/loss/mod.rs b/ml/neural/src/func/loss/mod.rs index a9c2cf48..1a7fb86f 100644 --- a/ml/neural/src/func/loss/mod.rs +++ b/ml/neural/src/func/loss/mod.rs @@ -19,7 +19,10 @@ use ndarray::prelude::{Array, Array1, Array2, Dimension, NdFloat}; use num::{Float, FromPrimitive}; use std::ops; -pub trait Loss where T: Float { +pub trait Loss +where + T: Float, +{ fn loss(&self, pred: &Array, target: &Array1) -> T; } diff --git a/ml/neural/src/func/loss/regress.rs b/ml/neural/src/func/loss/regress.rs index d9d05b87..65f8fb49 100644 --- a/ml/neural/src/func/loss/regress.rs +++ b/ml/neural/src/func/loss/regress.rs @@ -75,8 +75,6 @@ where pub struct MeanSquaredError; - - impl Loss for MeanSquaredError where T: NdFloat, diff --git a/ml/neural/src/func/mod.rs b/ml/neural/src/func/mod.rs index 9798361d..8105d97d 100644 --- a/ml/neural/src/func/mod.rs +++ b/ml/neural/src/func/mod.rs @@ -3,27 +3,24 @@ Contrib: FL03 */ //! # Functional -//! +//! //! This module implements several functional aspects of the neural network. -//! +//! //! ## Activation -//! +//! //! The activation functions are implemented as structs that implement the `Fn` trait. -//! +//! //! ## Loss -//! +//! //! The loss functions are implemented as structs that implement the `Fn` trait. pub use self::utils::*; pub mod activate; pub mod loss; - -pub(crate) mod utils{ -} +pub(crate) mod utils {} #[cfg(test)] mod tests { // use super::*; - } diff --git a/ml/neural/src/layers/kinds.rs b/ml/neural/src/layers/kinds.rs index e643e45d..00e312df 100644 --- a/ml/neural/src/layers/kinds.rs +++ b/ml/neural/src/layers/kinds.rs @@ -33,7 +33,9 @@ pub enum LayerType { Output, } -#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] pub struct Position { pub idx: usize, pub kind: LayerType, diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 0f585c10..434281fb 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -2,14 +2,13 @@ Appellation: model Contrib: FL03 */ -use super::{Features, LayerType, LayerParams, Position}; +use super::{Features, LayerParams, LayerType, Position}; use crate::prelude::{Activate, Forward, LinearActivation, Parameterized, Params}; use ndarray::prelude::{Array2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; - #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Layer where diff --git a/ml/neural/src/layers/linear/mod.rs b/ml/neural/src/layers/linear/mod.rs index 8a50af31..f1aed6cb 100644 --- a/ml/neural/src/layers/linear/mod.rs +++ b/ml/neural/src/layers/linear/mod.rs @@ -8,7 +8,6 @@ pub use self::{layer::*, regress::*, utils::*}; pub(crate) mod layer; pub(crate) mod regress; - pub(crate) mod utils { use ndarray::prelude::{Array1, Array2, NdFloat}; @@ -16,15 +15,17 @@ pub(crate) mod utils { data: &Array2, bias: &Array1, weights: &Array2, - ) -> Array2 where T: NdFloat { + ) -> Array2 + where + T: NdFloat, + { data.dot(&weights.t()) + bias } - pub fn linear_node( - data: &Array2, - bias: &T, - weights: &Array1, - ) -> Array1 where T: NdFloat { + pub fn linear_node(data: &Array2, bias: &T, weights: &Array1) -> Array1 + where + T: NdFloat, + { data.dot(&weights.t()) + bias.clone() } } diff --git a/ml/neural/src/layers/params.rs b/ml/neural/src/layers/params.rs index 533a1f2d..e99e46dc 100644 --- a/ml/neural/src/layers/params.rs +++ b/ml/neural/src/layers/params.rs @@ -3,8 +3,8 @@ Contrib: FL03 */ use super::Features; -use crate::prelude::Params; use crate::core::prelude::GenerateRandom; +use crate::prelude::Params; use ndarray::prelude::{Array1, Array2, Ix2}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; diff --git a/ml/neural/src/layers/sublayer.rs b/ml/neural/src/layers/sublayer.rs index 4563e8d0..dfac9517 100644 --- a/ml/neural/src/layers/sublayer.rs +++ b/ml/neural/src/layers/sublayer.rs @@ -12,7 +12,11 @@ use num::{Float, FromPrimitive}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct Sublayer where A: Activate>, T: Float { +pub struct Sublayer +where + A: Activate>, + T: Float, +{ layer: Layer, norm: LayerNorm, } @@ -33,7 +37,7 @@ where T: FromPrimitive + NdFloat, { type Output = Array2; - + fn forward(&self, data: &Array2) -> Self::Output { let norm = self.norm.forward(data); let layer = data + self.layer.forward(&norm); diff --git a/ml/neural/src/models/features.rs b/ml/neural/src/models/features.rs deleted file mode 100644 index 52c2a07c..00000000 --- a/ml/neural/src/models/features.rs +++ /dev/null @@ -1,115 +0,0 @@ -/* - Appellation: features - Contrib: FL03 -*/ -use ndarray::IntoDimension; -use serde::{Deserialize, Serialize}; - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct Features { - pub inputs: usize, - pub outputs: usize, -} - -impl Features { - pub fn new(inputs: usize, outputs: usize) -> Self { - Self { inputs, outputs } - } - - pub fn neuron(inputs: usize) -> Self { - Self::new(inputs, 1) - } - - pub fn uniform_scale(&self) -> T { - (T::one() / T::from(self.inputs).unwrap()).sqrt() - } - - pub fn inputs(&self) -> usize { - self.inputs - } - - pub fn outputs(&self) -> usize { - self.outputs - } - - pub fn set_inputs(&mut self, inputs: usize) { - self.inputs = inputs; - } - - pub fn set_outputs(&mut self, outputs: usize) { - self.outputs = outputs; - } - - pub fn with_inputs(mut self, inputs: usize) -> Self { - self.inputs = inputs; - self - } - - pub fn with_outputs(mut self, outputs: usize) -> Self { - self.outputs = outputs; - self - } - - pub fn in_by_out(&self) -> (usize, usize) { - (self.inputs, self.outputs) - } - - pub fn out_by_in(&self) -> (usize, usize) { - (self.outputs, self.inputs) - } -} - -impl std::fmt::Display for Features { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "({}, {})", self.inputs, self.outputs) - } -} - -impl IntoDimension for Features { - type Dim = ndarray::Ix2; - - fn into_dimension(self) -> Self::Dim { - ndarray::Ix2(self.outputs, self.inputs) - } -} - -impl From for ndarray::Ix2 { - fn from(features: Features) -> Self { - ndarray::Ix2(features.inputs, features.outputs) - } -} - -impl From for ndarray::IxDyn { - fn from(features: Features) -> Self { - ndarray::IxDyn(&[features.inputs, features.outputs]) - } -} - -impl From for [usize; 2] { - fn from(features: Features) -> Self { - [features.inputs, features.outputs] - } -} - -impl From<[usize; 2]> for Features { - fn from(features: [usize; 2]) -> Self { - Self { - inputs: features[0], - outputs: features[1], - } - } -} - -impl From for (usize, usize) { - fn from(features: Features) -> Self { - (features.inputs, features.outputs) - } -} - -impl From<(usize, usize)> for Features { - fn from((inputs, outputs): (usize, usize)) -> Self { - Self { inputs, outputs } - } -} diff --git a/ml/neural/src/models/mod.rs b/ml/neural/src/models/mod.rs index 3d1981cb..5993c3f7 100644 --- a/ml/neural/src/models/mod.rs +++ b/ml/neural/src/models/mod.rs @@ -4,9 +4,8 @@ */ //! # Model //! -pub use self::{features::*, model::*, params::*, utils::*}; +pub use self::{model::*, params::*, utils::*}; -pub(crate) mod features; pub(crate) mod model; pub(crate) mod params; diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs index 2ce91ed2..b90a43b8 100644 --- a/ml/neural/src/models/model.rs +++ b/ml/neural/src/models/model.rs @@ -2,78 +2,11 @@ Appellation: model Contrib: FL03 */ -use super::{Features, ModelParams,}; -use crate::prelude::{Biased, Parameterized, Weighted}; -use ndarray::prelude::{Array2, NdFloat}; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use num::Float; +use super::ModelParams; +use crate::prelude::Features; pub struct Model { pub features: Features, children: Vec>, - params: ModelParams -} - -impl Model -where - T: Float, -{ - pub fn new(features: Features) -> Self { - Self { - features, - children: Vec::new(), - params: ModelParams::new(features), - } - } - - pub fn features(&self) -> &Features { - &self.features - } - - pub fn features_mut(&mut self) -> &mut Features { - &mut self.features - } -} - -impl Model -where - T: NdFloat, -{ - pub fn linear(&self, args: &Array2) -> Array2 { - args.dot(&self.weights().t()) + self.bias() - } -} - -impl Model -where - T: Float + SampleUniform, -{ - pub fn init(mut self, biased: bool) -> Self { - self.params = self.params.init(biased); - self - } -} - -impl Parameterized for Model -where - T: Float, -{ - type Features = Features; - type Params = ModelParams; - - fn features(&self) -> &Features { - &self.features - } - - fn features_mut(&mut self) -> &mut Features { - &mut self.features - } - - fn params(&self) -> &ModelParams { - &self.params - } - - fn params_mut(&mut self) -> &mut ModelParams { - &mut self.params - } + params: ModelParams, } diff --git a/ml/neural/src/models/params.rs b/ml/neural/src/models/params.rs index 5521eb68..cc4b4caf 100644 --- a/ml/neural/src/models/params.rs +++ b/ml/neural/src/models/params.rs @@ -2,9 +2,8 @@ Appellation: model Contrib: FL03 */ -use super::Features; use crate::core::prelude::GenerateRandom; -use crate::prelude::Params; +use crate::prelude::{Features, Params}; use ndarray::prelude::{Array1, Array2, Ix2}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; @@ -77,7 +76,6 @@ where } } - impl Params for ModelParams where T: Float, @@ -105,4 +103,4 @@ where fn set_weights(&mut self, weights: Array2) { self.weights = weights; } -} \ No newline at end of file +} diff --git a/ml/neural/src/models/stack.rs b/ml/neural/src/models/stack.rs index 3bb37b5b..1953e4d5 100644 --- a/ml/neural/src/models/stack.rs +++ b/ml/neural/src/models/stack.rs @@ -3,12 +3,11 @@ Contrib: FL03 */ use crate::layers::Layer; -use crate::prelude::{Activate, LinearActivation,}; +use crate::prelude::{Activate, LinearActivation}; use ndarray::prelude::Array2; use num::Float; use serde::{Deserialize, Serialize}; - #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Stack where @@ -24,9 +23,6 @@ where T: Float, { pub fn new() -> Self { - Self { - layers: Vec::new(), - } + Self { layers: Vec::new() } } } - diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index f0a825ea..b088b31e 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -35,7 +35,7 @@ pub(crate) mod utils {} #[cfg(test)] mod tests { use super::*; - use crate::func::activate::{Activate, Softmax, softmax}; + use crate::func::activate::{softmax, Activate, Softmax}; use crate::prelude::Forward; // use lazy_static::lazy_static; use ndarray::{array, Array1}; diff --git a/ml/neural/src/neurons/neuron.rs b/ml/neural/src/neurons/neuron.rs index 4becc79b..cc05223f 100644 --- a/ml/neural/src/neurons/neuron.rs +++ b/ml/neural/src/neurons/neuron.rs @@ -2,8 +2,8 @@ Appellation: neuron Contrib: FL03 */ -use crate::func::activate::{Activate, LinearActivation}; use crate::core::GenerateRandom; +use crate::func::activate::{Activate, LinearActivation}; use crate::prelude::Forward; use ndarray::prelude::{Array1, Array2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; diff --git a/ml/neural/src/neurons/params.rs b/ml/neural/src/neurons/params.rs index 403428fe..1cbd10f5 100644 --- a/ml/neural/src/neurons/params.rs +++ b/ml/neural/src/neurons/params.rs @@ -43,7 +43,6 @@ where self.features = features; } - pub fn with_bias(mut self, bias: Array0) -> Self { self.bias = bias.into(); self @@ -108,7 +107,6 @@ impl Params for NeuronParams where T: Float, { - fn bias(&self) -> &Array0 { &self.bias } @@ -131,7 +129,5 @@ where fn set_weights(&mut self, weights: Array1) { self.weights = weights; - } } - diff --git a/ml/neural/src/nn/deep.rs b/ml/neural/src/nn/deep.rs index 8eced8a0..64910169 100644 --- a/ml/neural/src/nn/deep.rs +++ b/ml/neural/src/nn/deep.rs @@ -8,13 +8,25 @@ use crate::prelude::{Forward, Layer, Parameterized}; use ndarray::prelude::{Array2, NdFloat}; use num::Float; -pub struct DeepNetwork where T: Float, I: Activate>, H: Activate>, O: Activate> { +pub struct DeepNetwork +where + T: Float, + I: Activate>, + H: Activate>, + O: Activate>, +{ pub input: Layer, pub hidden: Vec>, pub output: Layer, } -impl DeepNetwork where T: Float, I: Activate>, H: Activate>, O: Activate> { +impl DeepNetwork +where + T: Float, + I: Activate>, + H: Activate>, + O: Activate>, +{ pub fn new(input: Layer, hidden: Vec>, output: Layer) -> Self { Self { input, @@ -26,10 +38,9 @@ impl DeepNetwork where T: Float, I: Activate>, pub fn validate_dims(&self) -> bool { let mut dim = true; for (i, layer) in self.hidden.iter().enumerate() { - if i== 0 { + if i == 0 { dim = self.input.features().outputs() == self.hidden[i].features().inputs() - } - else if i == self.hidden.len() - 1 { + } else if i == self.hidden.len() - 1 { dim = dim && layer.features().outputs() == self.output.features().inputs(); } else { dim = dim && layer.features().outputs() == self.hidden[i + 1].features().inputs(); @@ -39,7 +50,13 @@ impl DeepNetwork where T: Float, I: Activate>, } } -impl Forward> for DeepNetwork where T: NdFloat, I: Activate>, H: Activate>, O: Activate> { +impl Forward> for DeepNetwork +where + T: NdFloat, + I: Activate>, + H: Activate>, + O: Activate>, +{ type Output = Array2; fn forward(&self, args: &Array2) -> Self::Output { @@ -50,5 +67,3 @@ impl Forward> for DeepNetwork where T: NdFloat self.output.forward(&out) } } - - diff --git a/ml/neural/src/params/mod.rs b/ml/neural/src/params/mod.rs index d458d655..b3bafca8 100644 --- a/ml/neural/src/params/mod.rs +++ b/ml/neural/src/params/mod.rs @@ -13,18 +13,16 @@ pub(crate) mod param; pub(crate) mod shapes; pub(crate) mod weight; -use ndarray::IntoDimension; use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Dimension, Ix2,}; +use ndarray::prelude::{Array, Dimension, Ix2}; +use ndarray::IntoDimension; use num::Float; - - pub trait Biased where D: Dimension, T: Float, - Self: Weighted + Self: Weighted, { /// Returns an owned reference to the bias of the layer. fn bias(&self) -> &Array; @@ -32,8 +30,6 @@ where fn bias_mut(&mut self) -> &mut Array; } - - pub trait Weighted where D: Dimension, @@ -51,7 +47,6 @@ where D: Dimension, T: Float, { - } pub trait Params @@ -59,7 +54,6 @@ where D: Dimension, T: Float, { - /// Returns an owned reference to the bias of the layer. fn bias(&self) -> &Array; /// Returns a mutable reference to the bias of the layer. @@ -74,7 +68,6 @@ where fn set_weights(&mut self, weights: Array); } - pub trait Parameterized where D: Dimension, @@ -106,7 +99,7 @@ where P: Parameterized, T: Float, ::Smaller: Dimension, -

>::Params: 'static, +

>::Params: 'static, { fn bias(&self) -> &Array { self.params().bias() diff --git a/ml/neural/src/params/param.rs b/ml/neural/src/params/param.rs index c2605604..209615fe 100644 --- a/ml/neural/src/params/param.rs +++ b/ml/neural/src/params/param.rs @@ -2,8 +2,8 @@ Appellation: model Contrib: FL03 */ -use crate::layers::Features; use crate::core::prelude::GenerateRandom; +use crate::layers::Features; use ndarray::prelude::{Array1, Array2}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; @@ -24,7 +24,10 @@ pub trait ParamFeatures { } #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub struct ParamGroup where T: Float { +pub struct ParamGroup +where + T: Float, +{ bias: Array1, pub features: Features, weights: Array2, diff --git a/ml/optim/examples/descent.rs b/ml/optim/examples/descent.rs index f53c4f8d..0ef46df6 100644 --- a/ml/optim/examples/descent.rs +++ b/ml/optim/examples/descent.rs @@ -11,11 +11,11 @@ fn main() -> anyhow::Result<()> { let _n = samples * inputs; - let (epochs, gamma) = (500, 0.001); + let (epochs, gamma) = (100, 0.05); - basic_descent(epochs, features, gamma)?; + // basic_descent(epochs, features, gamma)?; - // sample_descent(epochs, features, gamma)?; + sample_descent(epochs, features, gamma)?; Ok(()) } @@ -52,7 +52,7 @@ pub fn sample_descent(epochs: usize, features: Features, gamma: f64) -> anyhow:: let mut grad = GradientDescent::new(gamma, model); let mut losses = Array1::zeros(epochs); for e in 0..epochs { - let cost = grad.step(&x, &y)?; + let cost = grad.gradient(&x, &y, Sigmoid::gradient)?; losses[e] = cost; } println!("Losses:\n\n{:?}\n", &losses); diff --git a/ml/optim/src/grad/descent.rs b/ml/optim/src/grad/descent.rs index 47b2a60c..b789a020 100644 --- a/ml/optim/src/grad/descent.rs +++ b/ml/optim/src/grad/descent.rs @@ -3,9 +3,9 @@ Contrib: FL03 */ use crate::neural::layers::linear::Linear; -use crate::neural::prelude::Forward; +use crate::neural::prelude::{Forward, Layer}; use crate::prelude::Norm; -use ndarray::prelude::{Array1, Array2,}; +use ndarray::prelude::{Array1, Array2}; use ndarray_stats::DeviationExt; #[derive(Clone)] @@ -53,18 +53,23 @@ impl GradientDescent { self } - pub fn gradient(&mut self, data: &Array2, targets: &Array1) -> anyhow::Result { - // let pred = self.model.forward(data); - let gradient = |p: &Array1| { - let error = targets - &data.dot(&(p / p.l2())); - let scale = -1.0 / (2.0 * data.len() as f64); - let grad = scale * error.dot(data); + pub fn gradient( + &mut self, + data: &Array2, + targets: &Array1, + grad: impl Fn(&Array1) -> Array1, + ) -> anyhow::Result { + let lr = self.gamma(); + let (samples, _inputs) = data.dim(); + let pred = self.model.forward(data); - &grad / grad.l2() - }; - self.model.apply_gradient(self.gamma, &gradient); + let errors = targets - &pred; + let dz = errors * grad(&pred); + let dw = data.t().dot(&dz) / (2.0 * samples as f64); - let loss = targets.mean_sq_err(&self.model.forward(data))?; + self.model_mut().update_with_gradient(lr, &dw); + + let loss = targets.mean_sq_err(&self.model().forward(data))?; Ok(loss) } diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs index 27818c44..f51d4495 100644 --- a/ml/optim/src/grad/gradient.rs +++ b/ml/optim/src/grad/gradient.rs @@ -2,7 +2,7 @@ Appellation: grad Contrib: FL03 */ -use crate::neural::models::ModelParams; +use crate::neural::prelude::Params; use ndarray::prelude::{Array1, Array2, NdFloat}; use num::Float; @@ -11,11 +11,14 @@ where T: Float, { gamma: T, - params: Vec>, - objective: fn(&Array2) -> Array2, + params: Vec>, + objective: fn(&T) -> T, } -impl Grad where T: Float { +impl Grad +where + T: Float, +{ pub fn gamma(&self) -> T { self.gamma } @@ -24,85 +27,29 @@ impl Grad where T: Float { &mut self.gamma } - pub fn objective(&self) -> fn(&Array2) -> Array2 { + pub fn objective(&self) -> fn(&T) -> T { self.objective } - pub fn params(&self) -> &Vec> { + pub fn params(&self) -> &Vec> { &self.params } - pub fn params_mut(&mut self) -> &mut Vec> { + pub fn params_mut(&mut self) -> &mut Vec> { &mut self.params } - - } -impl Grad where T: NdFloat { - - pub fn step(&mut self, x: &Array2, y: &Array2) -> anyhow::Result { - let mut cost = T::zero(); - Ok(cost) - } - - -} - -pub struct GradStep +impl Grad where - T: Float, + T: NdFloat, { - gamma: T, - - params: ModelParams, -} - -impl GradStep -where - T: Float, -{ - pub fn new(gamma: T, params: ModelParams) -> Self { - Self { gamma, params } - } - - pub fn gamma(&self) -> T { - self.gamma - } - - pub fn gamma_mut(&mut self) -> &mut T { - &mut self.gamma - } - - pub fn params(&self) -> &ModelParams { - &self.params - } - - pub fn params_mut(&mut self) -> &mut ModelParams { - &mut self.params - } - - pub fn set_gamma(&mut self, gamma: T) { - self.gamma = gamma; - } - - pub fn set_params(&mut self, params: ModelParams) { - self.params = params; - } - - pub fn with_gamma(mut self, gamma: T) -> Self { - self.gamma = gamma; - self - } - - pub fn with_params(mut self, params: ModelParams) -> Self { - self.params = params; - self + pub fn step(&mut self, x: &Array2, y: &Array1) -> anyhow::Result { + let mut cost = T::zero(); + Ok(cost) } } -impl GradStep where T: Float {} - #[cfg(test)] mod tests { diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index fec658c0..4fc94c4a 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -26,6 +26,7 @@ pub struct DescentParams { } pub(crate) mod utils { + // use crate::neural::prelude::{Activate, Layer,}; use ndarray::prelude::{Array, Array1, Dimension, NdFloat}; use num::FromPrimitive; @@ -34,7 +35,11 @@ pub(crate) mod utils { epochs: usize, gamma: T, partial: impl Fn(&Array) -> Array, - ) -> Array1 where D: Dimension, T: FromPrimitive + NdFloat { + ) -> Array1 + where + D: Dimension, + T: FromPrimitive + NdFloat, + { let mut losses = Array1::zeros(epochs); for e in 0..epochs { let grad = partial(params); @@ -44,15 +49,15 @@ pub(crate) mod utils { losses } - pub fn gradient_descent_step( - params: &mut Array, - gamma: T, - partial: impl Fn(&Array) -> Array, - ) -> T where D: Dimension, T: FromPrimitive + NdFloat { - let grad = partial(params); - params.scaled_add(-gamma, &grad); - params.mean().unwrap_or_else(T::zero) - } + // pub fn gradient_descent_step( + // args: &Array2, + // layer: &mut Layer, + // gamma: T, + // partial: impl Fn(&Array2) -> Array2, + // ) -> T where A: Activate>, T: FromPrimitive + NdFloat { + // let grad = partial(args); + // layer.weights_mut().scaled_add(-gamma, &grad); + // } } #[cfg(test)] @@ -67,7 +72,7 @@ mod tests { } #[test] - fn descent() { + fn descent() { let (_samples, inputs) = (20, 5); let outputs = 1; diff --git a/ml/optim/src/optimize/optimizer.rs b/ml/optim/src/optimize/optimizer.rs index 311b3e85..238d1884 100644 --- a/ml/optim/src/optimize/optimizer.rs +++ b/ml/optim/src/optimize/optimizer.rs @@ -2,5 +2,9 @@ Appellation: optimizer Contrib: FL03 */ +use crate::neural::prelude::Params; +use ndarray::prelude::Dimension; -pub struct Optimizer {} +pub struct Optimizer { + params: Vec>, +} diff --git a/ml/optim/src/specs.rs b/ml/optim/src/specs.rs index ed8c9113..05640094 100644 --- a/ml/optim/src/specs.rs +++ b/ml/optim/src/specs.rs @@ -5,10 +5,16 @@ use ndarray::prelude::{Array, Array1, Array2, Dimension}; use num::Float; -pub trait Gradient where T: Float { +pub trait Gradient +where + T: Float, +{ fn partial(&self, x: T) -> T; - fn gradient(&self, args: &Array) -> Array where D: Dimension { + fn gradient(&self, args: &Array) -> Array + where + D: Dimension, + { args.mapv(|xs| self.partial(xs)) } } @@ -29,7 +35,6 @@ pub trait Minimize { fn minimize(&self, scale: T) -> Self; } - pub trait LearningRate where T: Float, diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index c579da0f..9acec789 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -59,7 +59,10 @@ where } } -impl AttentionHead where T: NdFloat { +impl AttentionHead +where + T: NdFloat, +{ pub fn attention(&mut self, data: &Array2) -> Array2 { // multiply the data by the wieghted query, key, and value matrices, respectively let weighted = data * self.weights(); diff --git a/ml/transformers/src/attention/mod.rs b/ml/transformers/src/attention/mod.rs index da05cc6c..472915ae 100644 --- a/ml/transformers/src/attention/mod.rs +++ b/ml/transformers/src/attention/mod.rs @@ -29,8 +29,7 @@ pub type InputArray = Array; pub type AttentionArray = Array; pub trait Attention { - fn attention(&self, data: &Array2) -> BoxResult> - { + fn attention(&self, data: &Array2) -> BoxResult> { // let (seq, model) = data.dim(); let q = self.query().dot(data); diff --git a/ml/transformers/src/ffn/network.rs b/ml/transformers/src/ffn/network.rs index aeccec92..ed40aa62 100644 --- a/ml/transformers/src/ffn/network.rs +++ b/ml/transformers/src/ffn/network.rs @@ -3,8 +3,8 @@ Contrib: FL03 */ use super::FFNParams; -use crate::neural::layers::linear::LinearLayer; use crate::neural::func::activate::{Activate, ReLU}; +use crate::neural::layers::linear::LinearLayer; use crate::neural::prelude::Forward; use ndarray::prelude::Array2; use serde::{Deserialize, Serialize}; From 3e8091acf563a5a1a504631fe032ff01fcc0f514 Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 17 Nov 2023 14:37:36 -0600 Subject: [PATCH 055/118] update Signed-off-by: FL03 --- ml/optim/examples/descent.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ml/optim/examples/descent.rs b/ml/optim/examples/descent.rs index 0ef46df6..69ff0297 100644 --- a/ml/optim/examples/descent.rs +++ b/ml/optim/examples/descent.rs @@ -11,7 +11,7 @@ fn main() -> anyhow::Result<()> { let _n = samples * inputs; - let (epochs, gamma) = (100, 0.05); + let (epochs, gamma) = (10000, 0.05); // basic_descent(epochs, features, gamma)?; From ae7b979bb7546f8a85bc820610ad8495a31cf7fd Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 18 Nov 2023 09:13:30 -0600 Subject: [PATCH 056/118] Refactor neural network structure and optimize training process Signed-off-by: FL03 --- data/src/datasets/dataset.rs | 6 +- ml/neural/src/func/activate/activator.rs | 6 +- ml/neural/src/func/activate/linear.rs | 27 ++++ ml/neural/src/func/activate/mod.rs | 23 +-- ml/neural/src/func/activate/nonlinear.rs | 4 +- ml/neural/src/layers/features.rs | 8 + ml/neural/src/layers/layer.rs | 26 ++++ .../layers/{linear/regress.rs => linear.rs} | 0 ml/neural/src/layers/linear/layer.rs | 147 ------------------ ml/neural/src/layers/linear/mod.rs | 48 ------ ml/neural/src/models/model.rs | 13 +- ml/neural/src/params/mod.rs | 2 + ml/optim/examples/descent.rs | 37 ++++- ml/optim/examples/sgd.rs | 13 +- ml/optim/src/grad/descent.rs | 46 +++++- ml/optim/src/grad/sgd.rs | 65 ++++---- ml/optim/src/specs.rs | 2 +- .../src/attention/multi/attention.rs | 9 +- ml/transformers/src/ffn/network.rs | 16 +- 19 files changed, 215 insertions(+), 283 deletions(-) create mode 100644 ml/neural/src/func/activate/linear.rs rename ml/neural/src/layers/{linear/regress.rs => linear.rs} (100%) delete mode 100644 ml/neural/src/layers/linear/layer.rs delete mode 100644 ml/neural/src/layers/linear/mod.rs diff --git a/data/src/datasets/dataset.rs b/data/src/datasets/dataset.rs index acff96d8..826997ca 100644 --- a/data/src/datasets/dataset.rs +++ b/data/src/datasets/dataset.rs @@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)] #[serde(rename_all = "lowercase")] -pub struct Dataset +pub struct DataSet where D: Dimension, { @@ -17,7 +17,7 @@ where targets: Array, } -impl Dataset +impl DataSet where D: Dimension, T: Float, @@ -27,7 +27,7 @@ where } } -impl std::fmt::Display for Dataset +impl std::fmt::Display for DataSet where D: Dimension + Serialize, T: Serialize, diff --git a/ml/neural/src/func/activate/activator.rs b/ml/neural/src/func/activate/activator.rs index 609589c2..c53dc41e 100644 --- a/ml/neural/src/func/activate/activator.rs +++ b/ml/neural/src/func/activate/activator.rs @@ -12,10 +12,14 @@ impl Activator { pub fn new(method: Box>) -> Self { Self { method } } + + pub fn method(&self) -> &dyn Activate { + self.method.as_ref() + } } impl Activate for Activator { fn activate(&self, x: T) -> T { - self.method.activate(x) + self.method().activate(x) } } diff --git a/ml/neural/src/func/activate/linear.rs b/ml/neural/src/func/activate/linear.rs new file mode 100644 index 00000000..64fbd939 --- /dev/null +++ b/ml/neural/src/func/activate/linear.rs @@ -0,0 +1,27 @@ +/* + Appellation: linear + Contrib: FL03 +*/ +use super::{Activate, ActivationFn}; +use serde::{Deserialize, Serialize}; + +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] +pub struct LinearActivation; + +impl LinearActivation { + pub fn method() -> ActivationFn { + |x| x + } + + pub fn rho(args: T) -> T { + args + } +} + +impl Activate for LinearActivation { + fn activate(&self, x: T) -> T { + Self::method()(x) + } +} diff --git a/ml/neural/src/func/activate/mod.rs b/ml/neural/src/func/activate/mod.rs index 20a4039e..6a00a7e2 100644 --- a/ml/neural/src/func/activate/mod.rs +++ b/ml/neural/src/func/activate/mod.rs @@ -5,10 +5,11 @@ //! # activate //! //! This module contains the activation functions for the neurons. -pub use self::{activator::*, binary::*, nonlinear::*, utils::*}; +pub use self::{activator::*, binary::*, linear::*, nonlinear::*, utils::*}; pub(crate) mod activator; pub(crate) mod binary; +pub(crate) mod linear; pub(crate) mod nonlinear; pub type ActivationFn = fn(T) -> T; @@ -32,21 +33,6 @@ where } } -#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct LinearActivation; - -impl LinearActivation { - pub fn method() -> ActivationFn { - |x| x - } -} - -impl Activate for LinearActivation { - fn activate(&self, x: T) -> T { - Self::method()(x) - } -} - pub(crate) mod utils { use num::{One, Zero}; @@ -68,3 +54,8 @@ pub(crate) mod utils { } } } + +#[cfg(test)] +mod tests { + // use super::*; +} diff --git a/ml/neural/src/func/activate/nonlinear.rs b/ml/neural/src/func/activate/nonlinear.rs index 28ed3e9f..bfa5e277 100644 --- a/ml/neural/src/func/activate/nonlinear.rs +++ b/ml/neural/src/func/activate/nonlinear.rs @@ -70,14 +70,14 @@ impl Sigmoid { where T: Float, { - (T::one() + (-x).exp()).powi(-2) + T::one() / (T::one() + (-x).exp()) } pub fn derivative(x: T) -> T where T: Float, { - -(T::one() + (-x).exp()).powi(-2) * (-x).exp() + (-x).exp() / (T::one() + (-x).exp()).powi(2) } pub fn gradient(args: &Array) -> Array diff --git a/ml/neural/src/layers/features.rs b/ml/neural/src/layers/features.rs index a915e7ae..ffcf0b2c 100644 --- a/ml/neural/src/layers/features.rs +++ b/ml/neural/src/layers/features.rs @@ -5,6 +5,14 @@ use ndarray::IntoDimension; use serde::{Deserialize, Serialize}; +pub trait FromFeatures { + fn from_features(features: Features) -> Self; +} + +pub trait IntoFeatures { + fn into_features(self) -> Features; +} + #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 434281fb..189f48fd 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -34,6 +34,18 @@ where position, } } + + pub fn new_input(features: Features) -> Self { + Self::new(features, Position::input()) + } + + pub fn hidden(features: Features, index: usize) -> Self { + Self::new(features, Position::hidden(index)) + } + + pub fn output(features: Features, index: usize) -> Self { + Self::new(features, Position::output(index)) + } } impl Layer @@ -52,6 +64,20 @@ where pub fn position(&self) -> &Position { &self.position } + + pub fn set_position(&mut self, position: Position) { + self.position = position; + } +} + +impl Layer +where + A: Activate>, + T: Float + 'static, +{ + pub fn update_with_gradient(&mut self, gamma: T, grad: &Array2) { + self.params.weights_mut().scaled_add(-gamma, grad); + } } impl Layer diff --git a/ml/neural/src/layers/linear/regress.rs b/ml/neural/src/layers/linear.rs similarity index 100% rename from ml/neural/src/layers/linear/regress.rs rename to ml/neural/src/layers/linear.rs diff --git a/ml/neural/src/layers/linear/layer.rs b/ml/neural/src/layers/linear/layer.rs deleted file mode 100644 index 5fca892b..00000000 --- a/ml/neural/src/layers/linear/layer.rs +++ /dev/null @@ -1,147 +0,0 @@ -/* - Appellation: layer - Contrib: FL03 -*/ -use crate::core::prelude::GenerateRandom; -use crate::prelude::{Features, Forward}; - -use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Array1, Array2, NdFloat}; -use ndarray::Dimension; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use num::Float; -use serde::{Deserialize, Serialize}; -use std::ops::Add; - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -pub struct LinearLayer { - bias: Array1, - pub features: Features, - weights: Array2, -} - -impl LinearLayer -where - T: Float, -{ - pub fn new(inputs: usize, outputs: usize) -> Self { - Self { - bias: Array1::zeros(outputs), - features: Features::new(inputs, outputs), - weights: Array2::zeros((inputs, outputs)), - } - } - pub fn from_features(features: Features) -> Self { - Self { - bias: Array1::zeros(features.outputs()), - features, - weights: Array2::zeros(features.out_by_in()), - } - } - - pub fn bias(&self) -> &Array1 { - &self.bias - } - - pub fn bias_mut(&mut self) -> &mut Array1 { - &mut self.bias - } - - pub fn features(&self) -> &Features { - &self.features - } - - pub fn features_mut(&mut self) -> &mut Features { - &mut self.features - } - - pub fn weights(&self) -> &Array2 { - &self.weights - } - - pub fn weights_mut(&mut self) -> &mut Array2 { - &mut self.weights - } - - pub fn set_bias(&mut self, bias: Array1) { - self.bias = bias; - } - - pub fn set_features(&mut self, features: Features) { - self.features = features; - } - - pub fn set_weights(&mut self, weights: Array2) { - self.weights = weights; - } - - pub fn with_params(mut self, params: Features) -> Self { - self.features = params; - self - } -} - -impl LinearLayer -where - T: Float + SampleUniform, -{ - pub fn init_bias(mut self) -> Self { - let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); - self.bias = ndarray::Array1::uniform_between(dk, self.features().outputs()); - self - } - - pub fn init_weight(mut self) -> Self { - let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); - self.weights = Array2::uniform_between(dk, self.features().out_by_in()); - self - } -} - -impl LinearLayer -where - T: NdFloat, -{ - pub fn fit(&mut self, data: &Array2) -> Array2 - where - T: 'static, - { - self.forward(data) - } - - pub fn update_with_gradient(&mut self, gradient: &Array2, lr: T) { - self.weights = &self.weights - gradient * lr; - } - - pub fn apply_gradient(&mut self, gradient: &Array1, gamma: T) { - for (ws, g) in self.weights_mut().iter_mut().zip(gradient.iter()) { - *ws -= *g * gamma; - } - } -} - -impl Forward> for LinearLayer -where - D: Dimension, - S: Dimension, - T: NdFloat, - Array: Add, Output = Array> + Dot, Output = Array>, - Array: Add, Output = Array>, -{ - type Output = Array; - - fn forward(&self, data: &Array) -> Self::Output { - data.dot(&self.weights().t().to_owned()) + self.bias().clone() - } -} - -impl Forward for LinearLayer -where - T: NdFloat, -{ - type Output = Array2; - - fn forward(&self, data: &T) -> Self::Output { - &self.weights().t().to_owned() * data.clone() + self.bias().clone() - } -} diff --git a/ml/neural/src/layers/linear/mod.rs b/ml/neural/src/layers/linear/mod.rs deleted file mode 100644 index f1aed6cb..00000000 --- a/ml/neural/src/layers/linear/mod.rs +++ /dev/null @@ -1,48 +0,0 @@ -/* - Appellation: linear - Contrib: FL03 -*/ -//! # Linear Layer -pub use self::{layer::*, regress::*, utils::*}; - -pub(crate) mod layer; -pub(crate) mod regress; - -pub(crate) mod utils { - use ndarray::prelude::{Array1, Array2, NdFloat}; - - pub fn linear_transformation( - data: &Array2, - bias: &Array1, - weights: &Array2, - ) -> Array2 - where - T: NdFloat, - { - data.dot(&weights.t()) + bias - } - - pub fn linear_node(data: &Array2, bias: &T, weights: &Array1) -> Array1 - where - T: NdFloat, - { - data.dot(&weights.t()) + bias.clone() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::prelude::{Features, Forward}; - use ndarray::prelude::Array2; - - #[test] - fn test_linear_layer() { - let (samples, inputs, outputs) = (20, 2, 2); - let features = Features::new(inputs, outputs); - let data = Array2::::ones((20, features.inputs())); - let layer = LinearLayer::from_features(features).init_weight(); - let pred = layer.forward(&data); - assert_eq!(pred.dim(), (samples, outputs)); - } -} diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs index b90a43b8..3ba37326 100644 --- a/ml/neural/src/models/model.rs +++ b/ml/neural/src/models/model.rs @@ -3,7 +3,18 @@ Contrib: FL03 */ use super::ModelParams; -use crate::prelude::Features; +use crate::prelude::{Activate, Features, Params}; +use ndarray::prelude::{Array, Ix2}; +use num::Float; + +pub struct BaseModel +where + T: Float, +{ + pub features: Features, + activator: Box>, + params: Box>, +} pub struct Model { pub features: Features, diff --git a/ml/neural/src/params/mod.rs b/ml/neural/src/params/mod.rs index b3bafca8..c627e5e6 100644 --- a/ml/neural/src/params/mod.rs +++ b/ml/neural/src/params/mod.rs @@ -18,6 +18,8 @@ use ndarray::prelude::{Array, Dimension, Ix2}; use ndarray::IntoDimension; use num::Float; +pub type BoxedParams = Box>; + pub trait Biased where D: Dimension, diff --git a/ml/optim/examples/descent.rs b/ml/optim/examples/descent.rs index 69ff0297..cdff86e3 100644 --- a/ml/optim/examples/descent.rs +++ b/ml/optim/examples/descent.rs @@ -1,21 +1,24 @@ -use concision_neural::prelude::{Features, Linear, Sigmoid}; +use concision_core::prelude::linarr; +use concision_neural::prelude::{Features, Layer, Linear, LinearActivation, Sigmoid}; use concision_neural::prop::Forward; -use concision_optim::prelude::{gradient_descent, GradientDescent}; +use concision_optim::prelude::{gradient, gradient_descent, GradientDescent}; use ndarray::prelude::{Array, Array1}; fn main() -> anyhow::Result<()> { let (samples, inputs) = (20, 5); - let outputs = 1; + let outputs = 3; let features = Features::new(inputs, outputs); let _n = samples * inputs; - let (epochs, gamma) = (10000, 0.05); + let (epochs, gamma) = (100000, 0.05); // basic_descent(epochs, features, gamma)?; - sample_descent(epochs, features, gamma)?; + // sample_descent(epochs, features, gamma)?; + + sample_gradient(epochs, features, gamma)?; Ok(()) } @@ -60,3 +63,27 @@ pub fn sample_descent(epochs: usize, features: Features, gamma: f64) -> anyhow:: println!("Trained:\n\n{:?}", grad.model().forward(&x)); Ok(()) } + +pub fn sample_gradient(epochs: usize, features: Features, gamma: f64) -> anyhow::Result<()> { + let (samples, inputs) = (20, features.inputs()); + + // Generate some example data + let x = linarr((samples, inputs))?; + let y = linarr((samples, features.outputs()))?; + + let mut model = Layer::::new_input(features).init(false); + println!( + "Targets:\n\n{:?}\nPredictions:\n\n{:?}\n", + &y, + model.forward(&x) + ); + + let mut losses = Array1::zeros(epochs); + for e in 0..epochs { + let cost = gradient(gamma, &mut model, &x, &y, Sigmoid::gradient); + losses[e] = cost; + } + println!("Losses:\n\n{:?}\n", &losses); + println!("Trained:\n\n{:?}", model.forward(&x)); + Ok(()) +} diff --git a/ml/optim/examples/sgd.rs b/ml/optim/examples/sgd.rs index 06d78c21..43a52731 100644 --- a/ml/optim/examples/sgd.rs +++ b/ml/optim/examples/sgd.rs @@ -1,11 +1,13 @@ -use concision_neural::layers::linear::LinearLayer; -use concision_optim::grad::sgd::StochasticGradientDescent; +use concision_neural::prelude::{Features, Layer, Sigmoid}; +use concision_optim::grad::sgd::sgd; use ndarray::prelude::Array; fn main() -> anyhow::Result<()> { let (samples, inputs) = (20, 10); let outputs = 5; + let features = Features::new(inputs, outputs); + let n = samples * inputs; let (batch_size, epochs, gamma) = (20, 4, 0.01); @@ -19,10 +21,9 @@ fn main() -> anyhow::Result<()> { .unwrap() + 1.0; - let model = LinearLayer::::new(inputs, outputs); + let mut model = Layer::::new_input(features); - let mut sgd = StochasticGradientDescent::new(batch_size, epochs, gamma, model); - let losses = sgd.sgd(&x, &y); - println!("Losses {:?}", losses); + let cost = sgd(&x, &y, &mut model, epochs, gamma, batch_size).unwrap(); + println!("Losses {:?}", cost); Ok(()) } diff --git a/ml/optim/src/grad/descent.rs b/ml/optim/src/grad/descent.rs index b789a020..2adf2249 100644 --- a/ml/optim/src/grad/descent.rs +++ b/ml/optim/src/grad/descent.rs @@ -3,10 +3,50 @@ Contrib: FL03 */ use crate::neural::layers::linear::Linear; -use crate::neural::prelude::{Forward, Layer}; +use crate::neural::prelude::{Activate, Biased, Forward, Layer, Weighted}; use crate::prelude::Norm; -use ndarray::prelude::{Array1, Array2}; +use ndarray::prelude::{Array1, Array2, Axis, NdFloat}; use ndarray_stats::DeviationExt; +use num::{FromPrimitive, Signed}; + +pub fn gradient( + gamma: T, + model: &mut Layer, + data: &Array2, + targets: &Array2, + grad: impl Fn(&Array2) -> Array2, +) -> f64 +where + A: Activate>, + T: FromPrimitive + NdFloat + Signed, +{ + let (samples, _inputs) = data.dim(); + let pred = model.forward(data); + + let ns = T::from(samples).unwrap(); + + let errors = &pred - targets; + // compute the gradient of the objective function w.r.t. the model's weights + let dz = errors * grad(&pred); + // compute the gradient of the objective function w.r.t. the model's weights + let dw = data.t().dot(&dz) / ns; + // compute the gradient of the objective function w.r.t. the model's bias + // let db = dz.sum_axis(Axis(0)) / ns; + // // Apply the gradients to the model's learnable parameters + // model.bias_mut().scaled_add(-gamma, &db); + + model.weights_mut().scaled_add(-gamma, &dw.t()); + + let loss = targets + .mean_sq_err(&model.forward(data)) + .expect("Error when calculating the MSE of the model"); + loss +} + +pub struct GradLayer { + gamma: f64, + layer: Layer, +} #[derive(Clone)] pub struct GradientDescent { @@ -63,7 +103,7 @@ impl GradientDescent { let (samples, _inputs) = data.dim(); let pred = self.model.forward(data); - let errors = targets - &pred; + let errors = &pred - targets; let dz = errors * grad(&pred); let dw = data.t().dot(&dz) / (2.0 * samples as f64); diff --git a/ml/optim/src/grad/sgd.rs b/ml/optim/src/grad/sgd.rs index 862894a8..930b3857 100644 --- a/ml/optim/src/grad/sgd.rs +++ b/ml/optim/src/grad/sgd.rs @@ -5,22 +5,23 @@ //! # Stochastic Gradient Descent (SGD) //! //! -use crate::neural::layers::linear::LinearLayer; -use crate::neural::prelude::{mse, Forward}; +use crate::neural::prelude::{mse, Activate, Forward, Layer, Parameterized, Params, Weighted}; // use crate::prelude::ObjectiveFn; -use ndarray::prelude::{s, Array1, Array2, Axis}; -use ndarray::NdFloat; +use ndarray::prelude::{s, Array1, Array2, Axis, NdFloat}; use num::{Float, FromPrimitive}; use rand::seq::SliceRandom; -pub fn sgd( +pub fn sgd( x: &Array2, y: &Array1, - model: &mut LinearLayer, + model: &mut Layer, epochs: usize, learning_rate: f64, batch_size: usize, -) -> anyhow::Result> { +) -> anyhow::Result> +where + A: Activate> + Clone, +{ let layer = model.clone(); let features = layer.features(); let (samples, _inputs) = x.dim(); @@ -65,7 +66,7 @@ pub fn sgd( // gradient += &(input * cost); } gradient /= batch_size as f64; - model.update_with_gradient(&gradient, learning_rate); + model.weights_mut().scaled_add(-learning_rate, &gradient); println!("Gradient:\n{:?}", &gradient); } @@ -75,13 +76,16 @@ pub fn sgd( Ok(losses) } -pub fn sgd_step( +pub fn sgd_step( x: &Array2, y: &Array1, - model: &mut LinearLayer, + model: &mut Layer, learning_rate: f64, batch_size: usize, -) -> anyhow::Result { +) -> anyhow::Result +where + A: Activate> + Clone, +{ let layer = model.clone(); let features = layer.features(); let (samples, _inputs) = x.dim(); @@ -102,7 +106,7 @@ pub fn sgd_step( pub struct Sgd { batch_size: usize, gamma: f64, // learning rate - model: LinearLayer, + model: Layer, } impl Sgd { @@ -128,14 +132,14 @@ where batch_size: usize, epochs: usize, gamma: T, // learning rate - model: LinearLayer, + model: Layer, } impl StochasticGradientDescent where T: Float, { - pub fn new(batch_size: usize, epochs: usize, gamma: T, model: LinearLayer) -> Self { + pub fn new(batch_size: usize, epochs: usize, gamma: T, model: Layer) -> Self { Self { batch_size, epochs, @@ -156,7 +160,7 @@ where self.gamma } - pub fn model(&self) -> &LinearLayer { + pub fn model(&self) -> &Layer { &self.model } } @@ -190,8 +194,7 @@ where } gradient /= T::from(self.batch_size).unwrap(); - self.model - .update_with_gradient(&gradient.t().to_owned(), self.gamma); + self.model.update_with_gradient(self.gamma, &gradient); println!("Gradient:\n{:?}", &gradient); let loss = mse(&self.model.forward(x), y).unwrap(); @@ -208,37 +211,25 @@ where mod tests { use super::*; use crate::core::prelude::GenerateRandom; + use crate::neural::prelude::{Features, Sigmoid}; use ndarray::prelude::{Array, Array1}; #[test] fn test_sgd() { - let (samples, inputs) = (20, 5); + let (samples, inputs, outputs) = (20, 5, 4); let shape = (samples, inputs); - let (batch_size, epochs, gamma) = (10, 1, 0.01); - // Generate some example data - let x = Array::linspace(1., 100., 100).into_shape(shape).unwrap(); - let y = Array::linspace(1., 100., 5).into_shape(5).unwrap(); - - let model = LinearLayer::::new(inputs, 5); - - let mut sgd = StochasticGradientDescent::new(batch_size, epochs, gamma, model); - sgd.sgd(&x, &y); - } - - #[test] - fn test_stochastic() { - let (samples, inputs) = (20, 5); - let shape = (samples, inputs); + let features = Features::new(inputs, outputs); let (batch_size, epochs, gamma) = (10, 1, 0.01); // Generate some example data let x = Array::linspace(1., 100., 100).into_shape(shape).unwrap(); - let y = Array1::::uniform(0, 100); + let y = Array::linspace(1., 100., 5).into_shape(5).unwrap(); - let mut model = LinearLayer::::new(inputs, 5); + let mut model = Layer::::hidden(features, 5).init(true); - // let grad = sgd(&x, &y, &mut model, epochs, gamma, batch_size); - // assert!(grad.is_ok()); + // let mut sgd = StochasticGradientDescent::new(batch_size, epochs, gamma, model); + // sgd.sgd(&x, &y); + let sgd = sgd(&x, &y, &mut model, epochs, gamma, batch_size).unwrap(); } } diff --git a/ml/optim/src/specs.rs b/ml/optim/src/specs.rs index 05640094..dee54cfb 100644 --- a/ml/optim/src/specs.rs +++ b/ml/optim/src/specs.rs @@ -22,7 +22,7 @@ where pub trait Objective { type Model; - fn objective(&self, x: &Array2, y: &Array1) -> Array1; + fn objective(&self, args: &Array2) -> Array1; } pub trait PartialDerivative { diff --git a/ml/transformers/src/attention/multi/attention.rs b/ml/transformers/src/attention/multi/attention.rs index 456b2768..ebbe2857 100644 --- a/ml/transformers/src/attention/multi/attention.rs +++ b/ml/transformers/src/attention/multi/attention.rs @@ -4,8 +4,7 @@ */ use super::{multihead, MultiHeadParams}; use crate::attention::Weight; -use crate::neural::layers::linear::LinearLayer; -use crate::neural::prelude::{Forward, Mask}; +use crate::neural::prelude::{Forward, Layer, Mask}; use crate::ops::Split; use ndarray::prelude::{Array2, NdFloat}; use ndarray::{ScalarOperand, ShapeError}; @@ -15,7 +14,7 @@ use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct MultiHeadAttention { - linear: LinearLayer, + linear: Layer, params: MultiHeadParams, weights: Weight, } @@ -24,7 +23,7 @@ impl MultiHeadAttention where T: Float, { - pub fn linear(&self) -> &LinearLayer { + pub fn linear(&self) -> &Layer { &self.linear } @@ -45,7 +44,7 @@ where let params = MultiHeadParams::new(heads, model); let weights = Weight::uniform((model, model)); Self { - linear: LinearLayer::new(model, model), + linear: Layer::new_input((model, model).into()), params, weights, } diff --git a/ml/transformers/src/ffn/network.rs b/ml/transformers/src/ffn/network.rs index ed40aa62..6573767a 100644 --- a/ml/transformers/src/ffn/network.rs +++ b/ml/transformers/src/ffn/network.rs @@ -4,25 +4,25 @@ */ use super::FFNParams; use crate::neural::func::activate::{Activate, ReLU}; -use crate::neural::layers::linear::LinearLayer; -use crate::neural::prelude::Forward; +use crate::neural::prelude::{Features, Forward, Layer}; use ndarray::prelude::Array2; use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct FFN { - input: LinearLayer, - output: LinearLayer, + input: Layer, + output: Layer, pub params: FFNParams, } impl FFN { pub fn new(model: usize, network: Option) -> Self { let params = FFNParams::new(model, network.unwrap_or(crate::NETWORK_SIZE)); - let layer = LinearLayer::new(params.model, params.network); + let features = Features::new(model, params.network_size()); + Self { - input: layer.clone(), - output: layer, + input: Layer::new_input(features), + output: Layer::output(features, 1), params, } } From 448249626b60d2613a575abbf7e0b89f4703300b Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 18 Nov 2023 12:43:26 -0600 Subject: [PATCH 057/118] Fix bugs and optimize code Signed-off-by: FL03 --- core/src/lib.rs | 2 + core/src/utils.rs | 20 +++ core/tests/utils.rs | 10 +- data/src/datasets/dataset.rs | 8 + math/src/specs.rs | 29 +--- ml/neural/src/func/activate/activator.rs | 14 +- ml/neural/src/func/activate/binary.rs | 6 +- ml/neural/src/func/activate/linear.rs | 17 +- ml/neural/src/func/activate/mod.rs | 51 +++++- ml/neural/src/func/activate/nonlinear.rs | 153 ++++++++++++++++-- ml/neural/src/layers/layer.rs | 26 +-- ml/neural/src/layers/mod.rs | 22 +-- ml/neural/src/layers/params.rs | 54 +++++-- ml/neural/src/layers/sublayer.rs | 8 +- ml/neural/src/models/mod.rs | 6 +- ml/neural/src/models/model.rs | 11 +- ml/neural/src/models/params.rs | 104 +----------- ml/neural/src/models/stack.rs | 6 +- ml/neural/src/neurons/mod.rs | 21 +-- ml/neural/src/neurons/neuron.rs | 32 ++-- ml/neural/src/neurons/params.rs | 54 +++++-- ml/neural/src/nn/deep.rs | 22 +-- ml/neural/src/params/group.rs | 39 +++++ ml/neural/src/params/mod.rs | 115 ++++++++++--- ml/neural/src/params/param.rs | 124 +++++--------- ml/neural/src/prop/mod.rs | 14 +- ml/neural/src/specs.rs | 49 +----- ml/optim/examples/descent.rs | 45 +++--- ml/optim/examples/sgd.rs | 26 +-- ml/optim/src/cost/mod.rs | 23 --- ml/optim/src/grad/descent.rs | 45 +----- ml/optim/src/grad/mod.rs | 78 ++++++++- ml/optim/src/grad/sgd.rs | 34 ++-- ml/optim/src/optimize/mod.rs | 1 + ml/transformers/src/attention/head.rs | 2 +- ml/transformers/src/attention/mod.rs | 7 +- .../src/attention/multi/attention.rs | 2 +- ml/transformers/src/ffn/network.rs | 4 +- 38 files changed, 730 insertions(+), 554 deletions(-) create mode 100644 ml/neural/src/params/group.rs diff --git a/core/src/lib.rs b/core/src/lib.rs index 66b3a49c..a4ee39f3 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -14,6 +14,8 @@ pub mod errors; pub mod states; pub mod step; + + pub mod prelude { pub use crate::epochs::*; pub use crate::errors::*; diff --git a/core/src/utils.rs b/core/src/utils.rs index b72e6324..72c48d45 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -7,6 +7,26 @@ use ndarray::prelude::{Array, Axis, Dimension}; use ndarray::{concatenate, IntoDimension, RemoveAxis, ShapeError}; use num::Float; +#[macro_export] +macro_rules! linspace { + ( $x:expr ) => { + { + let dim = $x.into_dimension(); + let n = $dim.as_array_view().product(); + ndarray::Array::linspace(T::one(), T::from(n).unwrap(), n).into_shape(dim).unwrap() + } + }; + ( $( $x:expr ),* ) => { + { + let mut res = Vec::new(); + $( + res.push(linarr!($x)); + )* + res + } + }; +} + pub fn concat_iter(axis: usize, iter: impl IntoIterator>) -> Array where D: RemoveAxis, diff --git a/core/tests/utils.rs b/core/tests/utils.rs index 90bb11c3..f8bc311c 100644 --- a/core/tests/utils.rs +++ b/core/tests/utils.rs @@ -1,7 +1,15 @@ #[cfg(test)] extern crate concision_core; -use concision_core::prelude::now; +use concision_core::prelude::{linarr, now}; +use ndarray::prelude::{array, Array2}; + +#[test] +fn test_linarr() { + let args: Array2 = linarr((2, 3)).unwrap(); + // let b = linspace!((2, 3)); + assert_eq!(&args, &array![[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); +} #[test] fn test_timestamp() { diff --git a/data/src/datasets/dataset.rs b/data/src/datasets/dataset.rs index 826997ca..8c4e6b68 100644 --- a/data/src/datasets/dataset.rs +++ b/data/src/datasets/dataset.rs @@ -25,6 +25,14 @@ where pub fn new(data: Array2, targets: Array) -> Self { Self { data, targets } } + + pub fn data(&self) -> &Array2 { + &self.data + } + + pub fn targets(&self) -> &Array { + &self.targets + } } impl std::fmt::Display for DataSet diff --git a/math/src/specs.rs b/math/src/specs.rs index d69eb691..cc3de9c7 100644 --- a/math/src/specs.rs +++ b/math/src/specs.rs @@ -4,7 +4,7 @@ */ use ndarray::prelude::Array; use ndarray::{Dimension, ShapeError}; -use num::{Num, One}; +use num::Num; use std::ops; pub trait NumOpsAssign: @@ -15,27 +15,6 @@ pub trait NumOpsAssign: impl NumOpsAssign for T where T: ops::AddAssign + ops::DivAssign + ops::MulAssign + ops::SubAssign {} -pub trait Product -where - T: Num, -{ - fn product(&self) -> T; -} - -impl Product for I -where - I: Clone + IntoIterator, - T: One + Num + ops::MulAssign, -{ - fn product(&self) -> T { - let mut res = T::one(); - for i in self.clone().into_iter() { - res *= i; - } - res - } -} - trait Matmul where T: Num, @@ -88,11 +67,11 @@ where #[cfg(test)] mod tests { - use super::*; + // use super::*; #[test] fn test_product() { - let args = vec![2, 4, 6]; - assert_eq!(args.product(), 48); + let args = vec![2.0, 4.0, 6.0]; + assert_eq!(args.into_iter().product::(), 48.0); } } diff --git a/ml/neural/src/func/activate/activator.rs b/ml/neural/src/func/activate/activator.rs index c53dc41e..4604558f 100644 --- a/ml/neural/src/func/activate/activator.rs +++ b/ml/neural/src/func/activate/activator.rs @@ -2,24 +2,24 @@ Appellation: activator Contrib: FL03 */ -use super::Activate; +use super::ActivateMethod; pub struct Activator { - method: Box>, + method: Box>, } impl Activator { - pub fn new(method: Box>) -> Self { + pub fn new(method: Box>) -> Self { Self { method } } - pub fn method(&self) -> &dyn Activate { + pub fn method(&self) -> &dyn ActivateMethod { self.method.as_ref() } } -impl Activate for Activator { - fn activate(&self, x: T) -> T { - self.method().activate(x) +impl ActivateMethod for Activator { + fn rho(&self, x: T) -> T { + self.method().rho(x) } } diff --git a/ml/neural/src/func/activate/binary.rs b/ml/neural/src/func/activate/binary.rs index 44e84f22..0040a0e4 100644 --- a/ml/neural/src/func/activate/binary.rs +++ b/ml/neural/src/func/activate/binary.rs @@ -2,7 +2,7 @@ Appellation: binary Contrib: FL03 */ -use super::Activate; +use super::ActivateMethod; use ndarray::prelude::{Array, Dimension}; use num::{One, Zero}; use serde::{Deserialize, Serialize}; @@ -25,12 +25,12 @@ impl Heavyside { } } -impl Activate> for Heavyside +impl ActivateMethod> for Heavyside where D: Dimension, T: Clone + One + PartialOrd + Zero, { - fn activate(&self, x: Array) -> Array { + fn rho(&self, x: Array) -> Array { x.mapv(|x| Self::heavyside(x)) } } diff --git a/ml/neural/src/func/activate/linear.rs b/ml/neural/src/func/activate/linear.rs index 64fbd939..bca574a5 100644 --- a/ml/neural/src/func/activate/linear.rs +++ b/ml/neural/src/func/activate/linear.rs @@ -2,7 +2,8 @@ Appellation: linear Contrib: FL03 */ -use super::{Activate, ActivationFn}; +use super::{Activate, ActivateMethod, ActivationFn}; +use ndarray::prelude::{Array, Dimension}; use serde::{Deserialize, Serialize}; #[derive( @@ -20,8 +21,18 @@ impl LinearActivation { } } -impl Activate for LinearActivation { - fn activate(&self, x: T) -> T { +impl ActivateMethod for LinearActivation { + fn rho(&self, x: T) -> T { Self::method()(x) } } + +impl Activate for LinearActivation +where + D: Dimension, + T: num::Float, +{ + fn activate(&self, args: &Array) -> Array { + args.mapv(|x| self.rho(x)) + } +} diff --git a/ml/neural/src/func/activate/mod.rs b/ml/neural/src/func/activate/mod.rs index 6a00a7e2..15d3c4e6 100644 --- a/ml/neural/src/func/activate/mod.rs +++ b/ml/neural/src/func/activate/mod.rs @@ -14,21 +14,62 @@ pub(crate) mod nonlinear; pub type ActivationFn = fn(T) -> T; -pub type BoxedActivation = Box>; +pub type BoxedActivation = Box>; + +use ndarray::prelude::{Array, Dimension, Ix2}; +use num::Float; + +pub trait Activate +where + D: Dimension, + T: Float, +{ + fn activate(&self, args: &Array) -> Array; +} + +pub trait RhoGradient: Activate +where + D: Dimension, + T: Float, +{ + fn gradient(&self, args: &Array) -> Array; +} + +// impl Rho for F +// where +// D: Dimension, +// F: Fn(&Array) -> Array, +// T: Float +// { +// fn activate(&self, args: &Array) -> Array { +// self.call((args,)) +// } +// } + +// impl Rho for A +// where +// A: Activate, +// D: Dimension, +// T: Float +// { +// fn activate(&self, args: &Array) -> Array { +// args.mapv(|x| self.rho(x)) +// } +// } pub trait ActivationMethod { fn method_name(&self) -> &str; } -pub trait Activate { - fn activate(&self, x: T) -> T; +pub trait ActivateMethod { + fn rho(&self, x: T) -> T; } -impl Activate for F +impl ActivateMethod for F where F: Fn(T) -> T, { - fn activate(&self, x: T) -> T { + fn rho(&self, x: T) -> T { self.call((x,)) } } diff --git a/ml/neural/src/func/activate/nonlinear.rs b/ml/neural/src/func/activate/nonlinear.rs index bfa5e277..62330537 100644 --- a/ml/neural/src/func/activate/nonlinear.rs +++ b/ml/neural/src/func/activate/nonlinear.rs @@ -2,10 +2,10 @@ Appellation: nonlinear Contrib: FL03 */ -use super::Activate; +use super::{Activate, ActivateMethod}; use ndarray::prelude::{Array, Axis, Dimension, NdFloat}; use ndarray::RemoveAxis; -use num::{Float, Zero}; +use num::{Float, One, Zero}; use serde::{Deserialize, Serialize}; pub fn softmax(args: Array) -> Array @@ -38,6 +38,29 @@ where pub struct ReLU; impl ReLU { + pub fn new() -> Self { + Self + } + + pub fn derivative(x: T) -> T + where + T: One + PartialOrd + Zero, + { + if x > T::zero() { + T::one() + } else { + T::zero() + } + } + + pub fn gradient(args: &Array) -> Array + where + D: Dimension, + T: Clone + One + PartialOrd + Zero, + { + args.mapv(|x| Self::derivative(x)) + } + pub fn relu(args: T) -> T where T: PartialOrd + Zero, @@ -50,27 +73,32 @@ impl ReLU { } } -impl Activate> for ReLU +impl ActivateMethod for ReLU +where + T: PartialOrd + Zero, +{ + fn rho(&self, x: T) -> T { + Self::relu(x) + } +} + +impl Activate for ReLU where D: Dimension, T: Float, { - fn activate(&self, x: Array) -> Array { + fn activate(&self, x: &Array) -> Array { x.mapv(|x| Self::relu(x)) } } - #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] pub struct Sigmoid; impl Sigmoid { - pub fn sigmoid(x: T) -> T - where - T: Float, - { - T::one() / (T::one() + (-x).exp()) + pub fn new() -> Self { + Self } pub fn derivative(x: T) -> T @@ -87,14 +115,30 @@ impl Sigmoid { { args.mapv(|x| Self::derivative(x)) } + + pub fn sigmoid(x: T) -> T + where + T: Float, + { + T::one() / (T::one() + (-x).exp()) + } +} + +impl ActivateMethod for Sigmoid +where + T: Float, +{ + fn rho(&self, x: T) -> T { + Self::sigmoid(x) + } } -impl Activate> for Sigmoid +impl Activate for Sigmoid where D: Dimension, T: Float, { - fn activate(&self, x: Array) -> Array { + fn activate(&self, x: &Array) -> Array { x.mapv(|x| Self::sigmoid(x)) } } @@ -140,12 +184,12 @@ impl Softmax { } } -impl Activate> for Softmax +impl Activate for Softmax where D: Dimension + RemoveAxis, T: NdFloat, { - fn activate(&self, x: Array) -> Array { + fn activate(&self, x: &Array) -> Array { let exp = x.mapv(|x| x.exp()); if let Some(axis) = self.axis { let denom = exp.sum_axis(Axis(axis)); @@ -157,18 +201,97 @@ where } } +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] +pub struct Tanh; + +impl Tanh { + pub fn new() -> Self { + Self + } + + pub fn derivative(x: T) -> T + where + T: Float, + { + T::one() - x.tanh().powi(2) + } + + pub fn gradient(args: &Array) -> Array + where + D: Dimension, + T: Float, + { + args.mapv(|x| Self::derivative(x)) + } + + pub fn tanh(x: T) -> T + where + T: Float, + { + x.tanh() + } +} + +impl ActivateMethod for Tanh +where + T: Float, +{ + fn rho(&self, x: T) -> T { + x.tanh() + } +} + +impl Activate for Tanh +where + D: Dimension, + T: Float, +{ + fn activate(&self, x: &Array) -> Array { + x.mapv(|x| Self::tanh(x)) + } +} + #[cfg(test)] mod tests { use super::*; use computare::prelude::RoundTo; use ndarray::array; + #[test] + fn test_relu() { + let exp = array![0.0, 0.0, 3.0]; + let args = array![-1.0, 0.0, 3.0]; + + let res = ReLU::new().activate(&args); + assert_eq!(res, exp); + } + + #[test] + fn test_sigmoid() { + let exp = array![0.73105858, 0.88079708, 0.95257413]; + let args = array![1.0, 2.0, 3.0]; + + let res = Sigmoid::new().activate(&args).mapv(|i| i.round_to(8)); + assert_eq!(res, exp); + } + #[test] fn test_softmax() { let exp = array![0.09003057, 0.24472847, 0.66524096]; let args = array![1.0, 2.0, 3.0]; - let res = Activate::activate(&Softmax::new(None), args).mapv(|i| i.round_to(8)); + let res = Softmax::new(None).activate(&args).mapv(|i| i.round_to(8)); + assert_eq!(res, exp); + } + + #[test] + fn test_tanh() { + let exp = array![0.76159416, 0.96402758, 0.99505475]; + let args = array![1.0, 2.0, 3.0]; + + let res = Tanh::new().activate(&args).mapv(|i| i.round_to(8)); assert_eq!(res, exp); } } diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 189f48fd..7802069d 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -4,7 +4,7 @@ */ use super::{Features, LayerParams, LayerType, Position}; use crate::prelude::{Activate, Forward, LinearActivation, Parameterized, Params}; -use ndarray::prelude::{Array2, NdFloat}; +use ndarray::prelude::{Array2, Ix2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Layer where - A: Activate>, + A: Activate, T: Float, { activator: A, @@ -23,7 +23,7 @@ where impl Layer where - A: Activate> + Default, + A: Default + Activate, T: Float, { pub fn new(features: Features, position: Position) -> Self { @@ -35,7 +35,7 @@ where } } - pub fn new_input(features: Features) -> Self { + pub fn input(features: Features) -> Self { Self::new(features, Position::input()) } @@ -50,7 +50,7 @@ where impl Layer where - A: Activate>, + A: Activate, T: Float, { pub fn activator(&self) -> &A { @@ -72,7 +72,7 @@ where impl Layer where - A: Activate>, + A: Activate, T: Float + 'static, { pub fn update_with_gradient(&mut self, gamma: T, grad: &Array2) { @@ -82,7 +82,7 @@ where impl Layer where - A: Activate>, + A: Activate, T: NdFloat, { pub fn linear(&self, args: &Array2) -> Array2 { @@ -92,7 +92,7 @@ where impl Layer where - A: Activate>, + A: Activate, T: Float + SampleUniform, { pub fn init(mut self, biased: bool) -> Self { @@ -103,19 +103,19 @@ where impl Forward> for Layer where - A: Activate>, + A: Activate, T: NdFloat, { type Output = Array2; fn forward(&self, args: &Array2) -> Self::Output { - self.activator.activate(self.linear(args)) + self.activator.activate(&self.linear(args)) } } impl Parameterized for Layer where - A: Activate>, + A: Activate, T: Float, { type Features = Features; @@ -140,7 +140,7 @@ where impl PartialOrd for Layer where - A: Activate> + PartialEq, + A: Activate + PartialEq, T: Float, { fn partial_cmp(&self, other: &Self) -> Option { @@ -150,7 +150,7 @@ where impl From for Layer where - A: Activate> + Default, + A: Activate + Default, T: Float, { fn from(features: Features) -> Self { diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index 71b0414a..afaf1e0d 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -14,29 +14,11 @@ pub(crate) mod sublayer; pub mod linear; use crate::func::activate::Activate; -use ndarray::prelude::{Array1, Array2}; +use ndarray::prelude::Ix2; use num::Float; pub trait L { - fn forward_slice(&self, args: &Array2, rho: impl Activate) -> Array2 - where - T: 'static, - { - let z = args.dot(self.weights()) + self.bias(); - z.mapv(|x| rho.activate(x)) - } - // - fn process(&self, args: &Array2, rho: impl Activate) -> Array2 - where - T: 'static, - { - let z = args.dot(self.weights()) + self.bias(); - z.mapv(|x| rho.activate(x)) - } - - fn bias(&self) -> &Array1; - - fn weights(&self) -> &Array2; + type Rho: Activate; } pub(crate) mod utils {} diff --git a/ml/neural/src/layers/params.rs b/ml/neural/src/layers/params.rs index e99e46dc..599df324 100644 --- a/ml/neural/src/layers/params.rs +++ b/ml/neural/src/layers/params.rs @@ -4,7 +4,7 @@ */ use super::Features; use crate::core::prelude::GenerateRandom; -use crate::prelude::Params; +use crate::prelude::{Biased, Params, Weighted}; use ndarray::prelude::{Array1, Array2, Ix2}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; @@ -77,7 +77,36 @@ where } } -impl Params for LayerParams +// impl Params for LayerParams +// where +// T: Float, +// { +// fn bias(&self) -> &Array1 { +// &self.bias +// } + +// fn bias_mut(&mut self) -> &mut Array1 { +// &mut self.bias +// } + +// fn weights(&self) -> &Array2 { +// &self.weights +// } + +// fn weights_mut(&mut self) -> &mut Array2 { +// &mut self.weights +// } + +// fn set_bias(&mut self, bias: Array1) { +// self.bias = bias; +// } + +// fn set_weights(&mut self, weights: Array2) { +// self.weights = weights; +// } +// } + +impl Biased for LayerParams where T: Float, { @@ -89,19 +118,24 @@ where &mut self.bias } - fn weights(&self) -> &Array2 { - &self.weights - } - - fn weights_mut(&mut self) -> &mut Array2 { - &mut self.weights - } - fn set_bias(&mut self, bias: Array1) { self.bias = bias; } +} +impl Weighted for LayerParams +where + T: Float, +{ fn set_weights(&mut self, weights: Array2) { self.weights = weights; } + + fn weights(&self) -> &Array2 { + &self.weights + } + + fn weights_mut(&mut self) -> &mut Array2 { + &mut self.weights + } } diff --git a/ml/neural/src/layers/sublayer.rs b/ml/neural/src/layers/sublayer.rs index dfac9517..986a274c 100644 --- a/ml/neural/src/layers/sublayer.rs +++ b/ml/neural/src/layers/sublayer.rs @@ -7,14 +7,14 @@ use crate::func::activate::{Activate, LinearActivation}; use crate::ops::LayerNorm; use crate::prelude::Forward; -use ndarray::prelude::{Array2, NdFloat}; +use ndarray::prelude::{Array2, Ix2, NdFloat}; use num::{Float, FromPrimitive}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Sublayer where - A: Activate>, + A: Activate, T: Float, { layer: Layer, @@ -23,7 +23,7 @@ where impl Sublayer where - A: Activate>, + A: Activate, T: Float, { pub fn new(layer: Layer, norm: LayerNorm) -> Self { @@ -33,7 +33,7 @@ where impl Forward> for Sublayer where - A: Activate>, + A: Activate, T: FromPrimitive + NdFloat, { type Output = Array2; diff --git a/ml/neural/src/models/mod.rs b/ml/neural/src/models/mod.rs index 5993c3f7..9db033b2 100644 --- a/ml/neural/src/models/mod.rs +++ b/ml/neural/src/models/mod.rs @@ -11,12 +11,12 @@ pub(crate) mod params; pub mod stack; +use ndarray::prelude::Array2; + pub trait Module { fn add_module(&mut self, module: impl Module); - fn params(&self) -> &ModelParams; - - fn params_mut(&mut self) -> &mut ModelParams; + fn forward(&self, args: &Array2) -> Array2; } pub(crate) mod utils {} diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs index 3ba37326..8cbdefff 100644 --- a/ml/neural/src/models/model.rs +++ b/ml/neural/src/models/model.rs @@ -2,9 +2,8 @@ Appellation: model Contrib: FL03 */ -use super::ModelParams; -use crate::prelude::{Activate, Features, Params}; -use ndarray::prelude::{Array, Ix2}; +use crate::prelude::{Activate, Features, LayerParams, Params}; +use ndarray::prelude::Ix2; use num::Float; pub struct BaseModel @@ -12,12 +11,14 @@ where T: Float, { pub features: Features, - activator: Box>, + activator: Box>, params: Box>, } pub struct Model { pub features: Features, children: Vec>, - params: ModelParams, + layers: usize, + + params: LayerParams, } diff --git a/ml/neural/src/models/params.rs b/ml/neural/src/models/params.rs index cc4b4caf..c592d89a 100644 --- a/ml/neural/src/models/params.rs +++ b/ml/neural/src/models/params.rs @@ -1,106 +1,8 @@ /* - Appellation: model + Appellation: params Contrib: FL03 */ -use crate::core::prelude::GenerateRandom; -use crate::prelude::{Features, Params}; -use ndarray::prelude::{Array1, Array2, Ix2}; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use num::Float; -use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub struct ModelParams { - bias: Array1, - pub features: Features, - weights: Array2, -} - -impl ModelParams -where - T: Float, -{ - pub fn new(features: Features) -> Self { - Self { - bias: Array1::zeros(features.outputs()), - features, - weights: Array2::zeros(features.out_by_in()), - } - } - - pub fn reset(&mut self) { - self.bias = Array1::zeros(self.features.outputs()); - self.weights = Array2::zeros(self.features.out_by_in()); - } - - pub fn features(&self) -> &Features { - &self.features - } - - pub fn features_mut(&mut self) -> &mut Features { - &mut self.features - } - - pub fn with_bias(mut self, bias: Array1) -> Self { - self.bias = bias; - self - } - - pub fn with_weights(mut self, weights: Array2) -> Self { - self.weights = weights; - self - } -} - -impl ModelParams -where - T: Float + SampleUniform, -{ - pub fn init(mut self, biased: bool) -> Self { - if biased { - self = self.init_bias(); - } - self.init_weight() - } - - pub fn init_bias(mut self) -> Self { - let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); - self.bias = Array1::uniform_between(dk, self.features().outputs()); - self - } - - pub fn init_weight(mut self) -> Self { - let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); - self.weights = Array2::uniform_between(dk, self.features().out_by_in()); - self - } -} - -impl Params for ModelParams -where - T: Float, -{ - fn bias(&self) -> &Array1 { - &self.bias - } - - fn bias_mut(&mut self) -> &mut Array1 { - &mut self.bias - } - - fn weights(&self) -> &Array2 { - &self.weights - } - - fn weights_mut(&mut self) -> &mut Array2 { - &mut self.weights - } - - fn set_bias(&mut self, bias: Array1) { - self.bias = bias; - } - - fn set_weights(&mut self, weights: Array2) { - self.weights = weights; - } +pub struct ModelConfig { + pub layers: usize, } diff --git a/ml/neural/src/models/stack.rs b/ml/neural/src/models/stack.rs index 1953e4d5..360147bd 100644 --- a/ml/neural/src/models/stack.rs +++ b/ml/neural/src/models/stack.rs @@ -4,14 +4,14 @@ */ use crate::layers::Layer; use crate::prelude::{Activate, LinearActivation}; -use ndarray::prelude::Array2; +use ndarray::prelude::Ix2; use num::Float; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Stack where - A: Activate>, + A: Activate, T: Float, { layers: Vec>, @@ -19,7 +19,7 @@ where impl Stack where - A: Activate> + Default, + A: Activate + Default, T: Float, { pub fn new() -> Self { diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index b088b31e..7bfa6bb1 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -10,19 +10,22 @@ pub(crate) mod node; pub(crate) mod params; use crate::func::activate::Activate; -use ndarray::prelude::{Array1, Array2, NdFloat}; +use ndarray::prelude::{Array0, Array1, Array2, Ix1, NdFloat}; pub trait ArtificialNeuron where T: NdFloat, { - type Rho: Activate>; + type Rho: Activate; - fn bias(&self) -> T; + fn bias(&self) -> Array0; + + fn linear(&self, args: &Array2) -> Array1 { + args.dot(self.weights()) + self.bias() + } fn forward(&self, args: &Array2) -> Array1 { - self.rho() - .activate(args.dot(&self.weights().t()) + self.bias()) + self.rho().activate(&self.linear(args)) } fn rho(&self) -> &Self::Rho; @@ -35,7 +38,7 @@ pub(crate) mod utils {} #[cfg(test)] mod tests { use super::*; - use crate::func::activate::{softmax, Activate, Softmax}; + use crate::func::activate::{softmax, ActivateMethod, Softmax}; use crate::prelude::Forward; // use lazy_static::lazy_static; use ndarray::{array, Array1}; @@ -43,12 +46,10 @@ mod tests { fn _artificial( args: &Array1, bias: Option>, - rho: impl Activate>, + rho: impl ActivateMethod>, weights: &Array1, ) -> Array1 { - rho.activate( - args.dot(weights) + bias.unwrap_or_else(|| Array1::::zeros(args.shape()[0])), - ) + rho.rho(args.dot(weights) + bias.unwrap_or_else(|| Array1::::zeros(args.shape()[0]))) } #[test] diff --git a/ml/neural/src/neurons/neuron.rs b/ml/neural/src/neurons/neuron.rs index cc05223f..e5c29ae0 100644 --- a/ml/neural/src/neurons/neuron.rs +++ b/ml/neural/src/neurons/neuron.rs @@ -5,29 +5,29 @@ use crate::core::GenerateRandom; use crate::func::activate::{Activate, LinearActivation}; use crate::prelude::Forward; -use ndarray::prelude::{Array1, Array2, NdFloat}; +use ndarray::prelude::{Array0, Array1, Array2, Ix1, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; -use rand::Rng; /// Artificial Neuron #[derive(Clone, Debug, PartialEq)] pub struct Neuron where - A: Activate>, + A: Activate, + T: Float, { activation: A, - bias: T, + bias: Array0, features: usize, weights: Array1, } impl Neuron where - A: Activate>, + A: Activate, T: Float, { - pub fn bias(&self) -> &T { + pub fn bias(&self) -> &Array0 { &self.bias } @@ -43,7 +43,7 @@ where &mut self.weights } - pub fn set_bias(&mut self, bias: T) { + pub fn set_bias(&mut self, bias: Array0) { self.bias = bias; } @@ -51,7 +51,7 @@ where self.weights = weights; } - pub fn with_bias(mut self, bias: T) -> Self { + pub fn with_bias(mut self, bias: Array0) -> Self { self.bias = bias; self } @@ -70,12 +70,12 @@ where impl Neuron where T: NdFloat, - A: Activate> + Default, + A: Activate + Default, { pub fn new(features: usize) -> Self { Self { activation: A::default(), - bias: T::zero(), + bias: Array0::zeros(()), features, weights: Array1::zeros(features), } @@ -85,7 +85,7 @@ where impl Neuron where T: NdFloat, - A: Activate>, + A: Activate, { pub fn apply_gradient(&mut self, gamma: T, gradient: G) where @@ -99,7 +99,7 @@ where impl Neuron where T: Float + SampleUniform, - A: Activate>, + A: Activate, { pub fn init(mut self, biased: bool) -> Self { if biased { @@ -110,7 +110,7 @@ where pub fn init_bias(mut self) -> Self { let dk = (T::one() / T::from(self.features).unwrap()).sqrt(); - self.bias = rand::thread_rng().gen_range(-dk..dk); + self.bias = Array0::uniform_between(dk, ()); self } @@ -134,12 +134,12 @@ where impl Forward> for Neuron where T: NdFloat, - A: Activate>, + A: Activate, { type Output = Array1; fn forward(&self, args: &Array2) -> Self::Output { - let linstep = args.dot(&self.weights().t()) + self.bias; - self.rho().activate(linstep) + let linstep = args.dot(&self.weights().t()) + self.bias(); + self.rho().activate(&linstep) } } diff --git a/ml/neural/src/neurons/params.rs b/ml/neural/src/neurons/params.rs index 1cbd10f5..2250ac74 100644 --- a/ml/neural/src/neurons/params.rs +++ b/ml/neural/src/neurons/params.rs @@ -4,7 +4,7 @@ */ use crate::core::prelude::GenerateRandom; -use crate::prelude::Params; +use crate::prelude::{Biased, Weighted}; use ndarray::prelude::{Array0, Array1, Array2, Ix1, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::{Float, FromPrimitive}; @@ -103,7 +103,36 @@ where } } -impl Params for NeuronParams +// impl Params for NeuronParams +// where +// T: Float, +// { +// fn bias(&self) -> &Array0 { +// &self.bias +// } + +// fn bias_mut(&mut self) -> &mut Array0 { +// &mut self.bias +// } + +// fn weights(&self) -> &Array1 { +// &self.weights +// } + +// fn weights_mut(&mut self) -> &mut Array1 { +// &mut self.weights +// } + +// fn set_bias(&mut self, bias: Array0) { +// self.bias = bias; +// } + +// fn set_weights(&mut self, weights: Array1) { +// self.weights = weights; +// } +// } + +impl Biased for NeuronParams where T: Float, { @@ -115,19 +144,24 @@ where &mut self.bias } - fn weights(&self) -> &Array1 { - &self.weights - } - - fn weights_mut(&mut self) -> &mut Array1 { - &mut self.weights - } - fn set_bias(&mut self, bias: Array0) { self.bias = bias; } +} +impl Weighted for NeuronParams +where + T: Float, +{ fn set_weights(&mut self, weights: Array1) { self.weights = weights; } + + fn weights(&self) -> &Array1 { + &self.weights + } + + fn weights_mut(&mut self) -> &mut Array1 { + &mut self.weights + } } diff --git a/ml/neural/src/nn/deep.rs b/ml/neural/src/nn/deep.rs index 64910169..60bd57af 100644 --- a/ml/neural/src/nn/deep.rs +++ b/ml/neural/src/nn/deep.rs @@ -2,18 +2,18 @@ Appellation: network Contrib: FL03 */ -use crate::func::activate::{Activate, LinearActivation}; +use crate::func::activate::{Activate, ActivateMethod, LinearActivation}; use crate::prelude::{Forward, Layer, Parameterized}; -use ndarray::prelude::{Array2, NdFloat}; +use ndarray::prelude::{Array2, Ix2, NdFloat}; use num::Float; pub struct DeepNetwork where T: Float, - I: Activate>, - H: Activate>, - O: Activate>, + I: Activate, + H: Activate, + O: Activate, { pub input: Layer, pub hidden: Vec>, @@ -23,9 +23,9 @@ where impl DeepNetwork where T: Float, - I: Activate>, - H: Activate>, - O: Activate>, + I: Activate, + H: Activate, + O: Activate, { pub fn new(input: Layer, hidden: Vec>, output: Layer) -> Self { Self { @@ -53,9 +53,9 @@ where impl Forward> for DeepNetwork where T: NdFloat, - I: Activate>, - H: Activate>, - O: Activate>, + I: Activate, + H: Activate, + O: Activate, { type Output = Array2; diff --git a/ml/neural/src/params/group.rs b/ml/neural/src/params/group.rs new file mode 100644 index 00000000..ae6d8bd1 --- /dev/null +++ b/ml/neural/src/params/group.rs @@ -0,0 +1,39 @@ +/* + Appellation: group + Contrib: FL03 +*/ +use crate::core::prelude::GenerateRandom; +use crate::layers::Features; +use ndarray::prelude::{Array, Array1, Array2, Dimension, Ix2}; +use ndarray::{IntoDimension, RemoveAxis}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use num::Float; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ParamGroup +where + T: Float, + D: Dimension, + ::Smaller: Dimension, +{ + bias: Array, + weights: Array, +} + +// impl ParamGroup +// where +// T: Float, +// D: Dimension + RemoveAxis, +// ::Smaller: Dimension, +// { +// pub fn new(dim: impl IntoDimension) -> Self { +// let dim = dim.into_dimension(); +// let bias = Array::zeros(); +// let weights = Array::zeros(dim.clone()); +// Self { +// bias: Array::::zeros(&weights.shape()[..(dim.ndim() - 1)]), +// weights, +// } +// } +// } diff --git a/ml/neural/src/params/mod.rs b/ml/neural/src/params/mod.rs index c627e5e6..2fa53743 100644 --- a/ml/neural/src/params/mod.rs +++ b/ml/neural/src/params/mod.rs @@ -6,9 +6,10 @@ //! //! ## Overview //! -pub use self::{bias::*, param::*, shapes::*, utils::*, weight::*}; +pub use self::{bias::*, group::*, param::*, shapes::*, utils::*, weight::*}; pub(crate) mod bias; +pub(crate) mod group; pub(crate) mod param; pub(crate) mod shapes; pub(crate) mod weight; @@ -30,6 +31,8 @@ where fn bias(&self) -> &Array; /// Returns a mutable reference to the bias of the layer. fn bias_mut(&mut self) -> &mut Array; + /// Sets the bias of the layer. + fn set_bias(&mut self, bias: Array); } pub trait Weighted @@ -41,14 +44,19 @@ where fn weights(&self) -> &Array; /// Returns a mutable reference to the weights of the layer. fn weights_mut(&mut self) -> &mut Array; + /// Sets the weights of the layer. + fn set_weights(&mut self, weights: Array); } -pub trait WeightedExt: Weighted +pub trait ParamsExt: Biased where Array: Dot, Output = Array>, D: Dimension, T: Float, { + fn linear(&self, args: &Array) -> Array { + args.dot(self.weights()) + self.bias() + } } pub trait Params @@ -76,7 +84,7 @@ where T: Float, { type Features: IntoDimension; - type Params: Params; + type Params: Biased; fn features(&self) -> &Self::Features; @@ -85,48 +93,105 @@ where fn params(&self) -> &Self::Params; fn params_mut(&mut self) -> &mut Self::Params; - - fn set_bias(&mut self, bias: Array) { - self.params_mut().set_bias(bias); - } - - fn set_weights(&mut self, weights: Array) { - self.params_mut().set_weights(weights); - } } -impl Biased for P +// impl Params for S +// where +// S: Parameterized, +// D: Dimension, +// P: Biased, +// T: Float, +// ::Smaller: Dimension, +// { +// fn bias(&self) -> &Array { +// self.params().bias() +// } + +// fn bias_mut(&mut self) -> &mut Array { +// self.params_mut().bias_mut() +// } + +// fn weights(&self) -> &Array { +// self.params().weights() +// } + +// fn weights_mut(&mut self) -> &mut Array { +// self.params_mut().weights_mut() +// } + +// fn set_bias(&mut self, bias: Array) { +// self.params_mut().set_bias(bias) +// } + +// fn set_weights(&mut self, weights: Array) { +// self.params_mut().set_weights(weights) +// } +// } + +impl Params for P where D: Dimension, - P: Parameterized, + P: Biased, T: Float, ::Smaller: Dimension, -

>::Params: 'static, { fn bias(&self) -> &Array { - self.params().bias() + self.bias() } fn bias_mut(&mut self) -> &mut Array { - self.params_mut().bias_mut() + self.bias_mut() } -} -impl Weighted for P -where - P: Parameterized, - D: Dimension, - T: Float, -{ fn weights(&self) -> &Array { - self.params().weights() + self.weights() } fn weights_mut(&mut self) -> &mut Array { - self.params_mut().weights_mut() + self.weights_mut() + } + + fn set_bias(&mut self, bias: Array) { + self.set_bias(bias) + } + + fn set_weights(&mut self, weights: Array) { + self.set_weights(weights) } } +// impl Biased for P +// where +// D: Dimension, +// P: Parameterized, +// T: Float, +// ::Smaller: Dimension, +//

>::Params: 'static, +// { +// fn bias(&self) -> &Array { +// self.params().bias() +// } + +// fn bias_mut(&mut self) -> &mut Array { +// self.params_mut().bias_mut() +// } +// } + +// impl Weighted for P +// where +// P: Parameterized, +// D: Dimension, +// T: Float, +// { +// fn weights(&self) -> &Array { +// self.params().weights() +// } + +// fn weights_mut(&mut self) -> &mut Array { +// self.params_mut().weights_mut() +// } +// } + pub(crate) mod utils {} #[cfg(test)] diff --git a/ml/neural/src/params/param.rs b/ml/neural/src/params/param.rs index 209615fe..2d7b62b9 100644 --- a/ml/neural/src/params/param.rs +++ b/ml/neural/src/params/param.rs @@ -1,118 +1,66 @@ /* - Appellation: model + Appellation: param Contrib: FL03 */ -use crate::core::prelude::GenerateRandom; -use crate::layers::Features; -use ndarray::prelude::{Array1, Array2}; -use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray::prelude::{Array, Dimension, Ix2}; use num::Float; -use serde::{Deserialize, Serialize}; -pub trait ParamFeatures { - fn inputs(&self) -> usize; +pub struct ParamBuilder { + name: String, +} - fn outputs(&self) -> usize; +impl ParamBuilder { + pub fn new() -> Self { + Self { + name: String::new(), + } + } - fn in_by_out(&self) -> (usize, usize) { - (self.inputs(), self.outputs()) + pub fn with_name(mut self, name: String) -> Self { + self.name = name; + self } - fn out_by_in(&self) -> (usize, usize) { - (self.outputs(), self.inputs()) + pub fn build(self, params: Array) -> Param + where + T: Float, + D: Dimension, + { + Param::new(self.name, params) } } -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub struct ParamGroup +pub struct Param where T: Float, + D: Dimension, { - bias: Array1, - pub features: Features, - weights: Array2, + name: String, + params: Array, } -impl ParamGroup +impl Param where T: Float, + D: Dimension, { - pub fn new(features: Features) -> Self { - Self { - bias: Array1::zeros(features.outputs()), - features, - weights: Array2::zeros(features.out_by_in()), - } - } - - pub fn reset(&mut self) { - self.bias = Array1::zeros(self.features.outputs()); - self.weights = Array2::zeros(self.features.out_by_in()); + pub fn new(name: String, params: Array) -> Self { + Self { name, params } } - pub fn bias(&self) -> &Array1 { - &self.bias + pub fn name(&self) -> &str { + &self.name } - pub fn bias_mut(&mut self) -> &mut Array1 { - &mut self.bias + pub fn params(&self) -> &Array { + &self.params } - pub fn features(&self) -> &Features { - &self.features + pub fn params_mut(&mut self) -> &mut Array { + &mut self.params } - pub fn features_mut(&mut self) -> &mut Features { - &mut self.features - } - - pub fn weights(&self) -> &Array2 { - &self.weights - } - - pub fn weights_mut(&mut self) -> &mut Array2 { - &mut self.weights - } - - pub fn set_bias(&mut self, bias: Array1) { - self.bias = bias; - } - - pub fn set_weights(&mut self, weights: Array2) { - self.weights = weights; - } - - pub fn with_bias(mut self, bias: Array1) -> Self { - self.bias = bias; - self - } - - pub fn with_weights(mut self, weights: Array2) -> Self { - self.weights = weights; - self - } -} - -impl ParamGroup -where - T: Float + SampleUniform, -{ - pub fn init(mut self, biased: bool) -> Self { - if biased { - self = self.init_bias(); - } - self.init_weight() - } - - pub fn init_bias(mut self) -> Self { - let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); - self.bias = Array1::uniform_between(dk, self.features().outputs()); - self - } - - pub fn init_weight(mut self) -> Self { - let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); - self.weights = Array2::uniform_between(dk, self.features().out_by_in()); - self + pub fn set_name(&mut self, name: String) { + self.name = name; } } diff --git a/ml/neural/src/prop/mod.rs b/ml/neural/src/prop/mod.rs index b40bd16f..f7b99784 100644 --- a/ml/neural/src/prop/mod.rs +++ b/ml/neural/src/prop/mod.rs @@ -11,6 +11,7 @@ pub(crate) mod modes; pub(crate) mod propagation; // pub mod forward; +use ndarray::prelude::{Array, Array2, Dimension}; pub trait Backward { type Params; @@ -25,10 +26,17 @@ pub trait Forward { fn forward(&self, args: &T) -> Self::Output; } -pub trait ForwardIter { - type Output; +pub trait Propagate { + type Optimizer; + + fn backward( + &mut self, + args: &Array2, + targets: &Array, + opt: Self::Optimizer, + ) -> Array; - fn forward_iter(&self, args: &T) -> Self::Output; + fn forward(&self, args: &Array2) -> Array2; } pub(crate) mod utils {} diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index 7221e86f..78ed25ed 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -2,56 +2,17 @@ Appellation: specs Contrib: FL03 */ -use ndarray::prelude::{Array, Array2}; -use ndarray::{Dimension, IntoDimension}; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Uniform}; -use ndarray_rand::RandomExt; +use ndarray::prelude::{Array, Array2, Dimension, Ix2}; use num::Float; -pub trait InitRandom +pub trait Initializer where - T: Float + SampleUniform, -{ - type Dim: Dimension; - - fn bernoulli( - dim: impl IntoDimension, - p: Option, - ) -> Result, BernoulliError> { - let dist = Bernoulli::new(p.unwrap_or(0.5))?; - Ok(Array::random(dim.into_dimension(), dist)) - } - - fn uniform(axis: usize, dim: impl IntoDimension) -> Array { - let dim = dim.into_dimension(); - let k = (T::one() / T::from(dim[axis]).unwrap()).sqrt(); - Array::random(dim, Uniform::new(-k, k)) - } -} - -impl InitRandom for Array -where - T: Float + SampleUniform, D: Dimension, + T: Float, { - type Dim = D; - - fn bernoulli( - dim: impl IntoDimension, - p: Option, - ) -> Result, BernoulliError> { - let dist = Bernoulli::new(p.unwrap_or(0.5))?; - Ok(Array::random(dim.into_dimension(), dist)) - } - - fn uniform(axis: usize, dim: impl IntoDimension) -> Array { - let dim = dim.into_dimension(); - let k = (T::from(dim[axis]).unwrap()).sqrt(); - Array::random(dim, Uniform::new(-k, k)) - } + fn init_weight(&self) -> Array; } pub trait Trainable { - fn train(&mut self, args: &Array2) -> Array2; + fn train(&mut self, args: &Array2, targets: &Array2) -> Array2; } diff --git a/ml/optim/examples/descent.rs b/ml/optim/examples/descent.rs index cdff86e3..6537a917 100644 --- a/ml/optim/examples/descent.rs +++ b/ml/optim/examples/descent.rs @@ -2,23 +2,21 @@ use concision_core::prelude::linarr; use concision_neural::prelude::{Features, Layer, Linear, LinearActivation, Sigmoid}; use concision_neural::prop::Forward; use concision_optim::prelude::{gradient, gradient_descent, GradientDescent}; -use ndarray::prelude::{Array, Array1}; +use ndarray::prelude::Array1; fn main() -> anyhow::Result<()> { - let (samples, inputs) = (20, 5); - let outputs = 3; + let (samples, inputs) = (20, 8); + let outputs = 4; let features = Features::new(inputs, outputs); - let _n = samples * inputs; - - let (epochs, gamma) = (100000, 0.05); + let (epochs, gamma) = (100000, 0.005); // basic_descent(epochs, features, gamma)?; - // sample_descent(epochs, features, gamma)?; + // sample_descent(epochs, features, gamma, samples)?; - sample_gradient(epochs, features, gamma)?; + sample_gradient(epochs, features, gamma, samples)?; Ok(()) } @@ -33,17 +31,15 @@ pub fn basic_descent(epochs: usize, features: Features, gamma: f64) -> anyhow::R Ok(()) } -pub fn sample_descent(epochs: usize, features: Features, gamma: f64) -> anyhow::Result<()> { - let (samples, inputs) = (20, features.inputs()); - let n = samples * inputs; - +pub fn sample_descent( + epochs: usize, + features: Features, + gamma: f64, + samples: usize, +) -> anyhow::Result<()> { // Generate some example data - let x = Array::linspace(1., n as f64, n) - .into_shape((samples, inputs)) - .unwrap(); - let y = Array::linspace(1., samples as f64, samples) - .into_shape(samples) - .unwrap(); + let x = linarr((samples, features.inputs()))?; + let y = linarr(samples)?; let model = Linear::new(features.inputs()).init_weight(); println!( @@ -64,14 +60,17 @@ pub fn sample_descent(epochs: usize, features: Features, gamma: f64) -> anyhow:: Ok(()) } -pub fn sample_gradient(epochs: usize, features: Features, gamma: f64) -> anyhow::Result<()> { - let (samples, inputs) = (20, features.inputs()); - +pub fn sample_gradient( + epochs: usize, + features: Features, + gamma: f64, + samples: usize, +) -> anyhow::Result<()> { // Generate some example data - let x = linarr((samples, inputs))?; + let x = linarr((samples, features.inputs()))?; let y = linarr((samples, features.outputs()))?; - let mut model = Layer::::new_input(features).init(false); + let mut model = Layer::::input(features).init(false); println!( "Targets:\n\n{:?}\nPredictions:\n\n{:?}\n", &y, diff --git a/ml/optim/examples/sgd.rs b/ml/optim/examples/sgd.rs index 43a52731..325fc957 100644 --- a/ml/optim/examples/sgd.rs +++ b/ml/optim/examples/sgd.rs @@ -1,6 +1,7 @@ +use concision_core::prelude::linarr; use concision_neural::prelude::{Features, Layer, Sigmoid}; use concision_optim::grad::sgd::sgd; -use ndarray::prelude::Array; +use ndarray::prelude::Array2; fn main() -> anyhow::Result<()> { let (samples, inputs) = (20, 10); @@ -8,22 +9,23 @@ fn main() -> anyhow::Result<()> { let features = Features::new(inputs, outputs); - let n = samples * inputs; - let (batch_size, epochs, gamma) = (20, 4, 0.01); // Generate some example data - let base = Array::linspace(1., n as f64, n); - let x = Array::linspace(1., n as f64, n) - .into_shape((samples, inputs)) - .unwrap(); - let y = Array::linspace(1., n as f64, outputs) - .into_shape(outputs) - .unwrap() - + 1.0; + let (x, y) = sample_data::(inputs, outputs, samples)?; - let mut model = Layer::::new_input(features); + let mut model = Layer::::input(features); let cost = sgd(&x, &y, &mut model, epochs, gamma, batch_size).unwrap(); println!("Losses {:?}", cost); Ok(()) } + +fn sample_data( + inputs: usize, + outputs: usize, + samples: usize, +) -> anyhow::Result<(Array2, Array2)> { + let x = linarr((samples, inputs)).unwrap(); // (samples, inputs) + let y = linarr((samples, outputs)).unwrap(); // (samples, outputs) + Ok((x, y)) +} diff --git a/ml/optim/src/cost/mod.rs b/ml/optim/src/cost/mod.rs index f5c919ab..80412fbe 100644 --- a/ml/optim/src/cost/mod.rs +++ b/ml/optim/src/cost/mod.rs @@ -29,29 +29,6 @@ where } pub(crate) mod utils { - - use ndarray::prelude::Array; - use ndarray::Dimension; - use num::{Float, FromPrimitive}; - - pub fn mse<'a, T, D>(pred: &Array, target: &Array) -> T - where - T: Float + FromPrimitive, - D: Dimension, - { - (pred - target) - .mapv(|x| x.powi(2)) - .mean() - .unwrap_or_else(T::zero) - } - - pub fn mae<'a, T, D>(pred: &Array, target: &Array) -> Array - where - T: Float, - D: Dimension, - { - (pred - target).mapv(|x| x.abs()) - } } #[cfg(test)] diff --git a/ml/optim/src/grad/descent.rs b/ml/optim/src/grad/descent.rs index 2adf2249..2c229d3b 100644 --- a/ml/optim/src/grad/descent.rs +++ b/ml/optim/src/grad/descent.rs @@ -2,51 +2,10 @@ Appellation: grad Contrib: FL03 */ -use crate::neural::layers::linear::Linear; -use crate::neural::prelude::{Activate, Biased, Forward, Layer, Weighted}; +use crate::neural::prelude::{Forward, Linear}; use crate::prelude::Norm; -use ndarray::prelude::{Array1, Array2, Axis, NdFloat}; +use ndarray::prelude::{Array1, Array2}; use ndarray_stats::DeviationExt; -use num::{FromPrimitive, Signed}; - -pub fn gradient( - gamma: T, - model: &mut Layer, - data: &Array2, - targets: &Array2, - grad: impl Fn(&Array2) -> Array2, -) -> f64 -where - A: Activate>, - T: FromPrimitive + NdFloat + Signed, -{ - let (samples, _inputs) = data.dim(); - let pred = model.forward(data); - - let ns = T::from(samples).unwrap(); - - let errors = &pred - targets; - // compute the gradient of the objective function w.r.t. the model's weights - let dz = errors * grad(&pred); - // compute the gradient of the objective function w.r.t. the model's weights - let dw = data.t().dot(&dz) / ns; - // compute the gradient of the objective function w.r.t. the model's bias - // let db = dz.sum_axis(Axis(0)) / ns; - // // Apply the gradients to the model's learnable parameters - // model.bias_mut().scaled_add(-gamma, &db); - - model.weights_mut().scaled_add(-gamma, &dw.t()); - - let loss = targets - .mean_sq_err(&model.forward(data)) - .expect("Error when calculating the MSE of the model"); - loss -} - -pub struct GradLayer { - gamma: f64, - layer: Layer, -} #[derive(Clone)] pub struct GradientDescent { diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index 4fc94c4a..bc025791 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -26,9 +26,44 @@ pub struct DescentParams { } pub(crate) mod utils { - // use crate::neural::prelude::{Activate, Layer,}; - use ndarray::prelude::{Array, Array1, Dimension, NdFloat}; - use num::FromPrimitive; + use crate::neural::prelude::{Activate, Biased, Forward, Layer, Parameterized, Weighted}; + use ndarray::prelude::{Array, Array1, Array2, Axis, Dimension, Ix2, NdFloat}; + use ndarray_stats::DeviationExt; + use num::{FromPrimitive, Signed}; + + pub fn gradient( + gamma: T, + model: &mut Layer, + data: &Array2, + targets: &Array2, + grad: impl Fn(&Array2) -> Array2, + ) -> f64 + where + A: Activate, + T: FromPrimitive + NdFloat + Signed, + { + let (samples, _inputs) = data.dim(); + let pred = model.forward(data); + + let ns = T::from(samples).unwrap(); + + let errors = &pred - targets; + // compute the gradient of the objective function w.r.t. the model's weights + let dz = errors * grad(&pred); + // compute the gradient of the objective function w.r.t. the model's weights + let dw = data.t().dot(&dz) / ns; + // compute the gradient of the objective function w.r.t. the model's bias + // let db = dz.sum_axis(Axis(0)) / ns; + // // Apply the gradients to the model's learnable parameters + // model.params_mut().bias_mut().scaled_add(-gamma, &db.t()); + + model.params_mut().weights_mut().scaled_add(-gamma, &dw.t()); + + let loss = targets + .mean_sq_err(&model.forward(data)) + .expect("Error when calculating the MSE of the model"); + loss + } pub fn gradient_descent( params: &mut Array, @@ -64,8 +99,10 @@ pub(crate) mod utils { mod tests { use super::*; - use crate::neural::prelude::{Features, Layer, LinearActivation, Weighted}; - use ndarray::prelude::Array2; + use crate::core::prelude::linarr; + use crate::neural::func::activate::{LinearActivation, Sigmoid}; + use crate::neural::prelude::{Features, Layer, Parameterized, Weighted}; + use ndarray::prelude::{Array1, Array2}; fn test_grad(args: &Array2) -> Array2 { args.clone() @@ -82,7 +119,36 @@ mod tests { let mut model = Layer::::from(features).init(true); - let losses = gradient_descent(&mut model.weights_mut(), epochs, gamma, test_grad); + let losses = gradient_descent( + &mut model.params_mut().weights_mut(), + epochs, + gamma, + test_grad, + ); + assert_eq!(losses.len(), epochs); + } + + #[test] + fn test_gradient() { + let (samples, inputs) = (20, 5); + let outputs = 1; + + let (epochs, gamma) = (10, 0.001); + + let features = Features::new(inputs, outputs); + + // Generate some example data + let x = linarr((samples, features.inputs())).unwrap(); + let y = linarr((samples, features.outputs())).unwrap(); + + let mut model = Layer::::input(features).init(true); + + let mut losses = Array1::zeros(epochs); + for e in 0..epochs { + let cost = gradient(gamma, &mut model, &x, &y, Sigmoid::gradient); + losses[e] = cost; + } assert_eq!(losses.len(), epochs); + assert!(losses.first() > losses.last()); } } diff --git a/ml/optim/src/grad/sgd.rs b/ml/optim/src/grad/sgd.rs index 930b3857..849ad172 100644 --- a/ml/optim/src/grad/sgd.rs +++ b/ml/optim/src/grad/sgd.rs @@ -5,22 +5,23 @@ //! # Stochastic Gradient Descent (SGD) //! //! -use crate::neural::prelude::{mse, Activate, Forward, Layer, Parameterized, Params, Weighted}; +use crate::neural::prelude::{Activate, Forward, Layer, Parameterized, Weighted}; // use crate::prelude::ObjectiveFn; -use ndarray::prelude::{s, Array1, Array2, Axis, NdFloat}; -use num::{Float, FromPrimitive}; +use ndarray::prelude::{s, Array1, Array2, Axis, Ix2, NdFloat}; +use ndarray_stats::DeviationExt; +use num::{Float, FromPrimitive, Signed}; use rand::seq::SliceRandom; pub fn sgd( x: &Array2, - y: &Array1, + y: &Array2, model: &mut Layer, epochs: usize, learning_rate: f64, batch_size: usize, ) -> anyhow::Result> where - A: Activate> + Clone, + A: Clone + Activate, { let layer = model.clone(); let features = layer.features(); @@ -52,21 +53,24 @@ where .to_owned(); // (1, inputs) let prediction = model.forward(&input); // (1, outputs) - let inner = y[idx] - &prediction; + let inner = y - &prediction; let partial_w = (-2.0 / batch_size as f64) * input.dot(&inner); let partial_b = (-2.0 / batch_size as f64) * inner; gradient -= partial_w.sum(); // let mut weights = model.weights_mut().slice_mut(s![]) // model.set_weights(weights) - let cost = mse(&prediction, y).unwrap(); + let cost = y.mean_sq_err(&prediction)?; losses[epoch] += cost; // let error = &prediction - y[idx]; println!("Cost:\t{:?}", &cost); // gradient += &(input * cost); } gradient /= batch_size as f64; - model.weights_mut().scaled_add(-learning_rate, &gradient); + model + .params_mut() + .weights_mut() + .scaled_add(-learning_rate, &gradient.t()); println!("Gradient:\n{:?}", &gradient); } @@ -84,7 +88,7 @@ pub fn sgd_step( batch_size: usize, ) -> anyhow::Result where - A: Activate> + Clone, + A: Clone + Activate, { let layer = model.clone(); let features = layer.features(); @@ -167,9 +171,9 @@ where impl StochasticGradientDescent where - T: Default + FromPrimitive + NdFloat, + T: Default + FromPrimitive + NdFloat + Signed, { - pub fn sgd(&mut self, x: &Array2, y: &Array1) -> Array1 { + pub fn sgd(&mut self, x: &Array2, y: &Array2) -> Array1 { let (samples, inputs) = x.dim(); let mut indices: Vec = (0..samples).collect(); let mut losses = Array1::::zeros(self.epochs); @@ -197,7 +201,7 @@ where self.model.update_with_gradient(self.gamma, &gradient); println!("Gradient:\n{:?}", &gradient); - let loss = mse(&self.model.forward(x), y).unwrap(); + let loss = y.mean_sq_err(&self.model.forward(x)).unwrap(); println!("Epoch: {:?}\nLoss:\n{:?}", &epoch, &loss); losses[epoch] += gradient.mean().unwrap_or_default(); } @@ -224,12 +228,14 @@ mod tests { let (batch_size, epochs, gamma) = (10, 1, 0.01); // Generate some example data let x = Array::linspace(1., 100., 100).into_shape(shape).unwrap(); - let y = Array::linspace(1., 100., 5).into_shape(5).unwrap(); + let y = Array::linspace(1., 100., samples) + .into_shape(samples) + .unwrap(); let mut model = Layer::::hidden(features, 5).init(true); // let mut sgd = StochasticGradientDescent::new(batch_size, epochs, gamma, model); // sgd.sgd(&x, &y); - let sgd = sgd(&x, &y, &mut model, epochs, gamma, batch_size).unwrap(); + // let sgd = sgd(&x, &y, &mut model, epochs, gamma, batch_size).unwrap(); } } diff --git a/ml/optim/src/optimize/mod.rs b/ml/optim/src/optimize/mod.rs index 03482965..721b017d 100644 --- a/ml/optim/src/optimize/mod.rs +++ b/ml/optim/src/optimize/mod.rs @@ -9,6 +9,7 @@ pub use self::{optimizer::*, utils::*}; pub(crate) mod optimizer; pub trait Optimize { + // fn params(&self) -> &Params; fn optimize(&self) -> Self; } diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index 9acec789..a99e7cf9 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -70,7 +70,7 @@ where // compute the attention score let inner = (q.dot(&k.t()) + self.mask.clone()) * self.scale(); - Softmax::default().activate(inner).dot(&v) + Softmax::default().activate(&inner).dot(&v) } } diff --git a/ml/transformers/src/attention/mod.rs b/ml/transformers/src/attention/mod.rs index 472915ae..9af18a87 100644 --- a/ml/transformers/src/attention/mod.rs +++ b/ml/transformers/src/attention/mod.rs @@ -92,10 +92,9 @@ pub(crate) mod utils { let (seq, dk) = query.dim(); let mask = mask.unwrap_or_else(|| Array2::::zeros((seq, seq))); let scale = T::one() / (T::from(dk).unwrap()).sqrt(); - let softmax = Softmax::new(Some(1)); - softmax - .activate((query.dot(&key.t()) + mask) * scale) - .dot(value) + let score = (query.dot(&key.t()) + mask) * scale; + let pred = Softmax::new(Some(1)).activate(&score); + pred.dot(value) } } diff --git a/ml/transformers/src/attention/multi/attention.rs b/ml/transformers/src/attention/multi/attention.rs index ebbe2857..c6981710 100644 --- a/ml/transformers/src/attention/multi/attention.rs +++ b/ml/transformers/src/attention/multi/attention.rs @@ -44,7 +44,7 @@ where let params = MultiHeadParams::new(heads, model); let weights = Weight::uniform((model, model)); Self { - linear: Layer::new_input((model, model).into()), + linear: Layer::input((model, model).into()), params, weights, } diff --git a/ml/transformers/src/ffn/network.rs b/ml/transformers/src/ffn/network.rs index 6573767a..73c095df 100644 --- a/ml/transformers/src/ffn/network.rs +++ b/ml/transformers/src/ffn/network.rs @@ -21,7 +21,7 @@ impl FFN { let features = Features::new(model, params.network_size()); Self { - input: Layer::new_input(features), + input: Layer::input(features), output: Layer::output(features, 1), params, } @@ -33,6 +33,6 @@ impl Forward> for FFN { fn forward(&self, data: &Array2) -> Self::Output { self.output - .forward(&Activate::activate(&ReLU, self.input.forward(data))) + .forward(&ReLU::default().activate(&self.input.forward(data))) } } From e1e88a4d0077101c96db5d89cd0d28c43ccb7492 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 18 Nov 2023 13:05:31 -0600 Subject: [PATCH 058/118] update Signed-off-by: FL03 --- ml/neural/src/func/activate/linear.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ml/neural/src/func/activate/linear.rs b/ml/neural/src/func/activate/linear.rs index bca574a5..d194b1af 100644 --- a/ml/neural/src/func/activate/linear.rs +++ b/ml/neural/src/func/activate/linear.rs @@ -4,6 +4,7 @@ */ use super::{Activate, ActivateMethod, ActivationFn}; use ndarray::prelude::{Array, Dimension}; +use num::One; use serde::{Deserialize, Serialize}; #[derive( @@ -12,6 +13,25 @@ use serde::{Deserialize, Serialize}; pub struct LinearActivation; impl LinearActivation { + pub fn new() -> Self { + Self::default() + } + + pub fn derivative(_x: T) -> T + where + T: One, + { + T::one() + } + + pub fn gradient(args: &Array) -> Array + where + D: Dimension, + T: Clone + One, + { + args.mapv(|x| Self::derivative(x)) + } + pub fn method() -> ActivationFn { |x| x } From 3710a7e5073194d429f62a1c4f4d9f5f37cfc879 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 18 Nov 2023 13:18:39 -0600 Subject: [PATCH 059/118] Update neural network module Signed-off-by: FL03 --- core/src/lib.rs | 2 -- ml/neural/src/layers/params.rs | 2 +- ml/neural/src/nn/deep.rs | 2 +- ml/neural/src/nn/mod.rs | 3 ++- ml/neural/src/nn/shallow.rs | 47 ++++++++++++++++++++++++++++++++++ ml/neural/src/params/group.rs | 7 +---- ml/optim/examples/descent.rs | 2 +- ml/optim/src/cost/mod.rs | 3 +-- ml/optim/src/grad/mod.rs | 10 ++++---- 9 files changed, 59 insertions(+), 19 deletions(-) create mode 100644 ml/neural/src/nn/shallow.rs diff --git a/core/src/lib.rs b/core/src/lib.rs index a4ee39f3..66b3a49c 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -14,8 +14,6 @@ pub mod errors; pub mod states; pub mod step; - - pub mod prelude { pub use crate::epochs::*; pub use crate::errors::*; diff --git a/ml/neural/src/layers/params.rs b/ml/neural/src/layers/params.rs index 599df324..e864495e 100644 --- a/ml/neural/src/layers/params.rs +++ b/ml/neural/src/layers/params.rs @@ -4,7 +4,7 @@ */ use super::Features; use crate::core::prelude::GenerateRandom; -use crate::prelude::{Biased, Params, Weighted}; +use crate::prelude::{Biased, Weighted}; use ndarray::prelude::{Array1, Array2, Ix2}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; diff --git a/ml/neural/src/nn/deep.rs b/ml/neural/src/nn/deep.rs index 60bd57af..f6751cfc 100644 --- a/ml/neural/src/nn/deep.rs +++ b/ml/neural/src/nn/deep.rs @@ -2,7 +2,7 @@ Appellation: network Contrib: FL03 */ -use crate::func::activate::{Activate, ActivateMethod, LinearActivation}; +use crate::func::activate::{Activate, LinearActivation}; use crate::prelude::{Forward, Layer, Parameterized}; use ndarray::prelude::{Array2, Ix2, NdFloat}; diff --git a/ml/neural/src/nn/mod.rs b/ml/neural/src/nn/mod.rs index b2608fbb..bfd605f1 100644 --- a/ml/neural/src/nn/mod.rs +++ b/ml/neural/src/nn/mod.rs @@ -3,9 +3,10 @@ Contrib: FL03 */ //! # Neural Network -pub use self::{deep::*, utils::*}; +pub use self::{deep::*, shallow::*, utils::*}; pub(crate) mod deep; +pub(crate) mod shallow; use crate::layers::Layer; use crate::Trainable; diff --git a/ml/neural/src/nn/shallow.rs b/ml/neural/src/nn/shallow.rs new file mode 100644 index 00000000..60e57e88 --- /dev/null +++ b/ml/neural/src/nn/shallow.rs @@ -0,0 +1,47 @@ +/* + Appellation: network + Contrib: FL03 +*/ +use crate::func::activate::{Activate, LinearActivation}; +use crate::prelude::{Forward, Layer, Parameterized}; + +use ndarray::prelude::{Array2, Ix2, NdFloat}; +use num::Float; + +pub struct ShallowNetwork +where + T: Float, + I: Activate, + O: Activate, +{ + pub input: Layer, + pub output: Layer, +} + +impl ShallowNetwork +where + T: Float, + I: Activate, + O: Activate, +{ + pub fn new(input: Layer, output: Layer) -> Self { + Self { input, output } + } + + pub fn validate_dims(&self) -> bool { + self.input.features().outputs() == self.output.features().inputs() + } +} + +impl Forward> for ShallowNetwork +where + T: NdFloat, + I: Activate, + O: Activate, +{ + type Output = Array2; + + fn forward(&self, args: &Array2) -> Self::Output { + self.output.forward(&self.input.forward(args)) + } +} diff --git a/ml/neural/src/params/group.rs b/ml/neural/src/params/group.rs index ae6d8bd1..419c5406 100644 --- a/ml/neural/src/params/group.rs +++ b/ml/neural/src/params/group.rs @@ -2,13 +2,8 @@ Appellation: group Contrib: FL03 */ -use crate::core::prelude::GenerateRandom; -use crate::layers::Features; -use ndarray::prelude::{Array, Array1, Array2, Dimension, Ix2}; -use ndarray::{IntoDimension, RemoveAxis}; -use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray::prelude::{Array, Dimension, Ix2}; use num::Float; -use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Eq, PartialEq)] pub struct ParamGroup diff --git a/ml/optim/examples/descent.rs b/ml/optim/examples/descent.rs index 6537a917..2f8c0123 100644 --- a/ml/optim/examples/descent.rs +++ b/ml/optim/examples/descent.rs @@ -10,7 +10,7 @@ fn main() -> anyhow::Result<()> { let features = Features::new(inputs, outputs); - let (epochs, gamma) = (100000, 0.005); + let (epochs, gamma) = (1000000, 0.05); // basic_descent(epochs, features, gamma)?; diff --git a/ml/optim/src/cost/mod.rs b/ml/optim/src/cost/mod.rs index 80412fbe..5c2cbfb2 100644 --- a/ml/optim/src/cost/mod.rs +++ b/ml/optim/src/cost/mod.rs @@ -28,8 +28,7 @@ where fn cost(&self, pred: &Array, target: &Array1) -> Array; } -pub(crate) mod utils { -} +pub(crate) mod utils {} #[cfg(test)] mod tests {} diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index bc025791..54b99a59 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -26,8 +26,8 @@ pub struct DescentParams { } pub(crate) mod utils { - use crate::neural::prelude::{Activate, Biased, Forward, Layer, Parameterized, Weighted}; - use ndarray::prelude::{Array, Array1, Array2, Axis, Dimension, Ix2, NdFloat}; + use crate::neural::prelude::{Activate, Forward, Layer, Parameterized, Weighted}; + use ndarray::prelude::{Array, Array1, Array2, Dimension, Ix2, NdFloat}; use ndarray_stats::DeviationExt; use num::{FromPrimitive, Signed}; @@ -42,14 +42,14 @@ pub(crate) mod utils { A: Activate, T: FromPrimitive + NdFloat + Signed, { - let (samples, _inputs) = data.dim(); + let (_samples, _inputs) = data.dim(); let pred = model.forward(data); - let ns = T::from(samples).unwrap(); + let ns = T::from(data.len()).unwrap(); let errors = &pred - targets; // compute the gradient of the objective function w.r.t. the model's weights - let dz = errors * grad(&pred); + let dz = &errors * grad(&pred); // compute the gradient of the objective function w.r.t. the model's weights let dw = data.t().dot(&dz) / ns; // compute the gradient of the objective function w.r.t. the model's bias From 3ebd06452b6852db862d3453dfe69bcc7687d95c Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 19 Nov 2023 11:24:00 -0600 Subject: [PATCH 060/118] update Signed-off-by: FL03 --- .gitignore | 2 + concision/Cargo.toml | 2 + .../examples/gradients.rs | 28 +-- core/src/lib.rs | 1 + core/src/primitives.rs | 2 + ml/neural/src/{nn => arch}/deep.rs | 0 ml/neural/src/arch/mod.rs | 31 ++- ml/neural/src/{nn => arch}/shallow.rs | 10 +- ml/neural/src/func/activate/activator.rs | 14 +- ml/neural/src/func/activate/binary.rs | 8 +- ml/neural/src/func/activate/linear.rs | 17 +- ml/neural/src/func/activate/mod.rs | 19 +- ml/neural/src/func/activate/nonlinear.rs | 27 +-- ml/neural/src/layers/features.rs | 100 +++++++--- ml/neural/src/layers/kinds.rs | 38 ++-- ml/neural/src/layers/layer.rs | 76 ++++++-- ml/neural/src/layers/linear.rs | 176 ------------------ ml/neural/src/layers/mod.rs | 4 +- ml/neural/src/layers/params.rs | 10 +- ml/neural/src/lib.rs | 1 - ml/neural/src/models/{params.rs => config.rs} | 0 ml/neural/src/models/mod.rs | 6 +- ml/neural/src/models/model.rs | 21 +-- ml/neural/src/models/stack.rs | 172 ++++++++++++++++- ml/neural/src/neurons/mod.rs | 13 +- ml/neural/src/neurons/neuron.rs | 64 ++++--- ml/neural/src/neurons/node.rs | 152 +++++++++++++-- ml/neural/src/neurons/synapse.rs | 9 + ml/neural/src/nn/mod.rs | 22 ++- ml/neural/src/nn/position.rs | 6 + ml/neural/src/nn/sequential.rs | 12 ++ ml/neural/src/ops/dropout.rs | 22 ++- ml/optim/examples/norm.rs | 6 +- ml/optim/examples/sgd.rs | 4 +- ml/optim/src/grad/descent.rs | 16 +- ml/optim/src/grad/mod.rs | 6 +- ml/optim/src/grad/sgd.rs | 4 +- ml/optim/src/optimize/optimizer.rs | 23 +++ ml/optim/src/specs.rs | 26 +-- ml/transformers/src/attention/head.rs | 63 ++++--- ml/transformers/src/attention/mod.rs | 52 ++++-- ml/transformers/src/attention/multi/mod.rs | 6 +- ml/transformers/src/attention/params/dim.rs | 59 +++--- ml/transformers/src/ffn/network.rs | 4 +- 44 files changed, 837 insertions(+), 497 deletions(-) rename ml/optim/examples/descent.rs => concision/examples/gradients.rs (71%) rename ml/neural/src/{nn => arch}/deep.rs (100%) rename ml/neural/src/{nn => arch}/shallow.rs (86%) delete mode 100644 ml/neural/src/layers/linear.rs rename ml/neural/src/models/{params.rs => config.rs} (100%) create mode 100644 ml/neural/src/neurons/synapse.rs create mode 100644 ml/neural/src/nn/position.rs create mode 100644 ml/neural/src/nn/sequential.rs diff --git a/.gitignore b/.gitignore index adad90a9..93f115b6 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,5 @@ !**/*.env.example !**/*.config.js !**/*.config.cjs + +!**/config.rs diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 2bdfabb2..4df1499a 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -93,6 +93,8 @@ concision-optim = { features = [], optional = true, path = "../ml/optim", versio concision-transformers = { features = [], optional = true, path = "../ml/transformers", version = "0.1.12" } [dev-dependencies] +anyhow.workspace = true +ndarray.workspace = true [package.metadata.docs.rs] all-features = true diff --git a/ml/optim/examples/descent.rs b/concision/examples/gradients.rs similarity index 71% rename from ml/optim/examples/descent.rs rename to concision/examples/gradients.rs index 2f8c0123..46ed4803 100644 --- a/ml/optim/examples/descent.rs +++ b/concision/examples/gradients.rs @@ -1,14 +1,15 @@ -use concision_core::prelude::linarr; -use concision_neural::prelude::{Features, Layer, Linear, LinearActivation, Sigmoid}; -use concision_neural::prop::Forward; -use concision_optim::prelude::{gradient, gradient_descent, GradientDescent}; +use concision::prelude::{linarr, Forward, LayerShape, Weighted}; + +use concision::neural::prelude::{Layer, Neuron, Sigmoid}; +use concision::optim::grad::*; + use ndarray::prelude::Array1; fn main() -> anyhow::Result<()> { let (samples, inputs) = (20, 8); let outputs = 4; - let features = Features::new(inputs, outputs); + let features = LayerShape::new(inputs, outputs); let (epochs, gamma) = (1000000, 0.05); @@ -21,8 +22,8 @@ fn main() -> anyhow::Result<()> { Ok(()) } -pub fn basic_descent(epochs: usize, features: Features, gamma: f64) -> anyhow::Result<()> { - let mut model = Linear::new(features.inputs()).init_weight(); +pub fn basic_descent(epochs: usize, features: LayerShape, gamma: f64) -> anyhow::Result<()> { + let mut model = Neuron::::new(features.inputs()).init_weight(); println!( "{:?}", @@ -33,7 +34,7 @@ pub fn basic_descent(epochs: usize, features: Features, gamma: f64) -> anyhow::R pub fn sample_descent( epochs: usize, - features: Features, + features: LayerShape, gamma: f64, samples: usize, ) -> anyhow::Result<()> { @@ -41,7 +42,7 @@ pub fn sample_descent( let x = linarr((samples, features.inputs()))?; let y = linarr(samples)?; - let model = Linear::new(features.inputs()).init_weight(); + let model = Neuron::new(features.inputs()).init_weight(); println!( "Targets:\n\n{:?}\nPredictions:\n\n{:?}\n", &y, @@ -55,14 +56,13 @@ pub fn sample_descent( losses[e] = cost; } println!("Losses:\n\n{:?}\n", &losses); - println!("Trained:\n\n{:?}", grad.model().forward(&x)); Ok(()) } pub fn sample_gradient( epochs: usize, - features: Features, + features: LayerShape, gamma: f64, samples: usize, ) -> anyhow::Result<()> { @@ -70,10 +70,10 @@ pub fn sample_gradient( let x = linarr((samples, features.inputs()))?; let y = linarr((samples, features.outputs()))?; - let mut model = Layer::::input(features).init(false); + let mut model = Layer::::input(features).init(false); println!( - "Targets:\n\n{:?}\nPredictions:\n\n{:?}\n", - &y, + "Targets (dim):\t{:?}\nPredictions:\n\n{:?}\n", + &y.shape(), model.forward(&x) ); diff --git a/core/src/lib.rs b/core/src/lib.rs index 66b3a49c..d86f998f 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -15,6 +15,7 @@ pub mod states; pub mod step; pub mod prelude { + pub use crate::epochs::*; pub use crate::errors::*; pub use crate::states::*; diff --git a/core/src/primitives.rs b/core/src/primitives.rs index cf61ef2d..540dcf9a 100644 --- a/core/src/primitives.rs +++ b/core/src/primitives.rs @@ -4,6 +4,8 @@ */ pub use self::{constants::*, statics::*, types::*}; +pub use ndarray_rand::rand_distr::uniform::SampleUniform; + /// Collection of constants used throughout the system mod constants {} diff --git a/ml/neural/src/nn/deep.rs b/ml/neural/src/arch/deep.rs similarity index 100% rename from ml/neural/src/nn/deep.rs rename to ml/neural/src/arch/deep.rs diff --git a/ml/neural/src/arch/mod.rs b/ml/neural/src/arch/mod.rs index d92c10b6..11d37c48 100644 --- a/ml/neural/src/arch/mod.rs +++ b/ml/neural/src/arch/mod.rs @@ -5,10 +5,39 @@ //! # Architecture //! //! This module describes the architecture of various components of the neural network. -pub use self::{architecture::*, utils::*}; +pub use self::{architecture::*, deep::*, shallow::*, utils::*}; pub(crate) mod architecture; +pub(crate) mod deep; +pub(crate) mod shallow; pub trait Arch {} pub(crate) mod utils {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::models::stack::Stack; + use crate::prelude::{Activate, Layer, LayerShape, Softmax}; + + fn _stack + Default>( + shapes: impl IntoIterator, + ) -> Stack { + let mut stack = Stack::new(); + for (inputs, outputs) in shapes.into_iter() { + stack.push(Layer::::from(LayerShape::new(inputs, outputs)).init(true)); + } + stack + } + + #[test] + fn test_arch() { + assert!(true); + } + + #[test] + fn test_deep_network() { + assert!(true); + } +} diff --git a/ml/neural/src/nn/shallow.rs b/ml/neural/src/arch/shallow.rs similarity index 86% rename from ml/neural/src/nn/shallow.rs rename to ml/neural/src/arch/shallow.rs index 60e57e88..20c59733 100644 --- a/ml/neural/src/nn/shallow.rs +++ b/ml/neural/src/arch/shallow.rs @@ -1,5 +1,5 @@ /* - Appellation: network + Appellation: shallow Contrib: FL03 */ use crate::func::activate::{Activate, LinearActivation}; @@ -28,6 +28,14 @@ where Self { input, output } } + pub fn input(&self) -> &Layer { + &self.input + } + + pub fn output(&self) -> &Layer { + &self.output + } + pub fn validate_dims(&self) -> bool { self.input.features().outputs() == self.output.features().inputs() } diff --git a/ml/neural/src/func/activate/activator.rs b/ml/neural/src/func/activate/activator.rs index 4604558f..948d402a 100644 --- a/ml/neural/src/func/activate/activator.rs +++ b/ml/neural/src/func/activate/activator.rs @@ -2,24 +2,18 @@ Appellation: activator Contrib: FL03 */ -use super::ActivateMethod; +use super::Activate; pub struct Activator { - method: Box>, + method: Box>, } impl Activator { - pub fn new(method: Box>) -> Self { + pub fn new(method: Box>) -> Self { Self { method } } - pub fn method(&self) -> &dyn ActivateMethod { + pub fn method(&self) -> &dyn Activate { self.method.as_ref() } } - -impl ActivateMethod for Activator { - fn rho(&self, x: T) -> T { - self.method().rho(x) - } -} diff --git a/ml/neural/src/func/activate/binary.rs b/ml/neural/src/func/activate/binary.rs index 0040a0e4..0c9f514d 100644 --- a/ml/neural/src/func/activate/binary.rs +++ b/ml/neural/src/func/activate/binary.rs @@ -2,7 +2,7 @@ Appellation: binary Contrib: FL03 */ -use super::ActivateMethod; +use super::Activate; use ndarray::prelude::{Array, Dimension}; use num::{One, Zero}; use serde::{Deserialize, Serialize}; @@ -25,12 +25,12 @@ impl Heavyside { } } -impl ActivateMethod> for Heavyside +impl Activate for Heavyside where D: Dimension, T: Clone + One + PartialOrd + Zero, { - fn rho(&self, x: Array) -> Array { - x.mapv(|x| Self::heavyside(x)) + fn activate(&self, args: &Array) -> Array { + args.mapv(|x| Self::heavyside(x)) } } diff --git a/ml/neural/src/func/activate/linear.rs b/ml/neural/src/func/activate/linear.rs index d194b1af..a473f76c 100644 --- a/ml/neural/src/func/activate/linear.rs +++ b/ml/neural/src/func/activate/linear.rs @@ -32,6 +32,10 @@ impl LinearActivation { args.mapv(|x| Self::derivative(x)) } + pub fn linear(args: &T) -> T { + args.clone() + } + pub fn method() -> ActivationFn { |x| x } @@ -41,18 +45,21 @@ impl LinearActivation { } } -impl ActivateMethod for LinearActivation { - fn rho(&self, x: T) -> T { - Self::method()(x) +impl ActivateMethod for LinearActivation +where + T: Clone, +{ + fn rho(&self, x: &T) -> T { + Self::linear(x) } } impl Activate for LinearActivation where D: Dimension, - T: num::Float, + T: Clone, { fn activate(&self, args: &Array) -> Array { - args.mapv(|x| self.rho(x)) + args.mapv(|x| Self::linear(&x)) } } diff --git a/ml/neural/src/func/activate/mod.rs b/ml/neural/src/func/activate/mod.rs index 15d3c4e6..b2a59bf1 100644 --- a/ml/neural/src/func/activate/mod.rs +++ b/ml/neural/src/func/activate/mod.rs @@ -14,20 +14,23 @@ pub(crate) mod nonlinear; pub type ActivationFn = fn(T) -> T; -pub type BoxedActivation = Box>; +pub type BoxedActivation = Box; use ndarray::prelude::{Array, Dimension, Ix2}; use num::Float; +pub trait Activation { + fn activate(&self, args: &Array) -> Array; +} + pub trait Activate where D: Dimension, - T: Float, { fn activate(&self, args: &Array) -> Array; } -pub trait RhoGradient: Activate +pub trait ActivateExt: Activate where D: Dimension, T: Float, @@ -57,19 +60,15 @@ where // } // } -pub trait ActivationMethod { - fn method_name(&self) -> &str; -} - pub trait ActivateMethod { - fn rho(&self, x: T) -> T; + fn rho(&self, x: &T) -> T; } impl ActivateMethod for F where - F: Fn(T) -> T, + F: Fn(&T) -> T, { - fn rho(&self, x: T) -> T { + fn rho(&self, x: &T) -> T { self.call((x,)) } } diff --git a/ml/neural/src/func/activate/nonlinear.rs b/ml/neural/src/func/activate/nonlinear.rs index 62330537..8d2fdce7 100644 --- a/ml/neural/src/func/activate/nonlinear.rs +++ b/ml/neural/src/func/activate/nonlinear.rs @@ -61,12 +61,12 @@ impl ReLU { args.mapv(|x| Self::derivative(x)) } - pub fn relu(args: T) -> T + pub fn relu(args: &T) -> T where - T: PartialOrd + Zero, + T: Clone + PartialOrd + Zero, { - if args > T::zero() { - args + if args > &T::zero() { + args.clone() } else { T::zero() } @@ -75,9 +75,9 @@ impl ReLU { impl ActivateMethod for ReLU where - T: PartialOrd + Zero, + T: Clone + PartialOrd + Zero, { - fn rho(&self, x: T) -> T { + fn rho(&self, x: &T) -> T { Self::relu(x) } } @@ -85,10 +85,10 @@ where impl Activate for ReLU where D: Dimension, - T: Float, + T: Clone + PartialOrd + Zero, { fn activate(&self, x: &Array) -> Array { - x.mapv(|x| Self::relu(x)) + x.mapv(|x| Self::relu(&x)) } } #[derive( @@ -124,15 +124,6 @@ impl Sigmoid { } } -impl ActivateMethod for Sigmoid -where - T: Float, -{ - fn rho(&self, x: T) -> T { - Self::sigmoid(x) - } -} - impl Activate for Sigmoid where D: Dimension, @@ -238,7 +229,7 @@ impl ActivateMethod for Tanh where T: Float, { - fn rho(&self, x: T) -> T { + fn rho(&self, x: &T) -> T { x.tanh() } } diff --git a/ml/neural/src/layers/features.rs b/ml/neural/src/layers/features.rs index ffcf0b2c..8153e590 100644 --- a/ml/neural/src/layers/features.rs +++ b/ml/neural/src/layers/features.rs @@ -2,26 +2,74 @@ Appellation: features Contrib: FL03 */ +use ndarray::prelude::Ix2; use ndarray::IntoDimension; use serde::{Deserialize, Serialize}; -pub trait FromFeatures { - fn from_features(features: Features) -> Self; +pub trait Features { + fn inputs(&self) -> usize; + + fn outputs(&self) -> usize; + + fn in_by_out(&self) -> (usize, usize) { + (self.inputs(), self.outputs()) + } + + fn out_by_in(&self) -> (usize, usize) { + (self.outputs(), self.inputs()) + } + + fn input_scale(&self) -> T { + (T::one() / T::from(self.inputs()).unwrap()).sqrt() + } +} + +pub trait FeaturesExt: Features + IntoDimension { + fn new(inputs: usize, outputs: usize) -> Self; + + fn single(inputs: usize) -> Self + where + Self: Sized, + { + Self::new(inputs, 1) + } +} + +// impl FeaturesExt for T +// where +// T: Features + IntoDimension, +// { +// fn new(inputs: usize, outputs: usize) -> Self { +// Self::from_dimension(ndarray::Ix2(outputs, inputs)) +// } +// } + +pub trait FromFeatures { + fn from_features(features: LayerShape) -> Self; } pub trait IntoFeatures { - fn into_features(self) -> Features; + fn into_features(self) -> LayerShape; +} + +impl IntoFeatures for S +where + S: Into, +{ + fn into_features(self) -> LayerShape { + self.into() + } } #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] -pub struct Features { +pub struct LayerShape { pub inputs: usize, pub outputs: usize, } -impl Features { +impl LayerShape { pub fn new(inputs: usize, outputs: usize) -> Self { Self { inputs, outputs } } @@ -51,13 +99,23 @@ impl Features { } } -impl std::fmt::Display for Features { +impl std::fmt::Display for LayerShape { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "({}, {})", self.inputs, self.outputs) } } -impl IntoDimension for Features { +impl Features for LayerShape { + fn inputs(&self) -> usize { + self.inputs + } + + fn outputs(&self) -> usize { + self.outputs + } +} + +impl IntoDimension for LayerShape { type Dim = ndarray::Ix2; fn into_dimension(self) -> Self::Dim { @@ -65,46 +123,46 @@ impl IntoDimension for Features { } } -impl From for ndarray::Ix2 { - fn from(features: Features) -> Self { +impl From for ndarray::Ix2 { + fn from(features: LayerShape) -> Self { ndarray::Ix2(features.outputs, features.inputs) } } -impl From for ndarray::IxDyn { - fn from(features: Features) -> Self { +impl From for ndarray::IxDyn { + fn from(features: LayerShape) -> Self { ndarray::IxDyn(&[features.outputs, features.inputs]) } } -impl From for [usize; 2] { - fn from(features: Features) -> Self { +impl From for [usize; 2] { + fn from(features: LayerShape) -> Self { [features.outputs, features.inputs] } } -impl From<[usize; 2]> for Features { +impl From<[usize; 2]> for LayerShape { fn from(features: [usize; 2]) -> Self { Self { - inputs: features[1], - outputs: features[0], + inputs: features[0], + outputs: features[1], } } } -impl From for (usize, usize) { - fn from(features: Features) -> Self { +impl From for (usize, usize) { + fn from(features: LayerShape) -> Self { (features.outputs, features.inputs) } } -impl From<(usize, usize)> for Features { - fn from((outputs, inputs): (usize, usize)) -> Self { +impl From<(usize, usize)> for LayerShape { + fn from((inputs, outputs): (usize, usize)) -> Self { Self { inputs, outputs } } } -impl From for Features { +impl From for LayerShape { fn from(inputs: usize) -> Self { Self { inputs, outputs: 1 } } diff --git a/ml/neural/src/layers/kinds.rs b/ml/neural/src/layers/kinds.rs index 00e312df..dfcbc213 100644 --- a/ml/neural/src/layers/kinds.rs +++ b/ml/neural/src/layers/kinds.rs @@ -26,7 +26,7 @@ use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; #[repr(usize)] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] -pub enum LayerType { +pub enum LayerKind { #[default] Input = 0, Hidden(usize), @@ -36,26 +36,26 @@ pub enum LayerType { #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] -pub struct Position { +pub struct LayerPosition { pub idx: usize, - pub kind: LayerType, + pub kind: LayerKind, } -impl Position { - pub fn new(idx: usize, kind: LayerType) -> Self { +impl LayerPosition { + pub fn new(idx: usize, kind: LayerKind) -> Self { Self { idx, kind } } pub fn input() -> Self { - Self::new(0, LayerType::Input) + Self::new(0, LayerKind::Input) } pub fn hidden(idx: usize) -> Self { - Self::new(idx, LayerType::Hidden(idx)) + Self::new(idx, LayerKind::Hidden(idx)) } pub fn output(idx: usize) -> Self { - Self::new(idx, LayerType::Output) + Self::new(idx, LayerKind::Output) } pub fn is_input(&self) -> bool { @@ -70,7 +70,7 @@ impl Position { self.kind().is_output() } - pub fn kind(&self) -> &LayerType { + pub fn kind(&self) -> &LayerKind { &self.kind } @@ -79,38 +79,38 @@ impl Position { } } -impl AsRef for Position { +impl AsRef for LayerPosition { fn as_ref(&self) -> &usize { &self.idx } } -impl AsRef for Position { - fn as_ref(&self) -> &LayerType { +impl AsRef for LayerPosition { + fn as_ref(&self) -> &LayerKind { &self.kind } } -impl AsMut for Position { +impl AsMut for LayerPosition { fn as_mut(&mut self) -> &mut usize { &mut self.idx } } -impl AsMut for Position { - fn as_mut(&mut self) -> &mut LayerType { +impl AsMut for LayerPosition { + fn as_mut(&mut self) -> &mut LayerKind { &mut self.kind } } -impl From for usize { - fn from(pos: Position) -> Self { +impl From for usize { + fn from(pos: LayerPosition) -> Self { pos.idx } } -impl From for LayerType { - fn from(pos: Position) -> Self { +impl From for LayerKind { + fn from(pos: LayerPosition) -> Self { pos.kind } } diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 7802069d..d961fdf0 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -2,7 +2,7 @@ Appellation: model Contrib: FL03 */ -use super::{Features, LayerParams, LayerType, Position}; +use super::{LayerKind, LayerParams, LayerPosition, LayerShape}; use crate::prelude::{Activate, Forward, LinearActivation, Parameterized, Params}; use ndarray::prelude::{Array2, Ix2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; @@ -16,9 +16,10 @@ where T: Float, { activator: A, - pub features: Features, + pub features: LayerShape, + name: String, params: LayerParams, - position: Position, + position: LayerPosition, } impl Layer @@ -26,25 +27,26 @@ where A: Default + Activate, T: Float, { - pub fn new(features: Features, position: Position) -> Self { + pub fn new(features: LayerShape, position: LayerPosition) -> Self { Self { activator: A::default(), features, + name: String::new(), params: LayerParams::new(features), position, } } - pub fn input(features: Features) -> Self { - Self::new(features, Position::input()) + pub fn input(features: LayerShape) -> Self { + Self::new(features, LayerPosition::input()) } - pub fn hidden(features: Features, index: usize) -> Self { - Self::new(features, Position::hidden(index)) + pub fn hidden(features: LayerShape, index: usize) -> Self { + Self::new(features, LayerPosition::hidden(index)) } - pub fn output(features: Features, index: usize) -> Self { - Self::new(features, Position::output(index)) + pub fn output(features: LayerShape, index: usize) -> Self { + Self::new(features, LayerPosition::output(index)) } } @@ -57,16 +59,50 @@ where &self.activator } - pub fn kind(&self) -> &LayerType { + pub fn kind(&self) -> &LayerKind { self.position().kind() } - pub fn position(&self) -> &Position { + pub fn name(&self) -> &str { + &self.name + } + + pub fn position(&self) -> &LayerPosition { &self.position } - pub fn set_position(&mut self, position: Position) { - self.position = position; + pub fn set_name(&mut self, name: impl ToString) { + self.name = name.to_string(); + } + + pub fn update_position(&mut self, idx: usize, output: bool) { + self.position = if idx == 0 { + LayerPosition::input() + } else if output { + LayerPosition::output(idx) + } else { + LayerPosition::hidden(idx) + }; + } + + pub fn validate_layer(&self, other: &Self) -> bool { + let pos = self + .position() + .position() + .abs_diff(other.position().position()); + if pos == 1 { + if self.position().position() > other.position().position() { + return self.features().inputs() == other.features().outputs(); + } else { + return self.features().outputs() == other.features().inputs(); + } + } + false + } + + pub fn with_name(mut self, name: impl ToString) -> Self { + self.name = name.to_string(); + self } } @@ -118,14 +154,14 @@ where A: Activate, T: Float, { - type Features = Features; + type Features = LayerShape; type Params = LayerParams; - fn features(&self) -> &Features { + fn features(&self) -> &LayerShape { &self.features } - fn features_mut(&mut self) -> &mut Features { + fn features_mut(&mut self) -> &mut LayerShape { &mut self.features } @@ -148,12 +184,12 @@ where } } -impl From for Layer +impl From for Layer where A: Activate + Default, T: Float, { - fn from(features: Features) -> Self { - Self::new(features, Position::input()) + fn from(features: LayerShape) -> Self { + Self::new(features, LayerPosition::input()) } } diff --git a/ml/neural/src/layers/linear.rs b/ml/neural/src/layers/linear.rs deleted file mode 100644 index ba4f9141..00000000 --- a/ml/neural/src/layers/linear.rs +++ /dev/null @@ -1,176 +0,0 @@ -/* - Appellation: regress - Contrib: FL03 -*/ -use crate::core::prelude::GenerateRandom; - -use crate::prelude::Forward; -use ndarray::prelude::{Array1, Array2, Axis, NdFloat}; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_stats::CorrelationExt; -use num::{Float, FromPrimitive}; -use rand::Rng; - -#[derive(Clone, Debug, PartialEq)] -pub struct Linear -where - T: Float, -{ - bias: T, - pub features: usize, - weights: Array1, -} - -impl Linear -where - T: Float, -{ - pub fn new(features: usize) -> Self { - Self { - bias: T::zero(), - features, - weights: Array1::zeros(features), - } - } - - pub fn bias(&self) -> &T { - &self.bias - } - - pub fn bias_mut(&mut self) -> &mut T { - &mut self.bias - } - - pub fn features(&self) -> usize { - self.features - } - - pub fn features_mut(&mut self) -> &mut usize { - &mut self.features - } - - pub fn weights(&self) -> &Array1 { - &self.weights - } - - pub fn weights_mut(&mut self) -> &mut Array1 { - &mut self.weights - } - - pub fn set_bias(&mut self, bias: T) { - self.bias = bias; - } - - pub fn set_features(&mut self, features: usize) { - self.features = features; - } - - pub fn set_weights(&mut self, weights: Array1) { - self.weights = weights; - } - - pub fn with_bias(mut self, bias: T) -> Self { - self.bias = bias; - self - } - - pub fn with_features(mut self, features: usize) -> Self { - self.features = features; - self - } - - pub fn with_weights(mut self, weights: Array1) -> Self { - self.weights = weights; - self - } -} - -impl Linear -where - T: Float + SampleUniform, -{ - pub fn init(mut self, biased: bool) -> Self { - if biased { - self = self.init_bias(); - } - self.init_weight() - } - - pub fn init_bias(mut self) -> Self { - let dk = (T::one() / T::from(self.features).unwrap()).sqrt(); - self.bias = rand::thread_rng().gen_range(-dk..dk); - self - } - - pub fn init_weight(mut self) -> Self { - let features = self.features; - let dk = (T::one() / T::from(features).unwrap()).sqrt(); - self.weights = Array1::uniform_between(dk, features); - self - } -} - -impl Linear -where - T: FromPrimitive + NdFloat, -{ - pub fn fit(&mut self, data: &Array2, _targets: &Array1) -> anyhow::Result<()> { - let _m = data.cov(T::zero())? / data.var_axis(Axis(0), T::zero()); - // let covar = covariance(0.0, x, y); - // self.bias = targets.mean().unwrap_or_default() - m * data.mean().unwrap_or_default(); - // self.weights -= m; - Ok(()) - } - - pub fn predict(&self, data: &Array2) -> Array1 { - data.dot(&self.weights().t()) + *self.bias() - } - - pub fn apply_gradient(&mut self, gamma: T, gradient: G) - where - G: Fn(&Array1) -> Array1, - { - let mut grad = gradient(self.weights()); - grad /= grad.mapv(|ws| ws.powi(2)).sum().sqrt(); - self.weights_mut().scaled_add(-gamma, &grad); - self.weights /= self.weights().mapv(|ws| ws.powi(2)).sum().sqrt(); - } - - pub fn update_with_gradient(&mut self, gamma: T, gradient: &Array1) { - self.weights = &self.weights - gradient * gamma; - } -} - -impl Forward> for Linear -where - T: NdFloat, -{ - type Output = Array1; - - fn forward(&self, data: &Array2) -> Self::Output { - data.dot(&self.weights().t().to_owned()) + *self.bias() - } -} - -impl Forward> for Linear -where - T: NdFloat, -{ - type Output = T; - - fn forward(&self, data: &Array1) -> Self::Output { - data.dot(&self.weights().t().to_owned()) + *self.bias() - } -} - -// impl Forward> for Linear -// where -// D: Dimension, -// Array: Dot, Output = Array>, -// { -// type Output = Array1; - -// fn forward(&self, data: &Array) -> Self::Output { -// data.dot(&self.weights().t().to_owned()) + self.bias().clone() -// } -// } diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index afaf1e0d..7a08a046 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -11,8 +11,6 @@ pub(crate) mod layer; pub(crate) mod params; pub(crate) mod sublayer; -pub mod linear; - use crate::func::activate::Activate; use ndarray::prelude::Ix2; use num::Float; @@ -34,7 +32,7 @@ mod tests { #[test] fn test_layer() { let (samples, inputs, outputs) = (20, 5, 3); - let features = Features::new(inputs, outputs); + let features = LayerShape::new(inputs, outputs); let args = linarr::((samples, inputs)).unwrap(); diff --git a/ml/neural/src/layers/params.rs b/ml/neural/src/layers/params.rs index e864495e..ae843843 100644 --- a/ml/neural/src/layers/params.rs +++ b/ml/neural/src/layers/params.rs @@ -2,7 +2,7 @@ Appellation: params Contrib: FL03 */ -use super::Features; +use super::LayerShape; use crate::core::prelude::GenerateRandom; use crate::prelude::{Biased, Weighted}; use ndarray::prelude::{Array1, Array2, Ix2}; @@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct LayerParams { bias: Array1, - pub features: Features, + pub features: LayerShape, weights: Array2, } @@ -21,7 +21,7 @@ impl LayerParams where T: Float, { - pub fn new(features: Features) -> Self { + pub fn new(features: LayerShape) -> Self { Self { bias: Array1::zeros(features.outputs()), features, @@ -34,11 +34,11 @@ where self.weights = Array2::zeros(self.features.out_by_in()); } - pub fn features(&self) -> &Features { + pub fn features(&self) -> &LayerShape { &self.features } - pub fn features_mut(&mut self) -> &mut Features { + pub fn features_mut(&mut self) -> &mut LayerShape { &mut self.features } diff --git a/ml/neural/src/lib.rs b/ml/neural/src/lib.rs index 06eb494a..7af7ec54 100644 --- a/ml/neural/src/lib.rs +++ b/ml/neural/src/lib.rs @@ -29,7 +29,6 @@ pub(crate) use concision_core as core; pub mod prelude { pub use crate::arch::*; pub use crate::func::{activate::*, loss::*}; - pub use crate::layers::linear::*; pub use crate::layers::*; pub use crate::masks::*; pub use crate::neurons::*; diff --git a/ml/neural/src/models/params.rs b/ml/neural/src/models/config.rs similarity index 100% rename from ml/neural/src/models/params.rs rename to ml/neural/src/models/config.rs diff --git a/ml/neural/src/models/mod.rs b/ml/neural/src/models/mod.rs index 9db033b2..ba5a7901 100644 --- a/ml/neural/src/models/mod.rs +++ b/ml/neural/src/models/mod.rs @@ -4,10 +4,10 @@ */ //! # Model //! -pub use self::{model::*, params::*, utils::*}; +pub use self::{config::*, model::*, utils::*}; +pub(crate) mod config; pub(crate) mod model; -pub(crate) mod params; pub mod stack; @@ -16,6 +16,8 @@ use ndarray::prelude::Array2; pub trait Module { fn add_module(&mut self, module: impl Module); + fn layers(&self) -> &[impl Module]; + fn forward(&self, args: &Array2) -> Array2; } diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs index 8cbdefff..f7a8c8c2 100644 --- a/ml/neural/src/models/model.rs +++ b/ml/neural/src/models/model.rs @@ -2,23 +2,8 @@ Appellation: model Contrib: FL03 */ -use crate::prelude::{Activate, Features, LayerParams, Params}; -use ndarray::prelude::Ix2; -use num::Float; +use super::ModelConfig; -pub struct BaseModel -where - T: Float, -{ - pub features: Features, - activator: Box>, - params: Box>, -} - -pub struct Model { - pub features: Features, - children: Vec>, - layers: usize, - - params: LayerParams, +pub struct Model { + config: ModelConfig, } diff --git a/ml/neural/src/models/stack.rs b/ml/neural/src/models/stack.rs index 360147bd..607a0ca0 100644 --- a/ml/neural/src/models/stack.rs +++ b/ml/neural/src/models/stack.rs @@ -1,28 +1,192 @@ /* - Appellation: model + Appellation: stack Contrib: FL03 */ use crate::layers::Layer; -use crate::prelude::{Activate, LinearActivation}; +use crate::prelude::{Activate, LayerShape, LinearActivation, Parameterized}; use ndarray::prelude::Ix2; use num::Float; use serde::{Deserialize, Serialize}; +/// A [Stack] is a collection of [Layer]s, typically used to construct the hidden +/// layers of a deep neural network. #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Stack where A: Activate, T: Float, { - layers: Vec>, + children: Vec>, } impl Stack where A: Activate + Default, T: Float, +{ + pub fn build_layers(mut self, shapes: impl IntoIterator) -> Self { + for (inputs, outputs) in shapes.into_iter() { + self.children + .push(Layer::::from(LayerShape::new(inputs, outputs))); + } + self + } +} + +impl Stack +where + A: Activate + Clone + Default, + T: Float + crate::core::prelude::SampleUniform, +{ + pub fn init_layers(mut self, biased: bool) -> Self { + self.children + .iter_mut() + .for_each(|l| *l = l.clone().init(biased)); + self + } +} + +impl Stack +where + A: Activate, + T: Float, { pub fn new() -> Self { - Self { layers: Vec::new() } + Self { + children: Vec::new(), + } + } + pub fn with_capacity(depth: usize) -> Self { + Self { + children: Vec::with_capacity(depth), + } + } + + pub fn layers(&self) -> &[Layer] { + &self.children + } + + pub fn layers_mut(&mut self) -> &mut [Layer] { + &mut self.children + } + + pub fn depth(&self) -> usize { + self.children.len() + } + + pub fn validate_shapes(&self) -> bool { + let mut dim = true; + for (i, layer) in self.children[..(self.depth() - 1)].iter().enumerate() { + dim = dim && layer.features().outputs() == self.children[i + 1].features().inputs(); + } + dim + } + + pub fn push(&mut self, layer: Layer) { + self.children.push(layer); + } + + pub fn pop(&mut self) -> Option> { + self.children.pop() + } +} + +impl AsRef<[Layer]> for Stack +where + A: Activate, + T: Float, +{ + fn as_ref(&self) -> &[Layer] { + &self.children + } +} + +impl AsMut<[Layer]> for Stack +where + A: Activate, + T: Float, +{ + fn as_mut(&mut self) -> &mut [Layer] { + &mut self.children + } +} + +impl Default for Stack +where + A: Activate, + T: Float, +{ + fn default() -> Self { + Self::new() + } +} + +impl IntoIterator for Stack +where + A: Activate, + T: Float, +{ + type Item = Layer; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.children.into_iter() + } +} + +impl FromIterator> for Stack +where + A: Activate, + T: Float, +{ + fn from_iter>>(iter: I) -> Self { + Self { + children: iter.into_iter().collect(), + } + } +} + +impl From>> for Stack +where + A: Activate, + T: Float, +{ + fn from(layers: Vec>) -> Self { + Self { children: layers } + } +} + +impl From> for Stack +where + A: Activate, + T: Float, +{ + fn from(layer: Layer) -> Self { + Self { + children: vec![layer], + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::prelude::{LayerShape, Softmax}; + + #[test] + fn test_stack() { + let (samples, inputs, outputs) = (20, 5, 3); + let features = LayerShape::new(inputs, outputs); + + let shapes = [(inputs, outputs), (outputs, outputs), (outputs, 1)]; + + let stack = Stack::::new() + .build_layers(shapes) + .init_layers(true); + + assert!(stack.validate_shapes()); + for (layer, shape) in stack.layers().iter().zip(&shapes) { + assert_eq!(layer.features(), &LayerShape::new(shape.0, shape.1)); + } } } diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index 7bfa6bb1..1b9b89ab 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -3,11 +3,12 @@ Contrib: FL03 */ //! # neurons -pub use self::{neuron::*, node::*, params::*, utils::*}; +pub use self::{neuron::*, node::*, params::*, synapse::*, utils::*}; pub(crate) mod neuron; pub(crate) mod node; pub(crate) mod params; +pub(crate) mod synapse; use crate::func::activate::Activate; use ndarray::prelude::{Array0, Array1, Array2, Ix1, NdFloat}; @@ -38,18 +39,20 @@ pub(crate) mod utils {} #[cfg(test)] mod tests { use super::*; - use crate::func::activate::{softmax, ActivateMethod, Softmax}; + use crate::func::activate::{softmax, Activate, Softmax}; use crate::prelude::Forward; // use lazy_static::lazy_static; - use ndarray::{array, Array1}; + use ndarray::prelude::{array, Array1, Ix1}; fn _artificial( args: &Array1, bias: Option>, - rho: impl ActivateMethod>, + rho: impl Activate, weights: &Array1, ) -> Array1 { - rho.rho(args.dot(weights) + bias.unwrap_or_else(|| Array1::::zeros(args.shape()[0]))) + let bias = bias.unwrap_or_else(|| Array1::zeros(args.len())); + let linear = args.dot(weights) + bias; + rho.activate(&linear) } #[test] diff --git a/ml/neural/src/neurons/neuron.rs b/ml/neural/src/neurons/neuron.rs index e5c29ae0..50d6fb82 100644 --- a/ml/neural/src/neurons/neuron.rs +++ b/ml/neural/src/neurons/neuron.rs @@ -4,7 +4,7 @@ */ use crate::core::GenerateRandom; use crate::func::activate::{Activate, LinearActivation}; -use crate::prelude::Forward; +use crate::prelude::{Biased, Forward, Weighted}; use ndarray::prelude::{Array0, Array1, Array2, Ix1, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; @@ -27,30 +27,10 @@ where A: Activate, T: Float, { - pub fn bias(&self) -> &Array0 { - &self.bias - } - pub fn rho(&self) -> &A { &self.activation } - pub fn weights(&self) -> &Array1 { - &self.weights - } - - pub fn weights_mut(&mut self) -> &mut Array1 { - &mut self.weights - } - - pub fn set_bias(&mut self, bias: Array0) { - self.bias = bias; - } - - pub fn set_weights(&mut self, weights: Array1) { - self.weights = weights; - } - pub fn with_bias(mut self, bias: Array0) -> Self { self.bias = bias; self @@ -92,7 +72,11 @@ where G: Fn(&Array1) -> Array1, { let grad = gradient(&self.weights); - self.weights_mut().scaled_add(-gamma, &grad); + self.update_with_gradient(gamma, &grad); + } + + pub fn update_with_gradient(&mut self, gamma: T, grad: &Array1) { + self.weights_mut().scaled_add(-gamma, grad); } } @@ -122,6 +106,42 @@ where } } +impl Biased for Neuron +where + T: NdFloat, + A: Activate, +{ + fn bias(&self) -> &Array0 { + &self.bias + } + + fn bias_mut(&mut self) -> &mut Array0 { + &mut self.bias + } + + fn set_bias(&mut self, bias: Array0) { + self.bias = bias; + } +} + +impl Weighted for Neuron +where + T: NdFloat, + A: Activate, +{ + fn weights(&self) -> &Array1 { + &self.weights + } + + fn weights_mut(&mut self) -> &mut Array1 { + &mut self.weights + } + + fn set_weights(&mut self, weights: Array1) { + self.weights = weights; + } +} + // impl Forward> for Neuron { // type Output = f64; diff --git a/ml/neural/src/neurons/node.rs b/ml/neural/src/neurons/node.rs index 8ecc8bfd..f7671922 100644 --- a/ml/neural/src/neurons/node.rs +++ b/ml/neural/src/neurons/node.rs @@ -2,43 +2,155 @@ Appellation: node Contrib: FL03 */ -use super::Neuron; +use crate::core::prelude::GenerateRandom; -use ndarray::prelude::Array1; +use crate::prelude::{Biased, Forward, Weighted}; +use ndarray::prelude::{Array0, Array1, Array2, Ix1, NdFloat}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use num::{Float, FromPrimitive}; #[derive(Clone, Debug, PartialEq)] -pub struct Node { - data: Array1, - neuron: Neuron, +pub struct Node +where + T: Float, +{ + bias: Array0, + pub features: usize, + weights: Array1, } -impl Node { - pub fn new(neuron: Neuron) -> Self { - let shape = neuron.weights().shape(); +impl Node +where + T: Float, +{ + pub fn new(features: usize) -> Self { Self { - data: Array1::default([shape[0]]), - neuron, + bias: Array0::zeros(()), + features, + weights: Array1::zeros(features), } } - pub fn data(&self) -> &Array1 { - &self.data + pub fn features(&self) -> usize { + self.features } - pub fn dot(&self) -> f64 { - self.data.dot(self.neuron.weights()) + pub fn features_mut(&mut self) -> &mut usize { + &mut self.features } - pub fn neuron(&self) -> &Neuron { - &self.neuron + pub fn set_features(&mut self, features: usize) { + self.features = features; } - pub fn set_data(&mut self, data: Array1) { - self.data = data; + pub fn with_bias(mut self, bias: Array0) -> Self { + self.bias = bias.into(); + self + } + + pub fn with_features(mut self, features: usize) -> Self { + self.features = features; + self + } + + pub fn with_weights(mut self, weights: Array1) -> Self { + self.weights = weights; + self + } +} + +impl Node +where + T: Float + SampleUniform, +{ + pub fn init(mut self, biased: bool) -> Self { + if biased { + self = self.init_bias(); + } + self.init_weight() } - pub fn with_data(mut self, data: Array1) -> Self { - self.data = data; + pub fn init_bias(mut self) -> Self { + let dk = (T::one() / T::from(self.features).unwrap()).sqrt(); + self.bias = Array0::uniform_between(dk, ()); self } + + pub fn init_weight(mut self) -> Self { + let features = self.features; + let dk = (T::one() / T::from(features).unwrap()).sqrt(); + self.weights = Array1::uniform_between(dk, features); + self + } +} + +impl Node +where + T: FromPrimitive + NdFloat, +{ + pub fn apply_gradient(&mut self, gamma: T, gradient: G) + where + G: Fn(&Array1) -> Array1, + { + let mut grad = gradient(self.weights()); + grad /= grad.mapv(|ws| ws.powi(2)).sum().sqrt(); + self.weights_mut().scaled_add(-gamma, &grad); + self.weights /= self.weights().mapv(|ws| ws.powi(2)).sum().sqrt(); + } + + pub fn activate(&self, data: &Array2, activator: A) -> Array1 + where + A: Fn(&Array1) -> Array1, + { + activator(&self.linear(data)) + } + + pub fn linear(&self, data: &Array2) -> Array1 { + data.dot(&self.weights().t()) + self.bias() + } +} + +impl Forward> for Node +where + T: FromPrimitive + NdFloat, +{ + type Output = Array1; + + fn forward(&self, data: &Array2) -> Self::Output { + data.dot(&self.weights().t()) + self.bias() + } +} + +impl Biased for Node +where + T: Float, +{ + fn bias(&self) -> &Array0 { + &self.bias + } + + fn bias_mut(&mut self) -> &mut Array0 { + &mut self.bias + } + + fn set_bias(&mut self, bias: Array0) { + self.bias = bias; + } +} + +impl Weighted for Node +where + T: Float, +{ + fn set_weights(&mut self, weights: Array1) { + self.weights = weights; + } + + fn weights(&self) -> &Array1 { + &self.weights + } + + fn weights_mut(&mut self) -> &mut Array1 { + &mut self.weights + } } diff --git a/ml/neural/src/neurons/synapse.rs b/ml/neural/src/neurons/synapse.rs new file mode 100644 index 00000000..f04a1c80 --- /dev/null +++ b/ml/neural/src/neurons/synapse.rs @@ -0,0 +1,9 @@ +/* + Appellation: synapse + Contrib: FL03 +*/ + +pub struct Synapse { + pub layer: usize, + pub node: usize, +} diff --git a/ml/neural/src/nn/mod.rs b/ml/neural/src/nn/mod.rs index bfd605f1..5d743331 100644 --- a/ml/neural/src/nn/mod.rs +++ b/ml/neural/src/nn/mod.rs @@ -3,16 +3,19 @@ Contrib: FL03 */ //! # Neural Network -pub use self::{deep::*, shallow::*, utils::*}; +pub use self::{position::*, sequential::*, utils::*}; -pub(crate) mod deep; -pub(crate) mod shallow; +pub(crate) mod position; +pub(crate) mod sequential; use crate::layers::Layer; use crate::Trainable; use num::Float; -pub trait NeuralNet: Trainable { +pub trait NeuralNet: Trainable +where + T: Float, +{ fn depth(&self) -> usize { self.layers().len() } @@ -22,6 +25,17 @@ pub trait NeuralNet: Trainable { fn input_layer(&self) -> &Layer { &self.layers()[0] } + + fn output_layer(&self) -> &Layer { + &self.layers()[self.depth() - 1] + } +} + +pub trait DeepNeuralNet: NeuralNet +where + T: Float, +{ + fn hidden_layers(&self) -> &[Layer]; } pub(crate) mod utils {} diff --git a/ml/neural/src/nn/position.rs b/ml/neural/src/nn/position.rs new file mode 100644 index 00000000..77227851 --- /dev/null +++ b/ml/neural/src/nn/position.rs @@ -0,0 +1,6 @@ +/* + Appellation: position + Contrib: FL03 +*/ + +pub struct Position {} diff --git a/ml/neural/src/nn/sequential.rs b/ml/neural/src/nn/sequential.rs new file mode 100644 index 00000000..b3884a9e --- /dev/null +++ b/ml/neural/src/nn/sequential.rs @@ -0,0 +1,12 @@ +/* + Appellation: sequential + Contrib: FL03 +*/ + +pub struct Sequential {} + +impl Sequential { + pub fn new() -> Self { + Self {} + } +} diff --git a/ml/neural/src/ops/dropout.rs b/ml/neural/src/ops/dropout.rs index f44602de..48dae135 100644 --- a/ml/neural/src/ops/dropout.rs +++ b/ml/neural/src/ops/dropout.rs @@ -25,12 +25,13 @@ where #[derive(Clone, Copy, Debug, Deserialize, PartialEq, PartialOrd, Serialize)] pub struct Dropout { + axis: Option, p: f64, } impl Dropout { - pub fn new(p: f64) -> Self { - Self { p } + pub fn new(axis: Option, p: f64) -> Self { + Self { axis, p } } pub fn apply(&self, array: &Array) -> Array @@ -39,10 +40,25 @@ impl Dropout { { dropout(array, self.p) } + + pub fn dropout(&self, array: &Array, p: f64) -> Array + where + T: Float, + { + // Create a Bernoulli distribution for dropout + let distribution = Bernoulli::new(p).unwrap(); + + // Create a mask of the same shape as the input array + let mask: Array = Array::::random(array.dim(), distribution); + let mask = mask.mapv(|x| if x { T::zero() } else { T::one() }); + + // Element-wise multiplication to apply dropout + array * mask + } } impl Default for Dropout { fn default() -> Self { - Self::new(0.5) + Self::new(None, 0.5) } } diff --git a/ml/optim/examples/norm.rs b/ml/optim/examples/norm.rs index 065b5884..e5cc5dbe 100644 --- a/ml/optim/examples/norm.rs +++ b/ml/optim/examples/norm.rs @@ -1,4 +1,4 @@ -use concision_neural::prelude::Features; +use concision_neural::prelude::LayerShape; use concision_optim::prelude::Norm; use ndarray::prelude::Array; @@ -6,7 +6,7 @@ fn main() -> anyhow::Result<()> { let (samples, inputs) = (20, 3); let outputs = 8; - let features = Features::new(inputs, outputs); + let features = LayerShape::new(inputs, outputs); // basic_descent(epochs, features, gamma)?; @@ -15,7 +15,7 @@ fn main() -> anyhow::Result<()> { Ok(()) } -pub fn sample_norm(features: Features, samples: usize) -> anyhow::Result<()> { +pub fn sample_norm(features: LayerShape, samples: usize) -> anyhow::Result<()> { let n = samples * features.inputs(); let args = Array::linspace(1., n as f64, n) .into_shape((samples, features.inputs())) diff --git a/ml/optim/examples/sgd.rs b/ml/optim/examples/sgd.rs index 325fc957..ed9c0aaf 100644 --- a/ml/optim/examples/sgd.rs +++ b/ml/optim/examples/sgd.rs @@ -1,5 +1,5 @@ use concision_core::prelude::linarr; -use concision_neural::prelude::{Features, Layer, Sigmoid}; +use concision_neural::prelude::{Layer, LayerShape, Sigmoid}; use concision_optim::grad::sgd::sgd; use ndarray::prelude::Array2; @@ -7,7 +7,7 @@ fn main() -> anyhow::Result<()> { let (samples, inputs) = (20, 10); let outputs = 5; - let features = Features::new(inputs, outputs); + let features = LayerShape::new(inputs, outputs); let (batch_size, epochs, gamma) = (20, 4, 0.01); // Generate some example data diff --git a/ml/optim/src/grad/descent.rs b/ml/optim/src/grad/descent.rs index 2c229d3b..b351039b 100644 --- a/ml/optim/src/grad/descent.rs +++ b/ml/optim/src/grad/descent.rs @@ -2,7 +2,7 @@ Appellation: grad Contrib: FL03 */ -use crate::neural::prelude::{Forward, Linear}; +use crate::neural::prelude::{Forward, Neuron}; use crate::prelude::Norm; use ndarray::prelude::{Array1, Array2}; use ndarray_stats::DeviationExt; @@ -10,11 +10,11 @@ use ndarray_stats::DeviationExt; #[derive(Clone)] pub struct GradientDescent { pub gamma: f64, - model: Linear, + model: Neuron, } impl GradientDescent { - pub fn new(gamma: f64, model: Linear) -> Self { + pub fn new(gamma: f64, model: Neuron) -> Self { Self { gamma, model } } @@ -26,11 +26,11 @@ impl GradientDescent { &mut self.gamma } - pub fn model(&self) -> &Linear { + pub fn model(&self) -> &Neuron { &self.model } - pub fn model_mut(&mut self) -> &mut Linear { + pub fn model_mut(&mut self) -> &mut Neuron { &mut self.model } @@ -38,7 +38,7 @@ impl GradientDescent { self.gamma = gamma; } - pub fn set_model(&mut self, model: Linear) { + pub fn set_model(&mut self, model: Neuron) { self.model = model; } @@ -47,7 +47,7 @@ impl GradientDescent { self } - pub fn with_model(mut self, model: Linear) -> Self { + pub fn with_model(mut self, model: Neuron) -> Self { self.model = model; self } @@ -112,7 +112,7 @@ mod tests { // Generate some example data let (x, y) = sample_data(samples, inputs); - let model = Linear::new(inputs).init_weight(); + let model = Neuron::new(inputs).init_weight(); let mut grad = GradientDescent::new(gamma, model); let _s = grad.step(&x, &y); diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index 54b99a59..bf28d81c 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -101,7 +101,7 @@ mod tests { use super::*; use crate::core::prelude::linarr; use crate::neural::func::activate::{LinearActivation, Sigmoid}; - use crate::neural::prelude::{Features, Layer, Parameterized, Weighted}; + use crate::neural::prelude::{Layer, LayerShape, Parameterized, Weighted}; use ndarray::prelude::{Array1, Array2}; fn test_grad(args: &Array2) -> Array2 { @@ -115,7 +115,7 @@ mod tests { let (epochs, gamma) = (10, 0.001); - let features = Features::new(inputs, outputs); + let features = LayerShape::new(inputs, outputs); let mut model = Layer::::from(features).init(true); @@ -135,7 +135,7 @@ mod tests { let (epochs, gamma) = (10, 0.001); - let features = Features::new(inputs, outputs); + let features = LayerShape::new(inputs, outputs); // Generate some example data let x = linarr((samples, features.inputs())).unwrap(); diff --git a/ml/optim/src/grad/sgd.rs b/ml/optim/src/grad/sgd.rs index 849ad172..74227de6 100644 --- a/ml/optim/src/grad/sgd.rs +++ b/ml/optim/src/grad/sgd.rs @@ -215,7 +215,7 @@ where mod tests { use super::*; use crate::core::prelude::GenerateRandom; - use crate::neural::prelude::{Features, Sigmoid}; + use crate::neural::prelude::{LayerShape, Sigmoid}; use ndarray::prelude::{Array, Array1}; #[test] @@ -223,7 +223,7 @@ mod tests { let (samples, inputs, outputs) = (20, 5, 4); let shape = (samples, inputs); - let features = Features::new(inputs, outputs); + let features = LayerShape::new(inputs, outputs); let (batch_size, epochs, gamma) = (10, 1, 0.01); // Generate some example data diff --git a/ml/optim/src/optimize/optimizer.rs b/ml/optim/src/optimize/optimizer.rs index 238d1884..70b403ef 100644 --- a/ml/optim/src/optimize/optimizer.rs +++ b/ml/optim/src/optimize/optimizer.rs @@ -8,3 +8,26 @@ use ndarray::prelude::Dimension; pub struct Optimizer { params: Vec>, } + +impl Optimizer { + pub fn new() -> Self { + Self { params: Vec::new() } + } + + pub fn params(&self) -> &[Box] { + &self.params + } + + pub fn params_mut(&mut self) -> &mut [Box] { + &mut self.params + } + + pub fn set_params(&mut self, params: Vec>) { + self.params = params; + } + + pub fn with_params(mut self, params: Vec>) -> Self { + self.params = params; + self + } +} diff --git a/ml/optim/src/specs.rs b/ml/optim/src/specs.rs index dee54cfb..d21c303c 100644 --- a/ml/optim/src/specs.rs +++ b/ml/optim/src/specs.rs @@ -25,42 +25,36 @@ pub trait Objective { fn objective(&self, args: &Array2) -> Array1; } -pub trait PartialDerivative { - type Args; - - fn partial_derivative(&self, args: Self::Args) -> T; -} - pub trait Minimize { fn minimize(&self, scale: T) -> Self; } -pub trait LearningRate +pub trait Dampener where T: Float, { - fn gamma(&self) -> T; + fn tau(&self) -> T; // Momentum Damper } -pub trait Momentum +pub trait Decay where T: Float, { - fn mu(&self) -> T; // Momentum Rate - - fn nestrov(&self) -> bool; + fn lambda(&self) -> T; // Decay Rate } -pub trait Decay +pub trait LearningRate where T: Float, { - fn lambda(&self) -> T; // Decay Rate + fn gamma(&self) -> T; } -pub trait Dampener +pub trait Momentum where T: Float, { - fn tau(&self) -> T; // Momentum Damper + fn mu(&self) -> T; // Momentum Rate + + fn nestrov(&self) -> bool; } diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index a99e7cf9..a7448ed6 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -19,7 +19,10 @@ pub struct AttentionHead { weights: Weight, } -impl AttentionHead { +impl AttentionHead +where + T: Float, +{ pub fn dim(&self) -> HeadShape { self.dim } @@ -74,31 +77,19 @@ where } } -impl Head for AttentionHead { - fn key(&self) -> &Array2 { - &self.weights.key - } - - fn mask(&self) -> &Array2 { - &self.mask - } - - fn query(&self) -> &Array2 { - &self.weights.query - } - - fn value(&self) -> &Array2 { - &self.weights.value - } -} - -impl std::fmt::Display for AttentionHead { +impl std::fmt::Display for AttentionHead +where + T: Float + Serialize, +{ fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "{}", serde_json::to_string(self).unwrap()) } } -impl ops::Index for AttentionHead { +impl ops::Index for AttentionHead +where + T: Float, +{ type Output = Array2; fn index(&self, index: QKV) -> &Self::Output { @@ -106,13 +97,19 @@ impl ops::Index for AttentionHead { } } -impl ops::IndexMut for AttentionHead { +impl ops::IndexMut for AttentionHead +where + T: Float, +{ fn index_mut(&mut self, index: QKV) -> &mut Self::Output { &mut self.weights[index] } } -impl ops::Mul> for AttentionHead { +impl ops::Mul> for AttentionHead +where + T: NdFloat, +{ type Output = AttentionHead; fn mul(self, rhs: Array2) -> Self::Output { @@ -122,7 +119,10 @@ impl ops::Mul> for AttentionHead { } } -impl ops::Mul<&Array2> for AttentionHead { +impl ops::Mul<&Array2> for AttentionHead +where + T: NdFloat, +{ type Output = AttentionHead; fn mul(self, rhs: &Array2) -> Self::Output { @@ -132,19 +132,28 @@ impl ops::Mul<&Array2> for AttentionHead { } } -impl ops::MulAssign> for AttentionHead { +impl ops::MulAssign> for AttentionHead +where + T: NdFloat, +{ fn mul_assign(&mut self, rhs: Array2) { self.weights *= rhs; } } -impl ops::MulAssign<&Array2> for AttentionHead { +impl ops::MulAssign<&Array2> for AttentionHead +where + T: NdFloat, +{ fn mul_assign(&mut self, rhs: &Array2) { self.weights *= rhs; } } -impl ops::MulAssign<&Array2> for &mut AttentionHead { +impl ops::MulAssign<&Array2> for &mut AttentionHead +where + T: NdFloat, +{ fn mul_assign(&mut self, rhs: &Array2) { self.weights *= rhs; } diff --git a/ml/transformers/src/attention/mod.rs b/ml/transformers/src/attention/mod.rs index 9af18a87..dc616bf2 100644 --- a/ml/transformers/src/attention/mod.rs +++ b/ml/transformers/src/attention/mod.rs @@ -16,12 +16,14 @@ pub(crate) mod weights; pub mod multi; pub mod params; +use params::QKV; + use crate::core::prelude::BoxResult; use crate::prelude::BaseDim; -use ndarray::prelude::{Array, Array2, Dimension, Ix2, NdFloat}; +use ndarray::prelude::{Array, Array2, Ix2, NdFloat}; use num::Float; -use std::ops::Mul; +use std::ops::{self, Mul}; /// (batch, sample, seq, model) pub type InputArray = Array; @@ -36,7 +38,7 @@ pub trait Attention { let k = self.key().dot(data); let v = self.value().dot(data); - let score = attention(&q, &k, &v, Some(self.mask().clone())); + let score = scaled_dot_product_attention(&q, &k, &v, Some(self.mask().clone())); Ok(score) } @@ -49,22 +51,45 @@ pub trait Attention { fn value(&self) -> &Array2; } -pub trait Head { +pub trait Head +where + T: Float, +{ fn key(&self) -> &Array2; - fn mask(&self) -> &Array2; - fn query(&self) -> &Array2; fn value(&self) -> &Array2; + + fn query_size(&self) -> usize { + self.query().dim().1 + } + + fn qkv(&self) -> (&Array2, &Array2, &Array2) { + (self.query(), self.key(), self.value()) + } + + fn scale(&self) -> T { + T::one() / (T::from(self.key().dim().1).unwrap()).sqrt() + } } -pub trait Spaces { - type Dim: Dimension; +impl Head for S +where + S: ops::Index>, + T: Float, +{ + fn key(&self) -> &Array2 { + &self[QKV::Key] + } - fn query(&self) -> &Array; - fn key(&self) -> &Array; - fn value(&self) -> &Array; + fn query(&self) -> &Array2 { + &self[QKV::Query] + } + + fn value(&self) -> &Array2 { + &self[QKV::Value] + } } pub trait Weights: Mul, Output = Self> { @@ -83,7 +108,7 @@ pub(crate) mod utils { use crate::neural::prelude::{Activate, Softmax}; use ndarray::prelude::{Array2, NdFloat}; - pub fn attention( + pub fn scaled_dot_product_attention( query: &Array2, key: &Array2, value: &Array2, @@ -93,8 +118,7 @@ pub(crate) mod utils { let mask = mask.unwrap_or_else(|| Array2::::zeros((seq, seq))); let scale = T::one() / (T::from(dk).unwrap()).sqrt(); let score = (query.dot(&key.t()) + mask) * scale; - let pred = Softmax::new(Some(1)).activate(&score); - pred.dot(value) + Softmax::new(Some(1)).activate(&score).dot(value) } } diff --git a/ml/transformers/src/attention/multi/mod.rs b/ml/transformers/src/attention/multi/mod.rs index 334cc385..1e0bcd6a 100644 --- a/ml/transformers/src/attention/multi/mod.rs +++ b/ml/transformers/src/attention/multi/mod.rs @@ -30,7 +30,7 @@ where } pub(crate) mod utils { - use crate::attention::attention; + use crate::attention::scaled_dot_product_attention; use crate::neural::prelude::Mask; use crate::ops::Merge; use ndarray::prelude::{s, Array2, Array3, Array4, NdFloat}; @@ -50,7 +50,7 @@ pub(crate) mod utils { let q = query.slice(s![i, h, .., ..]).to_owned(); let k = key.slice(s![i, h, .., ..]).to_owned(); let v = value.slice(s![i, h, .., ..]).to_owned(); - let head = attention(&q, &k, &v, Some(mask.clone())); + let head = scaled_dot_product_attention(&q, &k, &v, Some(mask.clone())); score.slice_mut(s![i, h, .., ..]).assign(&head); } } @@ -73,7 +73,7 @@ pub(crate) mod utils { let q = query.slice(pos).to_owned(); let k = key.slice(pos).to_owned(); let v = value.slice(pos).to_owned(); - let head = attention(&q, &k, &v, mask.clone().into()); + let head = scaled_dot_product_attention(&q, &k, &v, mask.clone().into()); score.slice_mut(s![h, .., ..]).assign(&head); } score.merge() diff --git a/ml/transformers/src/attention/params/dim.rs b/ml/transformers/src/attention/params/dim.rs index 910f156e..3ef15f1e 100644 --- a/ml/transformers/src/attention/params/dim.rs +++ b/ml/transformers/src/attention/params/dim.rs @@ -11,7 +11,7 @@ //! - `heads`: The number of attention heads //! - `model`: The dimension of the model (embedding size) use crate::{HEADS, MODEL_SIZE, QUERY_SIZE}; -use ndarray::prelude::{Ix3, Ix4}; +use ndarray::prelude::{Dimension, Ix2, Ix3, Ix4}; use ndarray::IntoDimension; use serde::{Deserialize, Serialize}; @@ -31,31 +31,18 @@ impl Batched for Ix4 { } } -pub enum AttentionDims { - Base(BaseShape), // a 3d matrix (batch, seq, model) - Head(HeadShape), // a 2d matrix (seq, query) - MultiHead(MultiShape), // a 4d matrix (batch, heads, seq, query) +pub enum Shapes { + Batched((usize, Box)), + Data(BaseShape), + Head(HeadShape), + Mask { seq: usize }, + MultiHead(MultiShape), } -pub enum Shapes { - Data { - batch: usize, - seq: usize, - model: usize, - }, - Head { - seq: usize, - query: usize, - }, - Mask { - seq: usize, - }, - MultiHead { - batch: usize, - heads: usize, - model: usize, - seq: usize, - }, +impl Shapes { + pub fn batched(batch_size: usize, shape: impl Into) -> Self { + Shapes::Batched((batch_size, Box::new(shape.into()))) + } } #[derive( @@ -134,12 +121,7 @@ impl MultiShape { impl From for Shapes { fn from(shape: MultiShape) -> Self { - Self::MultiHead { - batch: 1, - heads: shape.heads(), - model: shape.model_size(), - seq: shape.seq_len(), - } + Shapes::MultiHead(shape) } } @@ -166,11 +148,26 @@ impl IntoDimension for MultiShape { pub trait HeadDimension { fn query_size(&self) -> usize; fn sequence(&self) -> usize; + + fn as_shape(&self) -> HeadShape { + HeadShape::new(self.query_size(), self.sequence()) + } +} + +pub trait AsShape { + fn as_shape(&self) -> Sh; +} + +pub trait ShapeExt +where + D: ndarray::Dimension, +{ + fn dim(&self) -> D; } impl HeadDimension for T where - T: Clone + IntoDimension, + T: Clone + IntoDimension, { fn query_size(&self) -> usize { self.clone().into_dimension()[1] diff --git a/ml/transformers/src/ffn/network.rs b/ml/transformers/src/ffn/network.rs index 73c095df..ff0aa169 100644 --- a/ml/transformers/src/ffn/network.rs +++ b/ml/transformers/src/ffn/network.rs @@ -4,7 +4,7 @@ */ use super::FFNParams; use crate::neural::func::activate::{Activate, ReLU}; -use crate::neural::prelude::{Features, Forward, Layer}; +use crate::neural::prelude::{Forward, Layer, LayerShape}; use ndarray::prelude::Array2; use serde::{Deserialize, Serialize}; @@ -18,7 +18,7 @@ pub struct FFN { impl FFN { pub fn new(model: usize, network: Option) -> Self { let params = FFNParams::new(model, network.unwrap_or(crate::NETWORK_SIZE)); - let features = Features::new(model, params.network_size()); + let features = LayerShape::new(model, params.network_size()); Self { input: Layer::input(features), From 7e083827ab77638583bf43c231a07b2b34b2caa6 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 19 Nov 2023 12:33:51 -0600 Subject: [PATCH 061/118] update Signed-off-by: FL03 --- core/src/lib.rs | 2 + {ml/neural => core}/src/masks/mask.rs | 0 {ml/neural => core}/src/masks/mod.rs | 0 ml/neural/src/arch/deep.rs | 36 +++++------ ml/neural/src/arch/mod.rs | 55 +++++++++++++---- ml/neural/src/lib.rs | 2 - ml/neural/src/models/mod.rs | 5 +- ml/neural/src/models/stack.rs | 60 ++++++++++++++----- ml/transformers/src/attention/head.rs | 2 +- .../src/attention/multi/attention.rs | 3 +- ml/transformers/src/attention/multi/mod.rs | 7 +-- ml/transformers/src/attention/params/dim.rs | 2 +- ml/transformers/src/codec/encode/encoder.rs | 3 +- ml/transformers/src/codec/encode/mod.rs | 2 +- ml/transformers/src/ffn/mod.rs | 20 ++++++- ml/transformers/src/ffn/network.rs | 11 ++-- 16 files changed, 143 insertions(+), 67 deletions(-) rename {ml/neural => core}/src/masks/mask.rs (100%) rename {ml/neural => core}/src/masks/mod.rs (100%) diff --git a/core/src/lib.rs b/core/src/lib.rs index d86f998f..78d2edc4 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -11,6 +11,7 @@ pub(crate) mod utils; pub mod epochs; pub mod errors; +pub mod masks; pub mod states; pub mod step; @@ -18,6 +19,7 @@ pub mod prelude { pub use crate::epochs::*; pub use crate::errors::*; + pub use crate::masks::*; pub use crate::states::*; pub use crate::step::*; diff --git a/ml/neural/src/masks/mask.rs b/core/src/masks/mask.rs similarity index 100% rename from ml/neural/src/masks/mask.rs rename to core/src/masks/mask.rs diff --git a/ml/neural/src/masks/mod.rs b/core/src/masks/mod.rs similarity index 100% rename from ml/neural/src/masks/mod.rs rename to core/src/masks/mod.rs diff --git a/ml/neural/src/arch/deep.rs b/ml/neural/src/arch/deep.rs index f6751cfc..39df89f7 100644 --- a/ml/neural/src/arch/deep.rs +++ b/ml/neural/src/arch/deep.rs @@ -3,6 +3,7 @@ Contrib: FL03 */ use crate::func::activate::{Activate, LinearActivation}; +use crate::models::Stack; use crate::prelude::{Forward, Layer, Parameterized}; use ndarray::prelude::{Array2, Ix2, NdFloat}; @@ -16,7 +17,7 @@ where O: Activate, { pub input: Layer, - pub hidden: Vec>, + pub hidden: Stack, pub output: Layer, } @@ -27,41 +28,40 @@ where H: Activate, O: Activate, { - pub fn new(input: Layer, hidden: Vec>, output: Layer) -> Self { + pub fn new(input: Layer, hidden: Stack, output: Layer) -> Self { Self { input, hidden, output, } } - +} +impl DeepNetwork +where + T: Float, + I: Activate + Clone, + H: Activate + Clone, + O: Activate + Clone, +{ pub fn validate_dims(&self) -> bool { - let mut dim = true; - for (i, layer) in self.hidden.iter().enumerate() { - if i == 0 { - dim = self.input.features().outputs() == self.hidden[i].features().inputs() - } else if i == self.hidden.len() - 1 { - dim = dim && layer.features().outputs() == self.output.features().inputs(); - } else { - dim = dim && layer.features().outputs() == self.hidden[i + 1].features().inputs(); - } - } - dim + self.hidden.validate_shapes() + && self.input.features().outputs() == self.hidden.first().unwrap().features().inputs() + && self.output.features().inputs() == self.hidden.last().unwrap().features().outputs() } } impl Forward> for DeepNetwork where T: NdFloat, - I: Activate, - H: Activate, - O: Activate, + I: Activate + Clone, + H: Activate + Clone, + O: Activate + Clone, { type Output = Array2; fn forward(&self, args: &Array2) -> Self::Output { let mut out = self.input.forward(args); - for layer in &self.hidden { + for layer in self.hidden.clone().into_iter() { out = layer.forward(&out); } self.output.forward(&out) diff --git a/ml/neural/src/arch/mod.rs b/ml/neural/src/arch/mod.rs index 11d37c48..567d0fcb 100644 --- a/ml/neural/src/arch/mod.rs +++ b/ml/neural/src/arch/mod.rs @@ -18,18 +18,10 @@ pub(crate) mod utils {} #[cfg(test)] mod tests { use super::*; + use crate::core::prelude::linarr; use crate::models::stack::Stack; - use crate::prelude::{Activate, Layer, LayerShape, Softmax}; - - fn _stack + Default>( - shapes: impl IntoIterator, - ) -> Stack { - let mut stack = Stack::new(); - for (inputs, outputs) in shapes.into_iter() { - stack.push(Layer::::from(LayerShape::new(inputs, outputs)).init(true)); - } - stack - } + use crate::prelude::{Forward, Layer, LayerShape, Sigmoid}; + use ndarray::prelude::Ix2; #[test] fn test_arch() { @@ -38,6 +30,45 @@ mod tests { #[test] fn test_deep_network() { - assert!(true); + let samples = 20; + let (inputs, outputs) = (5, 3); + let shapes = [(outputs, 4), (4, 4), (4, inputs)]; + + // sample data + let x = linarr::((samples, inputs)).unwrap(); + let _y = linarr::((samples, outputs)).unwrap(); + + // layers + let hidden = Stack::::new() + .build_layers(shapes) + .init_layers(true); + let input = Layer::::from(LayerShape::new(inputs, outputs)).init(false); + let output = Layer::::from(LayerShape::new(inputs, outputs)).init(false); + + let network = DeepNetwork::new(input, hidden, output); + assert!(network.validate_dims()); + + let pred = network.forward(&x); + assert_eq!(&pred.dim(), &(samples, outputs)); + } + + #[test] + fn test_shallow_network() { + let samples = 20; + let (inputs, outputs) = (5, 3); + + // sample data + let x = linarr::((samples, inputs)).unwrap(); + let _y = linarr::((samples, outputs)).unwrap(); + + // layers + let input = Layer::::from(LayerShape::new(inputs, outputs)).init(false); + let output = Layer::::from(LayerShape::new(outputs, outputs)).init(false); + + let network = ShallowNetwork::new(input, output); + assert!(network.validate_dims()); + + let pred = network.forward(&x); + assert_eq!(&pred.dim(), &(samples, outputs)); } } diff --git a/ml/neural/src/lib.rs b/ml/neural/src/lib.rs index 7af7ec54..92b749a0 100644 --- a/ml/neural/src/lib.rs +++ b/ml/neural/src/lib.rs @@ -16,7 +16,6 @@ pub(crate) mod utils; pub mod arch; pub mod func; pub mod layers; -pub mod masks; pub mod models; pub mod neurons; pub mod nn; @@ -30,7 +29,6 @@ pub mod prelude { pub use crate::arch::*; pub use crate::func::{activate::*, loss::*}; pub use crate::layers::*; - pub use crate::masks::*; pub use crate::neurons::*; pub use crate::nn::*; pub use crate::ops::*; diff --git a/ml/neural/src/models/mod.rs b/ml/neural/src/models/mod.rs index ba5a7901..91a79f77 100644 --- a/ml/neural/src/models/mod.rs +++ b/ml/neural/src/models/mod.rs @@ -4,12 +4,11 @@ */ //! # Model //! -pub use self::{config::*, model::*, utils::*}; +pub use self::{config::*, model::*, stack::*, utils::*}; pub(crate) mod config; pub(crate) mod model; - -pub mod stack; +pub(crate) mod stack; use ndarray::prelude::Array2; diff --git a/ml/neural/src/models/stack.rs b/ml/neural/src/models/stack.rs index 607a0ca0..1f96baa2 100644 --- a/ml/neural/src/models/stack.rs +++ b/ml/neural/src/models/stack.rs @@ -7,10 +7,13 @@ use crate::prelude::{Activate, LayerShape, LinearActivation, Parameterized}; use ndarray::prelude::Ix2; use num::Float; use serde::{Deserialize, Serialize}; +use std::ops; + +pub trait HiddenLayers {} /// A [Stack] is a collection of [Layer]s, typically used to construct the hidden /// layers of a deep neural network. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] pub struct Stack where A: Activate, @@ -62,6 +65,22 @@ where } } + pub fn is_empty(&self) -> bool { + self.children.is_empty() + } + + pub fn first(&self) -> Option<&Layer> { + self.children.first() + } + + pub fn first_mut(&mut self) -> Option<&mut Layer> { + self.children.first_mut() + } + + pub fn last(&self) -> Option<&Layer> { + self.children.last() + } + pub fn layers(&self) -> &[Layer] { &self.children } @@ -70,13 +89,13 @@ where &mut self.children } - pub fn depth(&self) -> usize { + pub fn len(&self) -> usize { self.children.len() } pub fn validate_shapes(&self) -> bool { let mut dim = true; - for (i, layer) in self.children[..(self.depth() - 1)].iter().enumerate() { + for (i, layer) in self.children[..(self.len() - 1)].iter().enumerate() { dim = dim && layer.features().outputs() == self.children[i + 1].features().inputs(); } dim @@ -111,16 +130,6 @@ where } } -impl Default for Stack -where - A: Activate, - T: Float, -{ - fn default() -> Self { - Self::new() - } -} - impl IntoIterator for Stack where A: Activate, @@ -168,6 +177,28 @@ where } } +impl ops::Index for Stack +where + A: Activate, + T: Float, +{ + type Output = Layer; + + fn index(&self, index: usize) -> &Self::Output { + &self.children[index] + } +} + +impl ops::IndexMut for Stack +where + A: Activate, + T: Float, +{ + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.children[index] + } +} + #[cfg(test)] mod tests { use super::*; @@ -175,8 +206,7 @@ mod tests { #[test] fn test_stack() { - let (samples, inputs, outputs) = (20, 5, 3); - let features = LayerShape::new(inputs, outputs); + let (inputs, outputs) = (5, 3); let shapes = [(inputs, outputs), (outputs, outputs), (outputs, 1)]; diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index a7448ed6..10218b77 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use super::params::{HeadShape, QKV}; -use super::{Head, Weight}; +use super::Weight; use crate::neural::func::activate::{Activate, Softmax}; use ndarray::prelude::{Array2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; diff --git a/ml/transformers/src/attention/multi/attention.rs b/ml/transformers/src/attention/multi/attention.rs index c6981710..e0aa2cc8 100644 --- a/ml/transformers/src/attention/multi/attention.rs +++ b/ml/transformers/src/attention/multi/attention.rs @@ -4,7 +4,8 @@ */ use super::{multihead, MultiHeadParams}; use crate::attention::Weight; -use crate::neural::prelude::{Forward, Layer, Mask}; +use crate::core::prelude::Mask; +use crate::neural::prelude::{Forward, Layer}; use crate::ops::Split; use ndarray::prelude::{Array2, NdFloat}; use ndarray::{ScalarOperand, ShapeError}; diff --git a/ml/transformers/src/attention/multi/mod.rs b/ml/transformers/src/attention/multi/mod.rs index 1e0bcd6a..420df30a 100644 --- a/ml/transformers/src/attention/multi/mod.rs +++ b/ml/transformers/src/attention/multi/mod.rs @@ -8,8 +8,7 @@ pub(crate) mod attention; pub(crate) mod params; use crate::attention::Weight; -use crate::core::prelude::BoxResult; -use crate::neural::prelude::Mask; +use crate::core::prelude::{BoxResult, Mask}; use crate::ops::Split; use ndarray::prelude::{Array2, NdFloat}; @@ -31,7 +30,7 @@ where pub(crate) mod utils { use crate::attention::scaled_dot_product_attention; - use crate::neural::prelude::Mask; + use crate::core::prelude::Mask; use crate::ops::Merge; use ndarray::prelude::{s, Array2, Array3, Array4, NdFloat}; use ndarray::ShapeError; @@ -83,7 +82,7 @@ pub(crate) mod utils { #[cfg(test)] mod tests { use super::*; - use crate::neural::prelude::Mask; + use crate::core::prelude::Mask; #[test] fn test_multihead_shape() { diff --git a/ml/transformers/src/attention/params/dim.rs b/ml/transformers/src/attention/params/dim.rs index 3ef15f1e..b00e8d93 100644 --- a/ml/transformers/src/attention/params/dim.rs +++ b/ml/transformers/src/attention/params/dim.rs @@ -11,7 +11,7 @@ //! - `heads`: The number of attention heads //! - `model`: The dimension of the model (embedding size) use crate::{HEADS, MODEL_SIZE, QUERY_SIZE}; -use ndarray::prelude::{Dimension, Ix2, Ix3, Ix4}; +use ndarray::prelude::{Ix2, Ix3, Ix4}; use ndarray::IntoDimension; use serde::{Deserialize, Serialize}; diff --git a/ml/transformers/src/codec/encode/encoder.rs b/ml/transformers/src/codec/encode/encoder.rs index 9ef05913..b8c6c73f 100644 --- a/ml/transformers/src/codec/encode/encoder.rs +++ b/ml/transformers/src/codec/encode/encoder.rs @@ -4,8 +4,9 @@ */ use super::EncoderParams; use crate::attention::multi::MultiHeadAttention; +use crate::core::prelude::Mask; use crate::ffn::FFN; -use crate::neural::prelude::{Forward, LayerNorm, Mask}; +use crate::neural::prelude::{Forward, LayerNorm}; use ndarray::prelude::Array2; pub struct Encoder { diff --git a/ml/transformers/src/codec/encode/mod.rs b/ml/transformers/src/codec/encode/mod.rs index c70c7402..046a6444 100644 --- a/ml/transformers/src/codec/encode/mod.rs +++ b/ml/transformers/src/codec/encode/mod.rs @@ -16,7 +16,7 @@ pub(crate) mod utils {} #[cfg(test)] mod tests { use super::*; - use crate::neural::prelude::Mask; + use crate::core::prelude::Mask; use ndarray::Array2; #[test] diff --git a/ml/transformers/src/ffn/mod.rs b/ml/transformers/src/ffn/mod.rs index 17c68075..25b53042 100644 --- a/ml/transformers/src/ffn/mod.rs +++ b/ml/transformers/src/ffn/mod.rs @@ -12,8 +12,24 @@ pub(crate) mod utils {} #[cfg(test)] mod tests { - // use super::*; + use super::*; + use crate::core::prelude::linarr; + use crate::neural::prelude::Forward; + use ndarray::prelude::Ix2; #[test] - fn test_ffn() {} + fn test_ffn() { + let samples = 20; + let (model, network) = (5, 3); + + // sample data + let x = linarr::((samples, model)).unwrap(); + let _y = linarr::((samples, model)).unwrap(); + + let ffn = FFN::new(model, Some(network)); + // assert!(network.validate_dims()); + + let pred = ffn.forward(&x); + assert_eq!(&pred.dim(), &(samples, model)); + } } diff --git a/ml/transformers/src/ffn/network.rs b/ml/transformers/src/ffn/network.rs index ff0aa169..e9db240f 100644 --- a/ml/transformers/src/ffn/network.rs +++ b/ml/transformers/src/ffn/network.rs @@ -4,7 +4,7 @@ */ use super::FFNParams; use crate::neural::func::activate::{Activate, ReLU}; -use crate::neural::prelude::{Forward, Layer, LayerShape}; +use crate::neural::prelude::{Forward, Layer}; use ndarray::prelude::Array2; use serde::{Deserialize, Serialize}; @@ -17,13 +17,12 @@ pub struct FFN { impl FFN { pub fn new(model: usize, network: Option) -> Self { - let params = FFNParams::new(model, network.unwrap_or(crate::NETWORK_SIZE)); - let features = LayerShape::new(model, params.network_size()); + let network = network.unwrap_or(crate::NETWORK_SIZE); Self { - input: Layer::input(features), - output: Layer::output(features, 1), - params, + input: Layer::input((model, network).into()), + output: Layer::output((network, model).into(), 1), + params: FFNParams::new(model, network), } } } From 4564928d10ae19be100a6405dbb4cff07cf8737e Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 19 Nov 2023 13:25:36 -0600 Subject: [PATCH 062/118] update Signed-off-by: FL03 --- ml/neural/src/layers/kinds.rs | 8 +++++--- ml/neural/src/layers/layer.rs | 11 ++++++----- ml/transformers/src/ffn/mod.rs | 2 +- ml/transformers/src/ffn/network.rs | 7 ++++--- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/ml/neural/src/layers/kinds.rs b/ml/neural/src/layers/kinds.rs index dfcbc213..09c3ab4d 100644 --- a/ml/neural/src/layers/kinds.rs +++ b/ml/neural/src/layers/kinds.rs @@ -70,13 +70,15 @@ impl LayerPosition { self.kind().is_output() } + pub fn index(&self) -> usize { + self.idx + } + pub fn kind(&self) -> &LayerKind { &self.kind } - pub fn position(&self) -> usize { - self.idx - } + } impl AsRef for LayerPosition { diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index d961fdf0..57efea62 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -59,6 +59,10 @@ where &self.activator } + pub fn index(&self) -> usize { + self.position().index() + } + pub fn kind(&self) -> &LayerKind { self.position().kind() } @@ -86,12 +90,9 @@ where } pub fn validate_layer(&self, other: &Self) -> bool { - let pos = self - .position() - .position() - .abs_diff(other.position().position()); + let pos = self.position().index().abs_diff(other.position().index()); if pos == 1 { - if self.position().position() > other.position().position() { + if self.position().index() > other.position().index() { return self.features().inputs() == other.features().outputs(); } else { return self.features().outputs() == other.features().inputs(); diff --git a/ml/transformers/src/ffn/mod.rs b/ml/transformers/src/ffn/mod.rs index 25b53042..d5fd0d86 100644 --- a/ml/transformers/src/ffn/mod.rs +++ b/ml/transformers/src/ffn/mod.rs @@ -20,7 +20,7 @@ mod tests { #[test] fn test_ffn() { let samples = 20; - let (model, network) = (5, 3); + let (model, network) = (5, 15); // sample data let x = linarr::((samples, model)).unwrap(); diff --git a/ml/transformers/src/ffn/network.rs b/ml/transformers/src/ffn/network.rs index e9db240f..9cd4115f 100644 --- a/ml/transformers/src/ffn/network.rs +++ b/ml/transformers/src/ffn/network.rs @@ -17,11 +17,12 @@ pub struct FFN { impl FFN { pub fn new(model: usize, network: Option) -> Self { + let network = network.unwrap_or(crate::NETWORK_SIZE); - + let features = network / model; Self { - input: Layer::input((model, network).into()), - output: Layer::output((network, model).into(), 1), + input: Layer::input((model, features).into()), + output: Layer::output((features, model).into(), 1), params: FFNParams::new(model, network), } } From bc032e227088c4290316bf46f1051798666b8ec7 Mon Sep 17 00:00:00 2001 From: FL03 Date: Tue, 21 Nov 2023 15:31:13 -0600 Subject: [PATCH 063/118] update Signed-off-by: FL03 --- ml/neural/src/func/activate/linear.rs | 20 ++-- ml/neural/src/func/activate/mod.rs | 111 ++++++++++++++++------- ml/neural/src/func/activate/nonlinear.rs | 55 +++-------- ml/neural/src/layers/layer.rs | 2 + ml/neural/src/layers/mod.rs | 37 +++++++- ml/neural/src/models/model.rs | 41 ++++++++- ml/neural/src/neurons/mod.rs | 3 +- ml/neural/src/prop/mod.rs | 2 + ml/neural/src/specs.rs | 20 ++++ 9 files changed, 200 insertions(+), 91 deletions(-) diff --git a/ml/neural/src/func/activate/linear.rs b/ml/neural/src/func/activate/linear.rs index a473f76c..1e328337 100644 --- a/ml/neural/src/func/activate/linear.rs +++ b/ml/neural/src/func/activate/linear.rs @@ -2,7 +2,7 @@ Appellation: linear Contrib: FL03 */ -use super::{Activate, ActivateMethod, ActivationFn}; +use super::{Activate, Activation, ActivationFn}; use ndarray::prelude::{Array, Dimension}; use num::One; use serde::{Deserialize, Serialize}; @@ -45,21 +45,19 @@ impl LinearActivation { } } -impl ActivateMethod for LinearActivation -where - T: Clone, -{ - fn rho(&self, x: &T) -> T { - Self::linear(x) - } -} - impl Activate for LinearActivation where D: Dimension, T: Clone, { fn activate(&self, args: &Array) -> Array { - args.mapv(|x| Self::linear(&x)) + args.clone() + } +} + + +impl Activation for LinearActivation where T: Clone { + fn activate(&self, args: &Array) -> Array { + args.clone() } } diff --git a/ml/neural/src/func/activate/mod.rs b/ml/neural/src/func/activate/mod.rs index b2a59bf1..2ea6c965 100644 --- a/ml/neural/src/func/activate/mod.rs +++ b/ml/neural/src/func/activate/mod.rs @@ -16,13 +16,17 @@ pub type ActivationFn = fn(T) -> T; pub type BoxedActivation = Box; +pub type ActivateDyn = Box>; + use ndarray::prelude::{Array, Dimension, Ix2}; use num::Float; -pub trait Activation { - fn activate(&self, args: &Array) -> Array; +pub trait Activation { + fn activate(&self, args: &Array) -> Array; } + + pub trait Activate where D: Dimension, @@ -30,6 +34,21 @@ where fn activate(&self, args: &Array) -> Array; } +// impl Activate for S where D: Dimension, S: Activation, { +// fn activate(&self, args: &Array) -> Array { +// Activation::activate::(self, args) +// } +// } + +impl Activate for Box> +where + D: Dimension, +{ + fn activate(&self, args: &Array) -> Array { + self.as_ref().activate(args) + } +} + pub trait ActivateExt: Activate where D: Dimension, @@ -38,43 +57,23 @@ where fn gradient(&self, args: &Array) -> Array; } -// impl Rho for F -// where -// D: Dimension, -// F: Fn(&Array) -> Array, -// T: Float -// { -// fn activate(&self, args: &Array) -> Array { -// self.call((args,)) -// } +// pub trait ActivateMethod { +// fn rho(&self, x: &T) -> T; // } -// impl Rho for A +// impl ActivateMethod for F // where -// A: Activate, -// D: Dimension, -// T: Float +// F: Fn(&T) -> T, // { -// fn activate(&self, args: &Array) -> Array { -// args.mapv(|x| self.rho(x)) +// fn rho(&self, x: &T) -> T { +// self.call((x,)) // } // } -pub trait ActivateMethod { - fn rho(&self, x: &T) -> T; -} - -impl ActivateMethod for F -where - F: Fn(&T) -> T, -{ - fn rho(&self, x: &T) -> T { - self.call((x,)) - } -} - pub(crate) mod utils { - use num::{One, Zero}; + use ndarray::RemoveAxis; + use ndarray::prelude::{Array, Axis, Dimension, NdFloat}; + use num::{Float, One, Zero}; pub fn linear_activation(x: &T) -> T where @@ -93,6 +92,56 @@ pub(crate) mod utils { T::zero() } } + + + pub fn relu(args: &T) -> T + where + T: Clone + PartialOrd + Zero, + { + if args > &T::zero() { + args.clone() + } else { + T::zero() + } + } + + pub fn sigmoid(x: &T) -> T + where + T: Float, + { + T::one() / (T::one() + (-x.clone()).exp()) + } + + pub fn softmax(args: &Array) -> Array + where + D: Dimension, + T: Float, + { + let denom = args.mapv(|x| x.exp()).sum(); + args.mapv(|x| x.exp() / denom) + } + + pub fn softmax_axis(args: &Array, axis: Option) -> Array + where + D: Dimension + RemoveAxis, + T: NdFloat, + { + let exp = args.mapv(|x| x.exp()); + if let Some(axis) = axis { + let denom = exp.sum_axis(Axis(axis)); + exp / denom + } else { + let denom = exp.sum(); + exp / denom + } + } + + pub fn tanh(x: &T) -> T + where + T: Float, + { + x.tanh() + } } #[cfg(test)] diff --git a/ml/neural/src/func/activate/nonlinear.rs b/ml/neural/src/func/activate/nonlinear.rs index 8d2fdce7..70586930 100644 --- a/ml/neural/src/func/activate/nonlinear.rs +++ b/ml/neural/src/func/activate/nonlinear.rs @@ -2,35 +2,12 @@ Appellation: nonlinear Contrib: FL03 */ -use super::{Activate, ActivateMethod}; +use super::{Activate, Activation}; use ndarray::prelude::{Array, Axis, Dimension, NdFloat}; use ndarray::RemoveAxis; use num::{Float, One, Zero}; use serde::{Deserialize, Serialize}; -pub fn softmax(args: Array) -> Array -where - D: Dimension, - T: Float, -{ - let denom = args.mapv(|x| x.exp()).sum(); - args.mapv(|x| x.exp() / denom) -} - -pub fn softmax_axis(args: Array, axis: Option) -> Array -where - D: Dimension + RemoveAxis, - T: NdFloat, -{ - let exp = args.mapv(|x| x.exp()); - if let Some(axis) = axis { - let denom = exp.sum_axis(Axis(axis)); - exp / denom - } else { - let denom = exp.sum(); - exp / denom - } -} #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, @@ -73,15 +50,6 @@ impl ReLU { } } -impl ActivateMethod for ReLU -where - T: Clone + PartialOrd + Zero, -{ - fn rho(&self, x: &T) -> T { - Self::relu(x) - } -} - impl Activate for ReLU where D: Dimension, @@ -91,6 +59,7 @@ where x.mapv(|x| Self::relu(&x)) } } + #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] @@ -225,25 +194,25 @@ impl Tanh { } } -impl ActivateMethod for Tanh +impl Activate for Tanh where + D: Dimension, T: Float, { - fn rho(&self, x: &T) -> T { - x.tanh() + fn activate(&self, x: &Array) -> Array { + x.mapv(Float::tanh) } } -impl Activate for Tanh -where - D: Dimension, - T: Float, +impl Activation for Tanh where T: Float { - fn activate(&self, x: &Array) -> Array { - x.mapv(|x| Self::tanh(x)) + fn activate(&self, x: &Array) -> Array { + x.mapv(Float::tanh) } } + + #[cfg(test)] mod tests { use super::*; @@ -282,7 +251,7 @@ mod tests { let exp = array![0.76159416, 0.96402758, 0.99505475]; let args = array![1.0, 2.0, 3.0]; - let res = Tanh::new().activate(&args).mapv(|i| i.round_to(8)); + let res = Activation::activate(&Tanh::new(), &args).mapv(|i| i.round_to(8)); assert_eq!(res, exp); } } diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 57efea62..9e163061 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -9,6 +9,8 @@ use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; + + #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Layer where diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index 7a08a046..b50b58e7 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -11,12 +11,41 @@ pub(crate) mod layer; pub(crate) mod params; pub(crate) mod sublayer; -use crate::func::activate::Activate; -use ndarray::prelude::Ix2; +use crate::func::activate::{Activation, ActivateDyn}; +use crate::prelude::Forward; +use ndarray::prelude::{Array2, Ix2}; use num::Float; -pub trait L { - type Rho: Activate; +pub type LayerDyn = Layer>; + +pub trait L: Forward> { + + fn features(&self) -> &LayerShape; + + fn features_mut(&mut self) -> &mut LayerShape; + + fn params(&self) -> &LayerParams; + + fn params_mut(&mut self) -> &mut LayerParams; + + +} + +pub trait LayerExt: L where T: Float { + type Rho: Activation; + + +} + +pub trait LayerWrapper: Forward> { + type Layer: Forward>; + + fn layer(&self) -> &Self::Layer; + + fn layer_mut(&mut self) -> &mut Self::Layer; + + fn wrapper(&self) -> &Self; + } pub(crate) mod utils {} diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs index f7a8c8c2..adf1b347 100644 --- a/ml/neural/src/models/model.rs +++ b/ml/neural/src/models/model.rs @@ -3,7 +3,46 @@ Contrib: FL03 */ use super::ModelConfig; +use crate::layers::LayerDyn; +use num::Float; -pub struct Model { +pub struct Model where T: Float { config: ModelConfig, + layers: Vec>, } + +impl Model where T: Float { + pub fn new(config: ModelConfig) -> Self { + Self { + config, + layers: Vec::new(), + } + } + + pub fn config(&self) -> &ModelConfig { + &self.config + } + + pub fn config_mut(&mut self) -> &mut ModelConfig { + &mut self.config + } + + pub fn layers(&self) -> &[LayerDyn] { + &self.layers + } + + pub fn layers_mut(&mut self) -> &mut [LayerDyn] { + &mut self.layers + } +} + +impl IntoIterator for Model where T: Float { + type Item = LayerDyn; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.layers.into_iter() + } +} + + diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index 1b9b89ab..1bd40ff3 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -65,7 +65,8 @@ mod tests { .with_rho(Softmax::default()) .with_weights(weights.clone()); - let exp = softmax(data.clone().dot(&weights) + bias); + let linear = data.dot(&weights) + bias; + let exp = softmax(&linear); assert_eq!(exp, neuron.forward(&data)); } diff --git a/ml/neural/src/prop/mod.rs b/ml/neural/src/prop/mod.rs index f7b99784..cd146909 100644 --- a/ml/neural/src/prop/mod.rs +++ b/ml/neural/src/prop/mod.rs @@ -13,6 +13,8 @@ pub(crate) mod propagation; // pub mod forward; use ndarray::prelude::{Array, Array2, Dimension}; +pub type ForwardDyn = Box, Output = Array2>>; + pub trait Backward { type Params; type Output; diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index 78ed25ed..de1fc93d 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -13,6 +13,26 @@ where fn init_weight(&self) -> Array; } +pub trait Linear +where + D: Dimension, + T: Float, +{ + type Output; + + fn linear(&self, args: &Array) -> Self::Output; +} + pub trait Trainable { fn train(&mut self, args: &Array2, targets: &Array2) -> Array2; } + + +pub trait NetworkModel where T: Float { + + fn forward(&self, args: &Array2) -> Array2; + + fn backward(&mut self, args: &Array2, targets: &Array2) -> Array2; + + +} From b8041c09c026a61e187b8154dde49e405de8f909 Mon Sep 17 00:00:00 2001 From: FL03 Date: Tue, 21 Nov 2023 23:14:02 -0600 Subject: [PATCH 064/118] update Signed-off-by: FL03 --- ml/neural/src/func/activate/linear.rs | 9 +--- ml/neural/src/func/activate/mod.rs | 42 ++++----------- ml/neural/src/func/activate/nonlinear.rs | 14 +---- ml/neural/src/func/mod.rs | 4 +- ml/neural/src/func/rms.rs | 53 +++++++++++++++++++ ml/neural/src/layers/kinds.rs | 2 - ml/neural/src/layers/layer.rs | 2 - ml/neural/src/layers/mod.rs | 15 +++--- ml/neural/src/models/model.rs | 17 ++++-- ml/neural/src/params/mod.rs | 22 ++++---- ml/neural/src/specs.rs | 11 ++-- ml/optim/src/grad/gradient.rs | 2 +- ml/optim/src/grad/mod.rs | 4 +- ml/optim/src/optimize/mod.rs | 5 -- .../src/attention/multi/attention.rs | 4 +- ml/transformers/src/ffn/network.rs | 1 - 16 files changed, 108 insertions(+), 99 deletions(-) create mode 100644 ml/neural/src/func/rms.rs diff --git a/ml/neural/src/func/activate/linear.rs b/ml/neural/src/func/activate/linear.rs index 1e328337..96934b44 100644 --- a/ml/neural/src/func/activate/linear.rs +++ b/ml/neural/src/func/activate/linear.rs @@ -2,7 +2,7 @@ Appellation: linear Contrib: FL03 */ -use super::{Activate, Activation, ActivationFn}; +use super::{Activate, ActivationFn}; use ndarray::prelude::{Array, Dimension}; use num::One; use serde::{Deserialize, Serialize}; @@ -54,10 +54,3 @@ where args.clone() } } - - -impl Activation for LinearActivation where T: Clone { - fn activate(&self, args: &Array) -> Array { - args.clone() - } -} diff --git a/ml/neural/src/func/activate/mod.rs b/ml/neural/src/func/activate/mod.rs index 2ea6c965..906ed788 100644 --- a/ml/neural/src/func/activate/mod.rs +++ b/ml/neural/src/func/activate/mod.rs @@ -14,19 +14,11 @@ pub(crate) mod nonlinear; pub type ActivationFn = fn(T) -> T; -pub type BoxedActivation = Box; - pub type ActivateDyn = Box>; use ndarray::prelude::{Array, Dimension, Ix2}; use num::Float; -pub trait Activation { - fn activate(&self, args: &Array) -> Array; -} - - - pub trait Activate where D: Dimension, @@ -57,43 +49,29 @@ where fn gradient(&self, args: &Array) -> Array; } -// pub trait ActivateMethod { -// fn rho(&self, x: &T) -> T; -// } - -// impl ActivateMethod for F -// where -// F: Fn(&T) -> T, -// { -// fn rho(&self, x: &T) -> T { -// self.call((x,)) -// } -// } - pub(crate) mod utils { - use ndarray::RemoveAxis; use ndarray::prelude::{Array, Axis, Dimension, NdFloat}; + use ndarray::RemoveAxis; use num::{Float, One, Zero}; - pub fn linear_activation(x: &T) -> T + pub fn linear_activation(args: &T) -> T where T: Clone, { - x.clone() + args.clone() } - pub fn heavyside(x: &T) -> T + pub fn heavyside(args: &T) -> T where - T: Clone + One + PartialOrd + Zero, + T: One + PartialOrd + Zero, { - if x.clone() > T::zero() { + if args > &T::zero() { T::one() } else { T::zero() } } - pub fn relu(args: &T) -> T where T: Clone + PartialOrd + Zero, @@ -105,11 +83,11 @@ pub(crate) mod utils { } } - pub fn sigmoid(x: &T) -> T + pub fn sigmoid(args: &T) -> T where T: Float, { - T::one() / (T::one() + (-x.clone()).exp()) + T::one() / (T::one() + (-args.clone()).exp()) } pub fn softmax(args: &Array) -> Array @@ -136,11 +114,11 @@ pub(crate) mod utils { } } - pub fn tanh(x: &T) -> T + pub fn tanh(args: &T) -> T where T: Float, { - x.tanh() + args.tanh() } } diff --git a/ml/neural/src/func/activate/nonlinear.rs b/ml/neural/src/func/activate/nonlinear.rs index 70586930..880a8faa 100644 --- a/ml/neural/src/func/activate/nonlinear.rs +++ b/ml/neural/src/func/activate/nonlinear.rs @@ -2,13 +2,12 @@ Appellation: nonlinear Contrib: FL03 */ -use super::{Activate, Activation}; +use super::Activate; use ndarray::prelude::{Array, Axis, Dimension, NdFloat}; use ndarray::RemoveAxis; use num::{Float, One, Zero}; use serde::{Deserialize, Serialize}; - #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] @@ -204,15 +203,6 @@ where } } -impl Activation for Tanh where T: Float -{ - fn activate(&self, x: &Array) -> Array { - x.mapv(Float::tanh) - } -} - - - #[cfg(test)] mod tests { use super::*; @@ -251,7 +241,7 @@ mod tests { let exp = array![0.76159416, 0.96402758, 0.99505475]; let args = array![1.0, 2.0, 3.0]; - let res = Activation::activate(&Tanh::new(), &args).mapv(|i| i.round_to(8)); + let res = Tanh::new().activate(&args).mapv(|i| i.round_to(8)); assert_eq!(res, exp); } } diff --git a/ml/neural/src/func/mod.rs b/ml/neural/src/func/mod.rs index 8105d97d..fb953059 100644 --- a/ml/neural/src/func/mod.rs +++ b/ml/neural/src/func/mod.rs @@ -13,11 +13,13 @@ //! ## Loss //! //! The loss functions are implemented as structs that implement the `Fn` trait. -pub use self::utils::*; +pub use self::{rms::*, utils::*}; pub mod activate; pub mod loss; +pub(crate) mod rms; + pub(crate) mod utils {} #[cfg(test)] diff --git a/ml/neural/src/func/rms.rs b/ml/neural/src/func/rms.rs new file mode 100644 index 00000000..853de250 --- /dev/null +++ b/ml/neural/src/func/rms.rs @@ -0,0 +1,53 @@ +/* + Appellation: rms + Contrib: FL03 +*/ +use ndarray::prelude::{Array, Dimension}; +use num::{Float, FromPrimitive}; + +pub fn rms(args: &Array) -> Option +where + D: Dimension, + T: Float + FromPrimitive, +{ + if let Some(avg) = args.mapv(|xs| xs.powi(2)).mean() { + return Some(avg.sqrt()); + } + None +} + +pub trait RootMeanSquare { + fn rms(&self) -> T; +} + +// impl RootMeanSquare for S where S: ExactSizeIterator, T: Float { +// fn rms(&self) -> T { +// let sum = self.iter().fold(T::zero(), |acc, x| acc + x.powi(2)); +// (sum / T::from(self.len()).unwrap()).sqrt() +// } +// } + +impl RootMeanSquare for Array +where + D: Dimension, + T: Float + FromPrimitive, +{ + fn rms(&self) -> T { + (self.mapv(|xs| xs.powi(2)).mean().unwrap_or_else(T::zero)).sqrt() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::prelude::linarr; + use computare::prelude::RoundTo; + use ndarray::prelude::Ix1; + + #[test] + fn test_rms() { + let v = linarr::(5).expect("Failed to create array"); + let rms = v.rms().round_to(5); + assert_eq!(rms, 3.31662); + } +} diff --git a/ml/neural/src/layers/kinds.rs b/ml/neural/src/layers/kinds.rs index 09c3ab4d..668ae7ac 100644 --- a/ml/neural/src/layers/kinds.rs +++ b/ml/neural/src/layers/kinds.rs @@ -77,8 +77,6 @@ impl LayerPosition { pub fn kind(&self) -> &LayerKind { &self.kind } - - } impl AsRef for LayerPosition { diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 9e163061..57efea62 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -9,8 +9,6 @@ use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; - - #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Layer where diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index b50b58e7..d8d198f0 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -11,7 +11,7 @@ pub(crate) mod layer; pub(crate) mod params; pub(crate) mod sublayer; -use crate::func::activate::{Activation, ActivateDyn}; +use crate::func::activate::{Activate, ActivateDyn}; use crate::prelude::Forward; use ndarray::prelude::{Array2, Ix2}; use num::Float; @@ -19,7 +19,6 @@ use num::Float; pub type LayerDyn = Layer>; pub trait L: Forward> { - fn features(&self) -> &LayerShape; fn features_mut(&mut self) -> &mut LayerShape; @@ -27,14 +26,13 @@ pub trait L: Forward> { fn params(&self) -> &LayerParams; fn params_mut(&mut self) -> &mut LayerParams; - - } -pub trait LayerExt: L where T: Float { - type Rho: Activation; - - +pub trait LayerExt: L +where + T: Float, +{ + type Rho: Activate; } pub trait LayerWrapper: Forward> { @@ -45,7 +43,6 @@ pub trait LayerWrapper: Forward> { fn layer_mut(&mut self) -> &mut Self::Layer; fn wrapper(&self) -> &Self; - } pub(crate) mod utils {} diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs index adf1b347..f1d09ec5 100644 --- a/ml/neural/src/models/model.rs +++ b/ml/neural/src/models/model.rs @@ -6,12 +6,18 @@ use super::ModelConfig; use crate::layers::LayerDyn; use num::Float; -pub struct Model where T: Float { +pub struct Model +where + T: Float, +{ config: ModelConfig, layers: Vec>, } -impl Model where T: Float { +impl Model +where + T: Float, +{ pub fn new(config: ModelConfig) -> Self { Self { config, @@ -36,7 +42,10 @@ impl Model where T: Float { } } -impl IntoIterator for Model where T: Float { +impl IntoIterator for Model +where + T: Float, +{ type Item = LayerDyn; type IntoIter = std::vec::IntoIter; @@ -44,5 +53,3 @@ impl IntoIterator for Model where T: Float { self.layers.into_iter() } } - - diff --git a/ml/neural/src/params/mod.rs b/ml/neural/src/params/mod.rs index 2fa53743..9d217392 100644 --- a/ml/neural/src/params/mod.rs +++ b/ml/neural/src/params/mod.rs @@ -48,17 +48,6 @@ where fn set_weights(&mut self, weights: Array); } -pub trait ParamsExt: Biased -where - Array: Dot, Output = Array>, - D: Dimension, - T: Float, -{ - fn linear(&self, args: &Array) -> Array { - args.dot(self.weights()) + self.bias() - } -} - pub trait Params where D: Dimension, @@ -78,6 +67,17 @@ where fn set_weights(&mut self, weights: Array); } +pub trait ParamsExt: Biased +where + Array: Dot, Output = Array>, + D: Dimension, + T: Float, +{ + fn linear(&self, args: &Array) -> Array { + args.dot(self.weights()) + self.bias() + } +} + pub trait Parameterized where D: Dimension, diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index de1fc93d..7d826863 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -19,7 +19,7 @@ where T: Float, { type Output; - + fn linear(&self, args: &Array) -> Self::Output; } @@ -27,12 +27,11 @@ pub trait Trainable { fn train(&mut self, args: &Array2, targets: &Array2) -> Array2; } - -pub trait NetworkModel where T: Float { - +pub trait NetworkModel +where + T: Float, +{ fn forward(&self, args: &Array2) -> Array2; fn backward(&mut self, args: &Array2, targets: &Array2) -> Array2; - - } diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs index f51d4495..f85bccc5 100644 --- a/ml/optim/src/grad/gradient.rs +++ b/ml/optim/src/grad/gradient.rs @@ -45,7 +45,7 @@ where T: NdFloat, { pub fn step(&mut self, x: &Array2, y: &Array1) -> anyhow::Result { - let mut cost = T::zero(); + let cost = T::zero(); Ok(cost) } } diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index bf28d81c..32d2ea15 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -33,13 +33,13 @@ pub(crate) mod utils { pub fn gradient( gamma: T, - model: &mut Layer, + model: &mut A, data: &Array2, targets: &Array2, grad: impl Fn(&Array2) -> Array2, ) -> f64 where - A: Activate, + A: Forward, Output = Array2> + Parameterized, T: FromPrimitive + NdFloat + Signed, { let (_samples, _inputs) = data.dim(); diff --git a/ml/optim/src/optimize/mod.rs b/ml/optim/src/optimize/mod.rs index 721b017d..96d4d9ff 100644 --- a/ml/optim/src/optimize/mod.rs +++ b/ml/optim/src/optimize/mod.rs @@ -13,11 +13,6 @@ pub trait Optimize { fn optimize(&self) -> Self; } -pub trait Optim: Iterator { - fn params(&self) -> &T; - fn optimize(&self) -> Self; -} - pub(crate) mod utils {} #[cfg(test)] diff --git a/ml/transformers/src/attention/multi/attention.rs b/ml/transformers/src/attention/multi/attention.rs index e0aa2cc8..6e90ff87 100644 --- a/ml/transformers/src/attention/multi/attention.rs +++ b/ml/transformers/src/attention/multi/attention.rs @@ -8,7 +8,7 @@ use crate::core::prelude::Mask; use crate::neural::prelude::{Forward, Layer}; use crate::ops::Split; use ndarray::prelude::{Array2, NdFloat}; -use ndarray::{ScalarOperand, ShapeError}; +use ndarray::ShapeError; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; @@ -54,7 +54,7 @@ where impl MultiHeadAttention where - T: NdFloat + ScalarOperand, + T: NdFloat, { pub fn attention(&self, data: &Array2, mask: &Mask) -> Result, ShapeError> { let weighted = data * self.weights(); diff --git a/ml/transformers/src/ffn/network.rs b/ml/transformers/src/ffn/network.rs index 9cd4115f..ec90e99c 100644 --- a/ml/transformers/src/ffn/network.rs +++ b/ml/transformers/src/ffn/network.rs @@ -17,7 +17,6 @@ pub struct FFN { impl FFN { pub fn new(model: usize, network: Option) -> Self { - let network = network.unwrap_or(crate::NETWORK_SIZE); let features = network / model; Self { From 38911d9a136591a2658344c7473443831cdad7e0 Mon Sep 17 00:00:00 2001 From: FL03 Date: Thu, 23 Nov 2023 12:30:32 -0600 Subject: [PATCH 065/118] update Signed-off-by: FL03 --- ml/neural/src/func/activate/mod.rs | 10 +++++----- ml/neural/src/layers/mod.rs | 9 --------- ml/neural/src/models/mod.rs | 5 +++-- 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/ml/neural/src/func/activate/mod.rs b/ml/neural/src/func/activate/mod.rs index 906ed788..33bb00ef 100644 --- a/ml/neural/src/func/activate/mod.rs +++ b/ml/neural/src/func/activate/mod.rs @@ -26,11 +26,11 @@ where fn activate(&self, args: &Array) -> Array; } -// impl Activate for S where D: Dimension, S: Activation, { -// fn activate(&self, args: &Array) -> Array { -// Activation::activate::(self, args) -// } -// } +impl Activate for fn(&Array) -> Array where D: Dimension, { + fn activate(&self, args: &Array) -> Array { + self.call((args,)) + } +} impl Activate for Box> where diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index d8d198f0..2096931e 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -35,15 +35,6 @@ where type Rho: Activate; } -pub trait LayerWrapper: Forward> { - type Layer: Forward>; - - fn layer(&self) -> &Self::Layer; - - fn layer_mut(&mut self) -> &mut Self::Layer; - - fn wrapper(&self) -> &Self; -} pub(crate) mod utils {} diff --git a/ml/neural/src/models/mod.rs b/ml/neural/src/models/mod.rs index 91a79f77..57dab5ff 100644 --- a/ml/neural/src/models/mod.rs +++ b/ml/neural/src/models/mod.rs @@ -10,14 +10,15 @@ pub(crate) mod config; pub(crate) mod model; pub(crate) mod stack; +use crate::prelude::Forward; use ndarray::prelude::Array2; -pub trait Module { +pub trait Module: Forward, Output=Array2> { fn add_module(&mut self, module: impl Module); fn layers(&self) -> &[impl Module]; - fn forward(&self, args: &Array2) -> Array2; + fn name(&self) -> &str; } pub(crate) mod utils {} From 0e027b2c47c87ba8d9c88a53c5936b4ba52bfa88 Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 24 Nov 2023 15:29:40 -0600 Subject: [PATCH 066/118] update Signed-off-by: FL03 --- concision/examples/gradients.rs | 9 +- core/src/specs.rs | 14 ++- ml/neural/src/arch/deep.rs | 4 +- ml/neural/src/arch/mod.rs | 14 ++- ml/neural/src/arch/shallow.rs | 4 +- ml/neural/src/func/activate/binary.rs | 10 +- ml/neural/src/func/activate/linear.rs | 6 +- ml/neural/src/func/activate/mod.rs | 103 ++++++++++++++- ml/neural/src/func/activate/nonlinear.rs | 118 ++++++++---------- ml/neural/src/layers/layer.rs | 20 ++- ml/neural/src/layers/mod.rs | 1 - ml/neural/src/layers/sublayer.rs | 4 +- ml/neural/src/lib.rs | 2 +- ml/neural/src/models/mod.rs | 2 +- ml/neural/src/models/stack.rs | 4 +- ml/neural/src/neurons/neuron.rs | 4 +- ml/neural/src/neurons/node.rs | 4 +- ml/neural/src/ops/dropout.rs | 15 ++- ml/neural/src/ops/norm.rs | 6 +- ml/neural/src/params/bias.rs | 32 +++-- ml/neural/src/params/group.rs | 73 ++++++++--- ml/neural/src/prop/mod.rs | 24 +--- ml/neural/src/specs.rs | 13 +- ml/optim/src/grad/mod.rs | 10 +- ml/optim/src/grad/sgd.rs | 2 - ml/optim/src/optimize/mod.rs | 5 +- ml/optim/src/specs.rs | 8 +- ml/transformers/src/attention/mod.rs | 14 +-- .../src/attention/multi/attention.rs | 27 ++-- ml/transformers/src/attention/weights.rs | 33 +++-- 30 files changed, 363 insertions(+), 222 deletions(-) diff --git a/concision/examples/gradients.rs b/concision/examples/gradients.rs index 46ed4803..19131252 100644 --- a/concision/examples/gradients.rs +++ b/concision/examples/gradients.rs @@ -1,6 +1,6 @@ use concision::prelude::{linarr, Forward, LayerShape, Weighted}; -use concision::neural::prelude::{Layer, Neuron, Sigmoid}; +use concision::neural::prelude::{Layer, Neuron, Objective, Sigmoid}; use concision::optim::grad::*; use ndarray::prelude::Array1; @@ -27,7 +27,8 @@ pub fn basic_descent(epochs: usize, features: LayerShape, gamma: f64) -> anyhow: println!( "{:?}", - gradient_descent(model.weights_mut(), epochs, gamma, Sigmoid::gradient) + gradient_descent(model.weights_mut(), epochs, gamma, |w| Sigmoid::new() + .gradient(w)) ); Ok(()) } @@ -52,7 +53,7 @@ pub fn sample_descent( let mut grad = GradientDescent::new(gamma, model); let mut losses = Array1::zeros(epochs); for e in 0..epochs { - let cost = grad.gradient(&x, &y, Sigmoid::gradient)?; + let cost = grad.gradient(&x, &y, |w| Sigmoid::new().gradient(w))?; losses[e] = cost; } println!("Losses:\n\n{:?}\n", &losses); @@ -79,7 +80,7 @@ pub fn sample_gradient( let mut losses = Array1::zeros(epochs); for e in 0..epochs { - let cost = gradient(gamma, &mut model, &x, &y, Sigmoid::gradient); + let cost = gradient(gamma, &mut model, &x, &y, |w| Sigmoid::new().gradient(w)); losses[e] = cost; } println!("Losses:\n\n{:?}\n", &losses); diff --git a/core/src/specs.rs b/core/src/specs.rs index 436f5d7e..da4618aa 100644 --- a/core/src/specs.rs +++ b/core/src/specs.rs @@ -20,9 +20,19 @@ impl IntoAxis for usize { } } -pub trait Borrowed: AsRef + AsMut {} +pub trait Apply { + fn apply(&self, f: F) -> T + where + F: FnOnce(&Self) -> T; +} + +pub trait ApplyTo { + fn apply_to(&self, args: &mut T) -> &mut T; +} + +pub trait As: AsRef + AsMut {} -impl Borrowed for S where S: AsRef + AsMut {} +impl As for S where S: AsRef + AsMut {} pub trait BinaryNum: One + Zero {} diff --git a/ml/neural/src/arch/deep.rs b/ml/neural/src/arch/deep.rs index 39df89f7..346da871 100644 --- a/ml/neural/src/arch/deep.rs +++ b/ml/neural/src/arch/deep.rs @@ -2,14 +2,14 @@ Appellation: network Contrib: FL03 */ -use crate::func::activate::{Activate, LinearActivation}; +use crate::func::activate::{Activate, Linear}; use crate::models::Stack; use crate::prelude::{Forward, Layer, Parameterized}; use ndarray::prelude::{Array2, Ix2, NdFloat}; use num::Float; -pub struct DeepNetwork +pub struct DeepNetwork where T: Float, I: Activate, diff --git a/ml/neural/src/arch/mod.rs b/ml/neural/src/arch/mod.rs index 567d0fcb..78e2629d 100644 --- a/ml/neural/src/arch/mod.rs +++ b/ml/neural/src/arch/mod.rs @@ -5,13 +5,25 @@ //! # Architecture //! //! This module describes the architecture of various components of the neural network. +//! +//! ## Contents +//! +//! ## Verification +//! +//! - Each layer must have the same number of inputs as the previous layer has outputs. +//! - Input Layers should be unbiased pub use self::{architecture::*, deep::*, shallow::*, utils::*}; pub(crate) mod architecture; pub(crate) mod deep; pub(crate) mod shallow; -pub trait Arch {} +pub trait NetworkOps {} + +pub trait Arch { + /// Validates the dimensions of the network. Each + fn validate_layers(&self) -> bool; +} pub(crate) mod utils {} diff --git a/ml/neural/src/arch/shallow.rs b/ml/neural/src/arch/shallow.rs index 20c59733..d88dcbfc 100644 --- a/ml/neural/src/arch/shallow.rs +++ b/ml/neural/src/arch/shallow.rs @@ -2,13 +2,13 @@ Appellation: shallow Contrib: FL03 */ -use crate::func::activate::{Activate, LinearActivation}; +use crate::func::activate::{Activate, Linear}; use crate::prelude::{Forward, Layer, Parameterized}; use ndarray::prelude::{Array2, Ix2, NdFloat}; use num::Float; -pub struct ShallowNetwork +pub struct ShallowNetwork where T: Float, I: Activate, diff --git a/ml/neural/src/func/activate/binary.rs b/ml/neural/src/func/activate/binary.rs index 0c9f514d..bd95ad08 100644 --- a/ml/neural/src/func/activate/binary.rs +++ b/ml/neural/src/func/activate/binary.rs @@ -13,11 +13,15 @@ use serde::{Deserialize, Serialize}; pub struct Heavyside; impl Heavyside { - pub fn heavyside(x: T) -> T + pub fn new() -> Self { + Self + } + + pub fn heavyside(args: &T) -> T where T: One + PartialOrd + Zero, { - if x > T::zero() { + if args > &T::zero() { T::one() } else { T::zero() @@ -31,6 +35,6 @@ where T: Clone + One + PartialOrd + Zero, { fn activate(&self, args: &Array) -> Array { - args.mapv(|x| Self::heavyside(x)) + args.mapv(|x| Self::heavyside(&x)) } } diff --git a/ml/neural/src/func/activate/linear.rs b/ml/neural/src/func/activate/linear.rs index 96934b44..9907a9b7 100644 --- a/ml/neural/src/func/activate/linear.rs +++ b/ml/neural/src/func/activate/linear.rs @@ -10,9 +10,9 @@ use serde::{Deserialize, Serialize}; #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] -pub struct LinearActivation; +pub struct Linear; -impl LinearActivation { +impl Linear { pub fn new() -> Self { Self::default() } @@ -45,7 +45,7 @@ impl LinearActivation { } } -impl Activate for LinearActivation +impl Activate for Linear where D: Dimension, T: Clone, diff --git a/ml/neural/src/func/activate/mod.rs b/ml/neural/src/func/activate/mod.rs index 33bb00ef..87c7e819 100644 --- a/ml/neural/src/func/activate/mod.rs +++ b/ml/neural/src/func/activate/mod.rs @@ -17,7 +17,6 @@ pub type ActivationFn = fn(T) -> T; pub type ActivateDyn = Box>; use ndarray::prelude::{Array, Dimension, Ix2}; -use num::Float; pub trait Activate where @@ -26,7 +25,19 @@ where fn activate(&self, args: &Array) -> Array; } -impl Activate for fn(&Array) -> Array where D: Dimension, { +// pub trait ActivateExt +// where +// D: Dimension, +// { +// fn new() -> Self; + +// fn method(&self) -> impl; +// } + +impl Activate for fn(&Array) -> Array +where + D: Dimension, +{ fn activate(&self, args: &Array) -> Array { self.call((args,)) } @@ -41,14 +52,40 @@ where } } -pub trait ActivateExt: Activate +pub trait Objective: Activate where D: Dimension, - T: Float, { fn gradient(&self, args: &Array) -> Array; } +impl Objective for fn(&Array) -> Array +where + D: Dimension, +{ + fn gradient(&self, args: &Array) -> Array { + self.call((args,)) + } +} + +impl Activate for Box> +where + D: Dimension, +{ + fn activate(&self, args: &Array) -> Array { + self.as_ref().activate(args) + } +} + +impl Objective for Box> +where + D: Dimension, +{ + fn gradient(&self, args: &Array) -> Array { + self.as_ref().gradient(args) + } +} + pub(crate) mod utils { use ndarray::prelude::{Array, Axis, Dimension, NdFloat}; use ndarray::RemoveAxis; @@ -124,5 +161,61 @@ pub(crate) mod utils { #[cfg(test)] mod tests { - // use super::*; + use super::*; + use computare::prelude::RoundTo; + use ndarray::array; + + #[test] + fn test_heavyside() { + let exp = array![0.0, 1.0, 1.0]; + let args = array![-1.0, 0.0, 1.0]; + + let res = Heavyside::new().activate(&args); + assert_eq!(res, exp); + } + + #[test] + fn test_linear() { + let exp = array![0.0, 1.0, 2.0]; + let args = array![0.0, 1.0, 2.0]; + + let res = Linear::new().activate(&args); + assert_eq!(res, exp); + } + + #[test] + fn test_relu() { + let exp = array![0.0, 0.0, 3.0]; + let args = array![-1.0, 0.0, 3.0]; + + let res = ReLU::new().activate(&args); + assert_eq!(res, exp); + } + + #[test] + fn test_sigmoid() { + let exp = array![0.73105858, 0.88079708, 0.95257413]; + let args = array![1.0, 2.0, 3.0]; + + let res = Sigmoid::new().activate(&args).mapv(|i| i.round_to(8)); + assert_eq!(res, exp); + } + + #[test] + fn test_softmax() { + let exp = array![0.09003057, 0.24472847, 0.66524096]; + let args = array![1.0, 2.0, 3.0]; + + let res = Softmax::new(None).activate(&args).mapv(|i| i.round_to(8)); + assert_eq!(res, exp); + } + + #[test] + fn test_tanh() { + let exp = array![0.76159416, 0.96402758, 0.99505475]; + let args = array![1.0, 2.0, 3.0]; + + let res = Tanh::new().activate(&args).mapv(|i| i.round_to(8)); + assert_eq!(res, exp); + } } diff --git a/ml/neural/src/func/activate/nonlinear.rs b/ml/neural/src/func/activate/nonlinear.rs index 880a8faa..6802b479 100644 --- a/ml/neural/src/func/activate/nonlinear.rs +++ b/ml/neural/src/func/activate/nonlinear.rs @@ -2,7 +2,7 @@ Appellation: nonlinear Contrib: FL03 */ -use super::Activate; +use super::{Activate, Objective}; use ndarray::prelude::{Array, Axis, Dimension, NdFloat}; use ndarray::RemoveAxis; use num::{Float, One, Zero}; @@ -29,14 +29,6 @@ impl ReLU { } } - pub fn gradient(args: &Array) -> Array - where - D: Dimension, - T: Clone + One + PartialOrd + Zero, - { - args.mapv(|x| Self::derivative(x)) - } - pub fn relu(args: &T) -> T where T: Clone + PartialOrd + Zero, @@ -59,6 +51,16 @@ where } } +impl Objective for ReLU +where + D: Dimension, + T: Clone + One + PartialOrd + Zero, +{ + fn gradient(&self, args: &Array) -> Array { + args.mapv(|x| Self::derivative(x)) + } +} + #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] @@ -76,14 +78,6 @@ impl Sigmoid { (-x).exp() / (T::one() + (-x).exp()).powi(2) } - pub fn gradient(args: &Array) -> Array - where - D: Dimension, - T: Float, - { - args.mapv(|x| Self::derivative(x)) - } - pub fn sigmoid(x: T) -> T where T: Float, @@ -102,6 +96,16 @@ where } } +impl Objective for Sigmoid +where + D: Dimension, + T: Float, +{ + fn gradient(&self, args: &Array) -> Array { + args.mapv(|x| Self::derivative(x)) + } +} + #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] @@ -160,6 +164,23 @@ where } } +impl Objective for Softmax +where + D: Dimension + RemoveAxis, + T: NdFloat, +{ + fn gradient(&self, args: &Array) -> Array { + let exp = args.mapv(|x| x.exp()); + if let Some(axis) = self.axis { + let denom = exp.sum_axis(Axis(axis)); + exp / denom + } else { + let denom = exp.sum(); + exp / denom + } + } +} + #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] @@ -170,26 +191,18 @@ impl Tanh { Self } - pub fn derivative(x: T) -> T + pub fn derivative(args: &T) -> T where T: Float, { - T::one() - x.tanh().powi(2) + T::one() - args.tanh().powi(2) } - pub fn gradient(args: &Array) -> Array + pub fn tanh(args: &T) -> T where - D: Dimension, T: Float, { - args.mapv(|x| Self::derivative(x)) - } - - pub fn tanh(x: T) -> T - where - T: Float, - { - x.tanh() + args.tanh() } } @@ -203,45 +216,12 @@ where } } -#[cfg(test)] -mod tests { - use super::*; - use computare::prelude::RoundTo; - use ndarray::array; - - #[test] - fn test_relu() { - let exp = array![0.0, 0.0, 3.0]; - let args = array![-1.0, 0.0, 3.0]; - - let res = ReLU::new().activate(&args); - assert_eq!(res, exp); - } - - #[test] - fn test_sigmoid() { - let exp = array![0.73105858, 0.88079708, 0.95257413]; - let args = array![1.0, 2.0, 3.0]; - - let res = Sigmoid::new().activate(&args).mapv(|i| i.round_to(8)); - assert_eq!(res, exp); - } - - #[test] - fn test_softmax() { - let exp = array![0.09003057, 0.24472847, 0.66524096]; - let args = array![1.0, 2.0, 3.0]; - - let res = Softmax::new(None).activate(&args).mapv(|i| i.round_to(8)); - assert_eq!(res, exp); - } - - #[test] - fn test_tanh() { - let exp = array![0.76159416, 0.96402758, 0.99505475]; - let args = array![1.0, 2.0, 3.0]; - - let res = Tanh::new().activate(&args).mapv(|i| i.round_to(8)); - assert_eq!(res, exp); +impl Objective for Tanh +where + D: Dimension, + T: Float, +{ + fn gradient(&self, args: &Array) -> Array { + args.mapv(|x| Self::derivative(&x)) } } diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 57efea62..88725948 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -3,14 +3,14 @@ Contrib: FL03 */ use super::{LayerKind, LayerParams, LayerPosition, LayerShape}; -use crate::prelude::{Activate, Forward, LinearActivation, Parameterized, Params}; +use crate::prelude::{Activate, Forward, Linear, Parameterized, Params}; use ndarray::prelude::{Array2, Ix2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct Layer +pub struct Layer where A: Activate, T: Float, @@ -107,6 +107,22 @@ where } } +impl Layer +where + A: Activate + Clone + 'static, + T: Float, +{ + pub fn as_dyn(&self) -> Layer>> { + Layer { + activator: Box::new(self.activator.clone()), + features: self.features.clone(), + name: self.name.clone(), + params: self.params.clone(), + position: self.position.clone(), + } + } +} + impl Layer where A: Activate, diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index 2096931e..1fb1ac3e 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -35,7 +35,6 @@ where type Rho: Activate; } - pub(crate) mod utils {} #[cfg(test)] diff --git a/ml/neural/src/layers/sublayer.rs b/ml/neural/src/layers/sublayer.rs index 986a274c..7b26a321 100644 --- a/ml/neural/src/layers/sublayer.rs +++ b/ml/neural/src/layers/sublayer.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use super::Layer; -use crate::func::activate::{Activate, LinearActivation}; +use crate::func::activate::{Activate, Linear}; use crate::ops::LayerNorm; use crate::prelude::Forward; @@ -12,7 +12,7 @@ use num::{Float, FromPrimitive}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct Sublayer +pub struct Sublayer where A: Activate, T: Float, diff --git a/ml/neural/src/lib.rs b/ml/neural/src/lib.rs index 92b749a0..72ab27b0 100644 --- a/ml/neural/src/lib.rs +++ b/ml/neural/src/lib.rs @@ -6,7 +6,7 @@ //! //! This library implements the neural network primitives and specifications. //! -#![feature(fn_traits)] +#![feature(fn_traits, unboxed_closures)] pub use self::{primitives::*, specs::*, utils::*}; pub(crate) mod primitives; diff --git a/ml/neural/src/models/mod.rs b/ml/neural/src/models/mod.rs index 57dab5ff..f58e1ba7 100644 --- a/ml/neural/src/models/mod.rs +++ b/ml/neural/src/models/mod.rs @@ -13,7 +13,7 @@ pub(crate) mod stack; use crate::prelude::Forward; use ndarray::prelude::Array2; -pub trait Module: Forward, Output=Array2> { +pub trait Module: Forward, Output = Array2> { fn add_module(&mut self, module: impl Module); fn layers(&self) -> &[impl Module]; diff --git a/ml/neural/src/models/stack.rs b/ml/neural/src/models/stack.rs index 1f96baa2..1e3036b2 100644 --- a/ml/neural/src/models/stack.rs +++ b/ml/neural/src/models/stack.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use crate::layers::Layer; -use crate::prelude::{Activate, LayerShape, LinearActivation, Parameterized}; +use crate::prelude::{Activate, LayerShape, Linear, Parameterized}; use ndarray::prelude::Ix2; use num::Float; use serde::{Deserialize, Serialize}; @@ -14,7 +14,7 @@ pub trait HiddenLayers {} /// A [Stack] is a collection of [Layer]s, typically used to construct the hidden /// layers of a deep neural network. #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -pub struct Stack +pub struct Stack where A: Activate, T: Float, diff --git a/ml/neural/src/neurons/neuron.rs b/ml/neural/src/neurons/neuron.rs index 50d6fb82..9d1217fa 100644 --- a/ml/neural/src/neurons/neuron.rs +++ b/ml/neural/src/neurons/neuron.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use crate::core::GenerateRandom; -use crate::func::activate::{Activate, LinearActivation}; +use crate::func::activate::{Activate, Linear}; use crate::prelude::{Biased, Forward, Weighted}; use ndarray::prelude::{Array0, Array1, Array2, Ix1, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; @@ -11,7 +11,7 @@ use num::Float; /// Artificial Neuron #[derive(Clone, Debug, PartialEq)] -pub struct Neuron +pub struct Neuron where A: Activate, T: Float, diff --git a/ml/neural/src/neurons/node.rs b/ml/neural/src/neurons/node.rs index f7671922..b76f481f 100644 --- a/ml/neural/src/neurons/node.rs +++ b/ml/neural/src/neurons/node.rs @@ -92,10 +92,8 @@ where where G: Fn(&Array1) -> Array1, { - let mut grad = gradient(self.weights()); - grad /= grad.mapv(|ws| ws.powi(2)).sum().sqrt(); + let grad = gradient(self.weights()); self.weights_mut().scaled_add(-gamma, &grad); - self.weights /= self.weights().mapv(|ws| ws.powi(2)).sum().sqrt(); } pub fn activate(&self, data: &Array2, activator: A) -> Array1 diff --git a/ml/neural/src/ops/dropout.rs b/ml/neural/src/ops/dropout.rs index 48dae135..62a627e1 100644 --- a/ml/neural/src/ops/dropout.rs +++ b/ml/neural/src/ops/dropout.rs @@ -2,21 +2,22 @@ Appellation: dropout Contrib: FL03 */ -use ndarray::prelude::{Array, Ix1}; +use ndarray::prelude::{Array, Dimension}; use ndarray_rand::rand_distr::Bernoulli; use ndarray_rand::RandomExt; use num::Float; use serde::{Deserialize, Serialize}; -pub fn dropout(array: &Array, p: f64) -> Array +pub fn dropout(array: &Array, p: f64) -> Array where + D: Dimension, T: Float, { // Create a Bernoulli distribution for dropout let distribution = Bernoulli::new(p).unwrap(); // Create a mask of the same shape as the input array - let mask: Array = Array::::random(array.dim(), distribution); + let mask: Array = Array::random(array.dim(), distribution); let mask = mask.mapv(|x| if x { T::zero() } else { T::one() }); // Element-wise multiplication to apply dropout @@ -34,22 +35,24 @@ impl Dropout { Self { axis, p } } - pub fn apply(&self, array: &Array) -> Array + pub fn apply(&self, array: &Array) -> Array where + D: Dimension, T: Float, { dropout(array, self.p) } - pub fn dropout(&self, array: &Array, p: f64) -> Array + pub fn dropout(&self, array: &Array, p: f64) -> Array where + D: Dimension, T: Float, { // Create a Bernoulli distribution for dropout let distribution = Bernoulli::new(p).unwrap(); // Create a mask of the same shape as the input array - let mask: Array = Array::::random(array.dim(), distribution); + let mask: Array = Array::random(array.dim(), distribution); let mask = mask.mapv(|x| if x { T::zero() } else { T::one() }); // Element-wise multiplication to apply dropout diff --git a/ml/neural/src/ops/norm.rs b/ml/neural/src/ops/norm.rs index 7fb25369..b982e63b 100644 --- a/ml/neural/src/ops/norm.rs +++ b/ml/neural/src/ops/norm.rs @@ -74,12 +74,12 @@ where { type Output = Array; - fn forward(&self, x: &Array) -> Self::Output { + fn forward(&self, data: &Array) -> Self::Output { let epsilon = T::from(1e-6).unwrap(); // Calculate the mean and standard deviation of the activations along the feature axis. - let mean = x.mean().unwrap_or_else(T::zero); + let mean = data.mean().unwrap_or_else(T::zero); // Normalize the activations. - let norm = (x - mean) / (x.std(T::one()) + epsilon); + let norm = (data - mean) / (data.std(T::one()) + epsilon); // Scale and shift the normalized activations with learnable parameters alpha and beta. norm * self.alpha().clone() + self.beta().clone() diff --git a/ml/neural/src/params/bias.rs b/ml/neural/src/params/bias.rs index b01b9574..7e010129 100644 --- a/ml/neural/src/params/bias.rs +++ b/ml/neural/src/params/bias.rs @@ -3,8 +3,7 @@ Contrib: FL03 */ use crate::generate_uniform_arr; -use ndarray::prelude::{Array, Array1}; -use ndarray::{Dimension, ScalarOperand}; +use ndarray::prelude::{Array, Array1, Dimension, Ix2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; @@ -17,6 +16,15 @@ pub struct Belief { pub features: usize, } +pub enum Biases +where + D: Dimension, + T: Float, +{ + Biased(Array), + Unbiased, +} + #[derive(Clone, Debug, Deserialize, EnumIs, PartialEq, Serialize, SmartDefault)] pub enum Bias { Biased(Array1), @@ -40,7 +48,7 @@ where impl Bias where - T: Float + ScalarOperand, + T: NdFloat, { pub fn forward(&self, data: &Array1) -> Array1 { match self { @@ -107,7 +115,7 @@ where impl ops::Add> for Bias where D: Dimension, - T: Float + ScalarOperand, + T: NdFloat, Array: ops::Add, Output = Array>, { type Output = Array; @@ -124,7 +132,7 @@ where impl ops::Add> for &Bias where D: Dimension, - T: Float + ScalarOperand, + T: NdFloat, Array: ops::Add, Output = Array>, { type Output = Array; @@ -141,7 +149,7 @@ where impl ops::Add<&Array> for Bias where D: Dimension, - T: Float + ScalarOperand, + T: NdFloat, Array: ops::Add, Output = Array>, { type Output = Array; @@ -158,7 +166,7 @@ where impl ops::Add> for Array where D: Dimension, - T: Float + ScalarOperand, + T: NdFloat, Array: ops::Add, Output = Array>, { type Output = Array; @@ -175,7 +183,7 @@ where impl ops::Add<&Bias> for Array where D: Dimension, - T: Float + ScalarOperand, + T: NdFloat, Array: ops::Add, Output = Array>, { type Output = Array; @@ -192,7 +200,7 @@ where impl ops::Sub> for Bias where D: Dimension, - T: Float + ScalarOperand, + T: NdFloat, Array: ops::Sub, Output = Array>, { type Output = Array; @@ -209,7 +217,7 @@ where impl ops::Sub<&Array> for Bias where D: Dimension, - T: Float + ScalarOperand, + T: NdFloat, Array: ops::Sub, Output = Array>, { type Output = Array; @@ -226,7 +234,7 @@ where impl ops::Sub> for Array where D: Dimension, - T: Float + ScalarOperand, + T: NdFloat, Array: ops::Sub, Output = Array>, { type Output = Array; @@ -243,7 +251,7 @@ where impl ops::Sub<&Bias> for Array where D: Dimension, - T: Float + ScalarOperand, + T: NdFloat, Array: ops::Sub, Output = Array>, { type Output = Array; diff --git a/ml/neural/src/params/group.rs b/ml/neural/src/params/group.rs index 419c5406..69603844 100644 --- a/ml/neural/src/params/group.rs +++ b/ml/neural/src/params/group.rs @@ -2,7 +2,9 @@ Appellation: group Contrib: FL03 */ -use ndarray::prelude::{Array, Dimension, Ix2}; +use super::{Biased, Weighted}; +use ndarray::prelude::{Array, Axis, Dimension, Ix2}; +use ndarray::{IntoDimension, RemoveAxis}; use num::Float; #[derive(Clone, Debug, Eq, PartialEq)] @@ -16,19 +18,56 @@ where weights: Array, } -// impl ParamGroup -// where -// T: Float, -// D: Dimension + RemoveAxis, -// ::Smaller: Dimension, -// { -// pub fn new(dim: impl IntoDimension) -> Self { -// let dim = dim.into_dimension(); -// let bias = Array::zeros(); -// let weights = Array::zeros(dim.clone()); -// Self { -// bias: Array::::zeros(&weights.shape()[..(dim.ndim() - 1)]), -// weights, -// } -// } -// } +impl ParamGroup +where + T: Float, + D: Dimension + RemoveAxis, + ::Smaller: Dimension, +{ + pub fn new(dim: impl IntoDimension) -> Self { + let dim = dim.into_dimension(); + let smaller = dim.clone().remove_axis(Axis(dim.ndim() - 1)); + Self { + bias: Array::zeros(smaller), + weights: Array::zeros(dim), + } + } +} + +impl Biased for ParamGroup +where + T: Float, + D: Dimension + RemoveAxis, + ::Smaller: Dimension, +{ + fn bias(&self) -> &Array { + &self.bias + } + + fn bias_mut(&mut self) -> &mut Array { + &mut self.bias + } + + fn set_bias(&mut self, bias: Array) { + self.bias = bias; + } +} + +impl Weighted for ParamGroup +where + T: Float, + D: Dimension + RemoveAxis, + ::Smaller: Dimension, +{ + fn weights(&self) -> &Array { + &self.weights + } + + fn weights_mut(&mut self) -> &mut Array { + &mut self.weights + } + + fn set_weights(&mut self, weights: Array) { + self.weights = weights; + } +} diff --git a/ml/neural/src/prop/mod.rs b/ml/neural/src/prop/mod.rs index cd146909..6ddf7a7f 100644 --- a/ml/neural/src/prop/mod.rs +++ b/ml/neural/src/prop/mod.rs @@ -11,15 +11,14 @@ pub(crate) mod modes; pub(crate) mod propagation; // pub mod forward; -use ndarray::prelude::{Array, Array2, Dimension}; +use ndarray::prelude::{Array, Ix2}; -pub type ForwardDyn = Box, Output = Array2>>; +pub type ForwardDyn = Box, Output = Array>>; -pub trait Backward { - type Params; - type Output; +pub trait Backward: Forward { + type Optim; - fn backward(&mut self, args: &T, params: &Self::Params) -> Self::Output; + fn backward(&mut self, data: &T, opt: Self::Optim); } pub trait Forward { @@ -28,17 +27,4 @@ pub trait Forward { fn forward(&self, args: &T) -> Self::Output; } -pub trait Propagate { - type Optimizer; - - fn backward( - &mut self, - args: &Array2, - targets: &Array, - opt: Self::Optimizer, - ) -> Array; - - fn forward(&self, args: &Array2) -> Array2; -} - pub(crate) mod utils {} diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index 7d826863..d9643c4e 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -2,9 +2,18 @@ Appellation: specs Contrib: FL03 */ +use crate::layers::LayerDyn; use ndarray::prelude::{Array, Array2, Dimension, Ix2}; use num::Float; +pub trait FeedForward +where + D: Dimension, + T: Float, +{ + fn forward(&self, args: &Array2) -> Array; +} + pub trait Initializer where D: Dimension, @@ -13,7 +22,7 @@ where fn init_weight(&self) -> Array; } -pub trait Linear +pub trait LinearStep where D: Dimension, T: Float, @@ -27,7 +36,7 @@ pub trait Trainable { fn train(&mut self, args: &Array2, targets: &Array2) -> Array2; } -pub trait NetworkModel +pub trait NetworkModel: IntoIterator> where T: Float, { diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index 32d2ea15..26426384 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -26,7 +26,7 @@ pub struct DescentParams { } pub(crate) mod utils { - use crate::neural::prelude::{Activate, Forward, Layer, Parameterized, Weighted}; + use crate::neural::prelude::{Forward, Parameterized, Weighted}; use ndarray::prelude::{Array, Array1, Array2, Dimension, Ix2, NdFloat}; use ndarray_stats::DeviationExt; use num::{FromPrimitive, Signed}; @@ -100,7 +100,7 @@ mod tests { use super::*; use crate::core::prelude::linarr; - use crate::neural::func::activate::{LinearActivation, Sigmoid}; + use crate::neural::func::activate::{Linear, Objective, Sigmoid}; use crate::neural::prelude::{Layer, LayerShape, Parameterized, Weighted}; use ndarray::prelude::{Array1, Array2}; @@ -117,7 +117,7 @@ mod tests { let features = LayerShape::new(inputs, outputs); - let mut model = Layer::::from(features).init(true); + let mut model = Layer::::from(features).init(true); let losses = gradient_descent( &mut model.params_mut().weights_mut(), @@ -141,11 +141,11 @@ mod tests { let x = linarr((samples, features.inputs())).unwrap(); let y = linarr((samples, features.outputs())).unwrap(); - let mut model = Layer::::input(features).init(true); + let mut model = Layer::::input(features).init(true); let mut losses = Array1::zeros(epochs); for e in 0..epochs { - let cost = gradient(gamma, &mut model, &x, &y, Sigmoid::gradient); + let cost = gradient(gamma, &mut model, &x, &y, |w| Sigmoid::new().gradient(w)); losses[e] = cost; } assert_eq!(losses.len(), epochs); diff --git a/ml/optim/src/grad/sgd.rs b/ml/optim/src/grad/sgd.rs index 74227de6..3b00fdff 100644 --- a/ml/optim/src/grad/sgd.rs +++ b/ml/optim/src/grad/sgd.rs @@ -38,8 +38,6 @@ where let pred = model.forward(&xs); let error = &pred - &ys; - let grad_w = xs.dot(&error.t()).sum() * (-2.0 / batch_size as f64); - let grad_b = error.sum() * (-2.0 / batch_size as f64); for batch in (0..samples).step_by(batch_size) { let mut gradient = Array2::zeros((features.outputs(), features.inputs())); diff --git a/ml/optim/src/optimize/mod.rs b/ml/optim/src/optimize/mod.rs index 96d4d9ff..dd1f650c 100644 --- a/ml/optim/src/optimize/mod.rs +++ b/ml/optim/src/optimize/mod.rs @@ -9,8 +9,9 @@ pub use self::{optimizer::*, utils::*}; pub(crate) mod optimizer; pub trait Optimize { - // fn params(&self) -> &Params; - fn optimize(&self) -> Self; + type Model; + + fn apply(&self, model: &mut Self::Model) -> &mut Self::Model; } pub(crate) mod utils {} diff --git a/ml/optim/src/specs.rs b/ml/optim/src/specs.rs index d21c303c..4f3e018d 100644 --- a/ml/optim/src/specs.rs +++ b/ml/optim/src/specs.rs @@ -2,7 +2,7 @@ Appellation: specs Contrib: FL03 */ -use ndarray::prelude::{Array, Array1, Array2, Dimension}; +use ndarray::prelude::{Array, Dimension}; use num::Float; pub trait Gradient @@ -19,12 +19,6 @@ where } } -pub trait Objective { - type Model; - - fn objective(&self, args: &Array2) -> Array1; -} - pub trait Minimize { fn minimize(&self, scale: T) -> Self; } diff --git a/ml/transformers/src/attention/mod.rs b/ml/transformers/src/attention/mod.rs index dc616bf2..b5cb610c 100644 --- a/ml/transformers/src/attention/mod.rs +++ b/ml/transformers/src/attention/mod.rs @@ -23,7 +23,7 @@ use crate::prelude::BaseDim; use ndarray::prelude::{Array, Array2, Ix2, NdFloat}; use num::Float; -use std::ops::{self, Mul}; +use std::ops; /// (batch, sample, seq, model) pub type InputArray = Array; @@ -92,18 +92,6 @@ where } } -pub trait Weights: Mul, Output = Self> { - fn key(&self) -> &Array2; - - fn query(&self) -> &Array2; - - fn value(&self) -> &Array2; - - fn qkv(&self) -> (&Array2, &Array2, &Array2) { - (self.query(), self.key(), self.value()) - } -} - pub(crate) mod utils { use crate::neural::prelude::{Activate, Softmax}; use ndarray::prelude::{Array2, NdFloat}; diff --git a/ml/transformers/src/attention/multi/attention.rs b/ml/transformers/src/attention/multi/attention.rs index 6e90ff87..0d2dfa18 100644 --- a/ml/transformers/src/attention/multi/attention.rs +++ b/ml/transformers/src/attention/multi/attention.rs @@ -15,8 +15,8 @@ use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct MultiHeadAttention { + features: MultiHeadParams, linear: Layer, - params: MultiHeadParams, weights: Weight, } @@ -28,8 +28,8 @@ where &self.linear } - pub fn params(&self) -> MultiHeadParams { - self.params + pub fn features(&self) -> MultiHeadParams { + self.features } pub fn weights(&self) -> &Weight { @@ -42,11 +42,11 @@ where T: Float + SampleUniform, { pub fn new(heads: usize, model: usize) -> Self { - let params = MultiHeadParams::new(heads, model); + let features = MultiHeadParams::new(heads, model); let weights = Weight::uniform((model, model)); Self { + features, linear: Layer::input((model, model).into()), - params, weights, } } @@ -58,7 +58,7 @@ where { pub fn attention(&self, data: &Array2, mask: &Mask) -> Result, ShapeError> { let weighted = data * self.weights(); - let (q, k, v) = weighted.split(self.params().heads())?; + let (q, k, v) = weighted.split(self.features().heads())?; let score = multihead(&q, &k, &v, mask)?; let res = self.linear().forward(&score); Ok(res) @@ -83,10 +83,13 @@ where // } // } -// impl Forward> for MultiHeadAttention { -// type Output = Result, ShapeError>; +impl Forward> for MultiHeadAttention +where + T: NdFloat, +{ + type Output = Result, ShapeError>; -// fn forward(&self, data: &Array2) -> Self::Output { -// self.attention(data) -// } -// } + fn forward(&self, data: &Array2) -> Self::Output { + self.attention(&data, &Mask::Unmasked) + } +} diff --git a/ml/transformers/src/attention/weights.rs b/ml/transformers/src/attention/weights.rs index 9c8b892c..302db46e 100644 --- a/ml/transformers/src/attention/weights.rs +++ b/ml/transformers/src/attention/weights.rs @@ -25,7 +25,6 @@ //! //! use super::params::QKV; -use super::Weights; use crate::core::GenerateRandom; use crate::ops::Split; use ndarray::linalg::Dot; @@ -55,11 +54,26 @@ where pub value: Array2, } -impl Weight { +impl Weight +where + T: Float, +{ pub fn dim(&self) -> Ix2 { self.dim } + pub fn key(&self) -> &Array2 { + &self.key + } + + pub fn query(&self) -> &Array2 { + &self.query + } + + pub fn value(&self) -> &Array2 { + &self.value + } + pub fn qkv(&self) -> (Array2, Array2, Array2) { self.clone().into() } @@ -116,21 +130,6 @@ impl Split<(Array3, Array3, Array3)> for Weight { Ok((key.split(heads)?, query.split(heads)?, value.split(heads)?)) } } - -impl Weights for Weight { - fn key(&self) -> &Array2 { - &self.key - } - - fn query(&self) -> &Array2 { - &self.query - } - - fn value(&self) -> &Array2 { - &self.value - } -} - impl From for Weight where D: IntoDimension, From 7995f1117100a2e66d0b1ce35b72e5e33d089353 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 25 Nov 2023 12:28:28 -0600 Subject: [PATCH 067/118] update Signed-off-by: FL03 --- concision/examples/gradients.rs | 2 +- ml/neural/src/func/activate/activator.rs | 8 + ml/neural/src/func/activate/binary.rs | 53 ++++- ml/neural/src/func/activate/linear.rs | 44 ++++- ml/neural/src/func/activate/mod.rs | 132 ++----------- ml/neural/src/func/activate/nl/mod.rs | 119 ++++++++++++ ml/neural/src/func/activate/nl/relu.rs | 93 +++++++++ ml/neural/src/func/activate/nl/sigmoid.rs | 85 ++++++++ ml/neural/src/func/activate/nl/softmax.rs | 137 +++++++++++++ ml/neural/src/func/activate/nl/tanh.rs | 85 ++++++++ ml/neural/src/func/activate/nonlinear.rs | 227 ---------------------- ml/neural/src/layers/layer.rs | 36 +++- ml/neural/src/layers/mod.rs | 25 ++- ml/neural/src/neurons/mod.rs | 7 +- ml/neural/src/neurons/neuron.rs | 187 ++++++++++++++---- ml/neural/src/neurons/node.rs | 35 ++++ ml/neural/src/neurons/params.rs | 167 ---------------- ml/neural/src/params/mod.rs | 55 ++++-- 18 files changed, 909 insertions(+), 588 deletions(-) create mode 100644 ml/neural/src/func/activate/nl/mod.rs create mode 100644 ml/neural/src/func/activate/nl/relu.rs create mode 100644 ml/neural/src/func/activate/nl/sigmoid.rs create mode 100644 ml/neural/src/func/activate/nl/softmax.rs create mode 100644 ml/neural/src/func/activate/nl/tanh.rs delete mode 100644 ml/neural/src/func/activate/nonlinear.rs delete mode 100644 ml/neural/src/neurons/params.rs diff --git a/concision/examples/gradients.rs b/concision/examples/gradients.rs index 19131252..4cd07a1a 100644 --- a/concision/examples/gradients.rs +++ b/concision/examples/gradients.rs @@ -1,4 +1,4 @@ -use concision::prelude::{linarr, Forward, LayerShape, Weighted}; +use concision::prelude::{linarr, Forward, LayerShape, ParameterizedExt}; use concision::neural::prelude::{Layer, Neuron, Objective, Sigmoid}; use concision::optim::grad::*; diff --git a/ml/neural/src/func/activate/activator.rs b/ml/neural/src/func/activate/activator.rs index 948d402a..ebb7a5af 100644 --- a/ml/neural/src/func/activate/activator.rs +++ b/ml/neural/src/func/activate/activator.rs @@ -4,6 +4,14 @@ */ use super::Activate; +pub trait Activations { + fn linear() -> Activator + where + Self: Sized; +} + +pub struct A(Box>); + pub struct Activator { method: Box>, } diff --git a/ml/neural/src/func/activate/binary.rs b/ml/neural/src/func/activate/binary.rs index bd95ad08..2b7aa7ef 100644 --- a/ml/neural/src/func/activate/binary.rs +++ b/ml/neural/src/func/activate/binary.rs @@ -2,7 +2,6 @@ Appellation: binary Contrib: FL03 */ -use super::Activate; use ndarray::prelude::{Array, Dimension}; use num::{One, Zero}; use serde::{Deserialize, Serialize}; @@ -29,12 +28,56 @@ impl Heavyside { } } -impl Activate for Heavyside +// impl Activate for Heavyside +// where +// D: Dimension, +// T: Clone + One + PartialOrd + Zero, +// { +// fn activate(&self, args: &Array) -> Array { +// args.mapv(|x| Self::heavyside(&x)) +// } +// } + +// impl FnOnce<(&Array,)> for Heavyside +// where +// D: Dimension, +// T: Clone + One + PartialOrd + Zero, +// { +// type Output = Array; + +// extern "rust-call" fn call_once(self, args: (&Array,)) -> Array { +// args.mapv(|x| Self::heavyside(&x)) +// } +// } + +impl Fn<(&Array,)> for Heavyside +where + D: Dimension, + T: One + PartialOrd + Zero, +{ + extern "rust-call" fn call(&self, args: (&Array,)) -> Self::Output { + args.0.map(Heavyside::heavyside) + } +} + +impl FnMut<(&Array,)> for Heavyside +where + D: Dimension, + T: One + PartialOrd + Zero, +{ + extern "rust-call" fn call_mut(&mut self, args: (&Array,)) -> Self::Output { + args.0.map(Heavyside::heavyside) + } +} + +impl FnOnce<(&Array,)> for Heavyside where D: Dimension, - T: Clone + One + PartialOrd + Zero, + T: One + PartialOrd + Zero, { - fn activate(&self, args: &Array) -> Array { - args.mapv(|x| Self::heavyside(&x)) + type Output = Array; + + extern "rust-call" fn call_once(self, args: (&Array,)) -> Self::Output { + args.0.map(Heavyside::heavyside) } } diff --git a/ml/neural/src/func/activate/linear.rs b/ml/neural/src/func/activate/linear.rs index 9907a9b7..909f3e5a 100644 --- a/ml/neural/src/func/activate/linear.rs +++ b/ml/neural/src/func/activate/linear.rs @@ -2,7 +2,6 @@ Appellation: linear Contrib: FL03 */ -use super::{Activate, ActivationFn}; use ndarray::prelude::{Array, Dimension}; use num::One; use serde::{Deserialize, Serialize}; @@ -33,11 +32,11 @@ impl Linear { } pub fn linear(args: &T) -> T { - args.clone() + Linear::method()(args) } - pub fn method() -> ActivationFn { - |x| x + pub fn method() -> fn(&T) -> T { + |x| x.clone() } pub fn rho(args: T) -> T { @@ -45,12 +44,41 @@ impl Linear { } } -impl Activate for Linear +// impl Activate for Linear +// where +// D: Dimension, +// T: Clone, +// { +// fn activate(&self, args: &Array) -> Array { +// args.clone() +// } +// } + +impl Fn<(&T,)> for Linear +where + T: Clone, +{ + extern "rust-call" fn call(&self, args: (&T,)) -> T { + args.0.clone() + } +} + +impl FnMut<(&T,)> for Linear where - D: Dimension, T: Clone, { - fn activate(&self, args: &Array) -> Array { - args.clone() + extern "rust-call" fn call_mut(&mut self, args: (&T,)) -> T { + args.0.clone() + } +} + +impl FnOnce<(&T,)> for Linear +where + T: Clone, +{ + type Output = T; + + extern "rust-call" fn call_once(self, args: (&T,)) -> Self::Output { + args.0.clone() } } diff --git a/ml/neural/src/func/activate/mod.rs b/ml/neural/src/func/activate/mod.rs index 87c7e819..49426cea 100644 --- a/ml/neural/src/func/activate/mod.rs +++ b/ml/neural/src/func/activate/mod.rs @@ -5,12 +5,12 @@ //! # activate //! //! This module contains the activation functions for the neurons. -pub use self::{activator::*, binary::*, linear::*, nonlinear::*, utils::*}; +pub use self::{activator::*, binary::*, linear::*, nl::*, utils::*}; pub(crate) mod activator; pub(crate) mod binary; pub(crate) mod linear; -pub(crate) mod nonlinear; +pub(crate) mod nl; pub type ActivationFn = fn(T) -> T; @@ -25,18 +25,10 @@ where fn activate(&self, args: &Array) -> Array; } -// pub trait ActivateExt -// where -// D: Dimension, -// { -// fn new() -> Self; - -// fn method(&self) -> impl; -// } - -impl Activate for fn(&Array) -> Array +impl Activate for F where D: Dimension, + F: Fn(&Array) -> Array, { fn activate(&self, args: &Array) -> Array { self.call((args,)) @@ -59,14 +51,14 @@ where fn gradient(&self, args: &Array) -> Array; } -impl Objective for fn(&Array) -> Array -where - D: Dimension, -{ - fn gradient(&self, args: &Array) -> Array { - self.call((args,)) - } -} +// impl Objective for fn(&Array) -> Array +// where +// D: Dimension, +// { +// fn gradient(&self, args: &Array) -> Array { +// self.call((args,)) +// } +// } impl Activate for Box> where @@ -87,9 +79,7 @@ where } pub(crate) mod utils { - use ndarray::prelude::{Array, Axis, Dimension, NdFloat}; - use ndarray::RemoveAxis; - use num::{Float, One, Zero}; + use num::{One, Zero}; pub fn linear_activation(args: &T) -> T where @@ -108,70 +98,20 @@ pub(crate) mod utils { T::zero() } } - - pub fn relu(args: &T) -> T - where - T: Clone + PartialOrd + Zero, - { - if args > &T::zero() { - args.clone() - } else { - T::zero() - } - } - - pub fn sigmoid(args: &T) -> T - where - T: Float, - { - T::one() / (T::one() + (-args.clone()).exp()) - } - - pub fn softmax(args: &Array) -> Array - where - D: Dimension, - T: Float, - { - let denom = args.mapv(|x| x.exp()).sum(); - args.mapv(|x| x.exp() / denom) - } - - pub fn softmax_axis(args: &Array, axis: Option) -> Array - where - D: Dimension + RemoveAxis, - T: NdFloat, - { - let exp = args.mapv(|x| x.exp()); - if let Some(axis) = axis { - let denom = exp.sum_axis(Axis(axis)); - exp / denom - } else { - let denom = exp.sum(); - exp / denom - } - } - - pub fn tanh(args: &T) -> T - where - T: Float, - { - args.tanh() - } } #[cfg(test)] mod tests { use super::*; - use computare::prelude::RoundTo; use ndarray::array; #[test] fn test_heavyside() { - let exp = array![0.0, 1.0, 1.0]; + let exp = array![0.0, 0.0, 1.0]; let args = array![-1.0, 0.0, 1.0]; - let res = Heavyside::new().activate(&args); - assert_eq!(res, exp); + assert_eq!(Heavyside::new().activate(&args), exp); + assert_eq!(Heavyside(&args), exp); } #[test] @@ -179,43 +119,7 @@ mod tests { let exp = array![0.0, 1.0, 2.0]; let args = array![0.0, 1.0, 2.0]; - let res = Linear::new().activate(&args); - assert_eq!(res, exp); - } - - #[test] - fn test_relu() { - let exp = array![0.0, 0.0, 3.0]; - let args = array![-1.0, 0.0, 3.0]; - - let res = ReLU::new().activate(&args); - assert_eq!(res, exp); - } - - #[test] - fn test_sigmoid() { - let exp = array![0.73105858, 0.88079708, 0.95257413]; - let args = array![1.0, 2.0, 3.0]; - - let res = Sigmoid::new().activate(&args).mapv(|i| i.round_to(8)); - assert_eq!(res, exp); - } - - #[test] - fn test_softmax() { - let exp = array![0.09003057, 0.24472847, 0.66524096]; - let args = array![1.0, 2.0, 3.0]; - - let res = Softmax::new(None).activate(&args).mapv(|i| i.round_to(8)); - assert_eq!(res, exp); - } - - #[test] - fn test_tanh() { - let exp = array![0.76159416, 0.96402758, 0.99505475]; - let args = array![1.0, 2.0, 3.0]; - - let res = Tanh::new().activate(&args).mapv(|i| i.round_to(8)); - assert_eq!(res, exp); + assert_eq!(Linear::new().activate(&args), exp); + assert_eq!(Linear(&args), exp); } } diff --git a/ml/neural/src/func/activate/nl/mod.rs b/ml/neural/src/func/activate/nl/mod.rs new file mode 100644 index 00000000..bd59a74a --- /dev/null +++ b/ml/neural/src/func/activate/nl/mod.rs @@ -0,0 +1,119 @@ +/* + Appellation: nonlinear + Contrib: FL03 +*/ +//! # NonLinear Activation Functions +//! +//! +pub use self::{relu::*, sigmoid::*, softmax::*, tanh::*, utils::*}; + +pub(crate) mod relu; +pub(crate) mod sigmoid; +pub(crate) mod softmax; +pub(crate) mod tanh; + +pub(crate) mod utils { + use ndarray::prelude::{Array, Axis, Dimension, NdFloat}; + use ndarray::RemoveAxis; + use num::{Float, Zero}; + + pub fn relu(args: &T) -> T + where + T: Clone + PartialOrd + Zero, + { + if args > &T::zero() { + args.clone() + } else { + T::zero() + } + } + + pub fn sigmoid(args: &T) -> T + where + T: Float, + { + T::one() / (T::one() + (-args.clone()).exp()) + } + + pub fn softmax(args: &Array) -> Array + where + D: Dimension, + T: Float, + { + let denom = args.mapv(|x| x.exp()).sum(); + args.mapv(|x| x.exp() / denom) + } + + pub fn softmax_axis(args: &Array, axis: Option) -> Array + where + D: Dimension + RemoveAxis, + T: NdFloat, + { + let exp = args.mapv(|x| x.exp()); + if let Some(axis) = axis { + let denom = exp.sum_axis(Axis(axis)); + exp / denom + } else { + let denom = exp.sum(); + exp / denom + } + } + + pub fn tanh(args: &T) -> T + where + T: Float, + { + args.tanh() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::prelude::Activate; + use computare::prelude::RoundTo; + use ndarray::array; + + #[test] + fn test_relu() { + let exp = array![0.0, 0.0, 3.0]; + let args = array![-1.0, 0.0, 3.0]; + + let res = ReLU::new().activate(&args); + assert_eq!(res, exp); + assert_eq!(ReLU(&args), exp); + } + + #[test] + fn test_sigmoid() { + let exp = array![0.73105858, 0.88079708, 0.95257413]; + let args = array![1.0, 2.0, 3.0]; + + let res = Sigmoid::new().activate(&args).mapv(|i| i.round_to(8)); + assert_eq!(res, exp); + let res = Sigmoid(&args).mapv(|i| i.round_to(8)); + assert_eq!(res, exp); + } + + #[test] + fn test_softmax() { + let exp = array![0.09003057, 0.24472847, 0.66524096]; + let args = array![1.0, 2.0, 3.0]; + + let res = Softmax::new(None).activate(&args).mapv(|i| i.round_to(8)); + assert_eq!(res, exp); + let res = Softmax::new(None)(&args).mapv(|i| i.round_to(8)); + assert_eq!(res, exp); + } + + #[test] + fn test_tanh() { + let exp = array![0.76159416, 0.96402758, 0.99505475]; + let args = array![1.0, 2.0, 3.0]; + + let res = Tanh::new().activate(&args).mapv(|i| i.round_to(8)); + assert_eq!(res, exp); + let res = Tanh(&args).mapv(|i| i.round_to(8)); + assert_eq!(res, exp); + } +} diff --git a/ml/neural/src/func/activate/nl/relu.rs b/ml/neural/src/func/activate/nl/relu.rs new file mode 100644 index 00000000..ed09e259 --- /dev/null +++ b/ml/neural/src/func/activate/nl/relu.rs @@ -0,0 +1,93 @@ +/* + Appellation: relu + Contrib: FL03 +*/ +use crate::func::activate::Objective; +use ndarray::prelude::{Array, Dimension}; +use num::{One, Zero}; +use serde::{Deserialize, Serialize}; + +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] +pub struct ReLU; + +impl ReLU { + pub fn new() -> Self { + Self + } + + pub fn derivative(args: &T) -> T + where + T: One + PartialOrd + Zero, + { + if args > &T::zero() { + T::one() + } else { + T::zero() + } + } + + pub fn relu(args: &T) -> T + where + T: Clone + PartialOrd + Zero, + { + if args > &T::zero() { + args.clone() + } else { + T::zero() + } + } +} + +// impl Activate for ReLU +// where +// D: Dimension, +// T: Clone + PartialOrd + Zero, +// { +// fn activate(&self, x: &Array) -> Array { +// x.map(Self::relu) +// } +// } + +impl Objective for ReLU +where + D: Dimension, + T: Clone + One + PartialOrd + Zero, +{ + fn gradient(&self, args: &Array) -> Array { + args.map(Self::derivative) + } +} + +impl Fn<(&Array,)> for ReLU +where + D: Dimension, + T: Clone + PartialOrd + Zero, +{ + extern "rust-call" fn call(&self, args: (&Array,)) -> Self::Output { + args.0.map(Self::relu) + } +} + +impl FnMut<(&Array,)> for ReLU +where + D: Dimension, + T: Clone + PartialOrd + Zero, +{ + extern "rust-call" fn call_mut(&mut self, args: (&Array,)) -> Self::Output { + args.0.map(Self::relu) + } +} + +impl FnOnce<(&Array,)> for ReLU +where + D: Dimension, + T: Clone + PartialOrd + Zero, +{ + type Output = Array; + + extern "rust-call" fn call_once(self, args: (&Array,)) -> Self::Output { + args.0.map(Self::relu) + } +} diff --git a/ml/neural/src/func/activate/nl/sigmoid.rs b/ml/neural/src/func/activate/nl/sigmoid.rs new file mode 100644 index 00000000..e294a76f --- /dev/null +++ b/ml/neural/src/func/activate/nl/sigmoid.rs @@ -0,0 +1,85 @@ +/* + Appellation: sigmoid + Contrib: FL03 +*/ +use crate::func::activate::Objective; +use ndarray::prelude::{Array, Dimension}; +use num::Float; +use serde::{Deserialize, Serialize}; + +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] +pub struct Sigmoid; + +impl Sigmoid { + pub fn new() -> Self { + Self + } + + pub fn derivative(x: T) -> T + where + T: Float, + { + (-x).exp() / (T::one() + (-x).exp()).powi(2) + } + + pub fn sigmoid(x: T) -> T + where + T: Float, + { + T::one() / (T::one() + (-x).exp()) + } +} + +// impl Activate for Sigmoid +// where +// D: Dimension, +// T: Float, +// { +// fn activate(&self, x: &Array) -> Array { +// x.mapv(|x| Self::sigmoid(x)) +// } +// } + +impl Objective for Sigmoid +where + D: Dimension, + T: Float, +{ + fn gradient(&self, args: &Array) -> Array { + args.mapv(|x| Self::derivative(x)) + } +} + +impl Fn<(&Array,)> for Sigmoid +where + D: Dimension, + T: Float, +{ + extern "rust-call" fn call(&self, args: (&Array,)) -> Self::Output { + args.0.mapv(|x| Self::sigmoid(x)) + } +} + +impl FnMut<(&Array,)> for Sigmoid +where + D: Dimension, + T: Float, +{ + extern "rust-call" fn call_mut(&mut self, args: (&Array,)) -> Self::Output { + args.0.mapv(|x| Self::sigmoid(x)) + } +} + +impl FnOnce<(&Array,)> for Sigmoid +where + D: Dimension, + T: Float, +{ + type Output = Array; + + extern "rust-call" fn call_once(self, args: (&Array,)) -> Self::Output { + args.0.mapv(|x| Self::sigmoid(x)) + } +} diff --git a/ml/neural/src/func/activate/nl/softmax.rs b/ml/neural/src/func/activate/nl/softmax.rs new file mode 100644 index 00000000..6acd484b --- /dev/null +++ b/ml/neural/src/func/activate/nl/softmax.rs @@ -0,0 +1,137 @@ +/* + Appellation: softmax + Contrib: FL03 +*/ +use crate::func::activate::Objective; +use ndarray::prelude::{Array, Axis, Dimension, NdFloat}; +use ndarray::RemoveAxis; +use num::Float; +use serde::{Deserialize, Serialize}; + +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] +pub struct Softmax { + axis: Option, +} + +impl Softmax { + pub fn new(axis: Option) -> Self { + Self { axis } + } + + pub fn axis(&self) -> Option { + self.axis + } + + pub fn softmax(args: Array) -> Array + where + D: Dimension, + T: Float, + { + let denom = args.mapv(|x| x.exp()).sum(); + args.mapv(|x| x.exp() / denom) + } + + pub fn softmax_axis(&self, args: Array) -> Array + where + T: NdFloat, + D: Dimension + RemoveAxis, + { + let exp = args.mapv(|x| x.exp()); + if let Some(axis) = self.axis { + let denom = exp.sum_axis(Axis(axis)); + exp / denom + } else { + let denom = exp.sum(); + exp / denom + } + } +} + +// impl Activate for Softmax +// where +// D: Dimension + RemoveAxis, +// T: NdFloat, +// { +// fn activate(&self, x: &Array) -> Array { +// let exp = x.mapv(|x| x.exp()); +// if let Some(axis) = self.axis { +// let denom = exp.sum_axis(Axis(axis)); +// exp / denom +// } else { +// let denom = exp.sum(); +// exp / denom +// } +// } +// } + +impl Objective for Softmax +where + D: Dimension + RemoveAxis, + T: NdFloat, +{ + fn gradient(&self, args: &Array) -> Array { + let exp = args.mapv(|x| x.exp()); + if let Some(axis) = self.axis { + let denom = exp.sum_axis(Axis(axis)); + exp / denom + } else { + let denom = exp.sum(); + exp / denom + } + } +} + +impl Fn<(&Array,)> for Softmax +where + D: Dimension + RemoveAxis, + T: NdFloat, +{ + extern "rust-call" fn call(&self, args: (&Array,)) -> Self::Output { + let exp = args.0.mapv(|x| x.exp()); + if let Some(axis) = self.axis { + let denom = exp.sum_axis(Axis(axis)); + exp / denom + } else { + let denom = exp.sum(); + exp / denom + } + } +} + +impl FnMut<(&Array,)> for Softmax +where + D: Dimension + RemoveAxis, + T: NdFloat, +{ + extern "rust-call" fn call_mut(&mut self, args: (&Array,)) -> Self::Output { + let exp = args.0.mapv(|x| x.exp()); + if let Some(axis) = self.axis { + let denom = exp.sum_axis(Axis(axis)); + exp / denom + } else { + let denom = exp.sum(); + exp / denom + } + } +} + +impl FnOnce<(&Array,)> for Softmax +where + D: Dimension + RemoveAxis, + T: NdFloat, +{ + type Output = Array; + + extern "rust-call" fn call_once(self, args: (&Array,)) -> Self::Output { + let exp = args.0.mapv(|x| x.exp()); + if let Some(axis) = self.axis { + let denom = exp.sum_axis(Axis(axis)); + exp / denom + } else { + let denom = exp.sum(); + exp / denom + } + } +} diff --git a/ml/neural/src/func/activate/nl/tanh.rs b/ml/neural/src/func/activate/nl/tanh.rs new file mode 100644 index 00000000..b0865502 --- /dev/null +++ b/ml/neural/src/func/activate/nl/tanh.rs @@ -0,0 +1,85 @@ +/* + Appellation: tanh + Contrib: FL03 +*/ +use crate::func::activate::Objective; +use ndarray::prelude::{Array, Dimension}; +use num::Float; +use serde::{Deserialize, Serialize}; + +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] +pub struct Tanh; + +impl Tanh { + pub fn new() -> Self { + Self + } + + pub fn derivative(args: &T) -> T + where + T: Float, + { + T::one() - args.tanh().powi(2) + } + + pub fn tanh(args: &T) -> T + where + T: Float, + { + args.tanh() + } +} + +// impl Activate for Tanh +// where +// D: Dimension, +// T: Float, +// { +// fn activate(&self, x: &Array) -> Array { +// x.mapv(Float::tanh) +// } +// } + +impl Objective for Tanh +where + D: Dimension, + T: Float, +{ + fn gradient(&self, args: &Array) -> Array { + args.mapv(|x| Self::derivative(&x)) + } +} + +impl Fn<(&Array,)> for Tanh +where + D: Dimension, + T: Float, +{ + extern "rust-call" fn call(&self, args: (&Array,)) -> Self::Output { + args.0.mapv(T::tanh) + } +} + +impl FnMut<(&Array,)> for Tanh +where + D: Dimension, + T: Float, +{ + extern "rust-call" fn call_mut(&mut self, args: (&Array,)) -> Self::Output { + args.0.mapv(T::tanh) + } +} + +impl FnOnce<(&Array,)> for Tanh +where + D: Dimension, + T: Float, +{ + type Output = Array; + + extern "rust-call" fn call_once(self, args: (&Array,)) -> Self::Output { + args.0.mapv(T::tanh) + } +} diff --git a/ml/neural/src/func/activate/nonlinear.rs b/ml/neural/src/func/activate/nonlinear.rs deleted file mode 100644 index 6802b479..00000000 --- a/ml/neural/src/func/activate/nonlinear.rs +++ /dev/null @@ -1,227 +0,0 @@ -/* - Appellation: nonlinear - Contrib: FL03 -*/ -use super::{Activate, Objective}; -use ndarray::prelude::{Array, Axis, Dimension, NdFloat}; -use ndarray::RemoveAxis; -use num::{Float, One, Zero}; -use serde::{Deserialize, Serialize}; - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct ReLU; - -impl ReLU { - pub fn new() -> Self { - Self - } - - pub fn derivative(x: T) -> T - where - T: One + PartialOrd + Zero, - { - if x > T::zero() { - T::one() - } else { - T::zero() - } - } - - pub fn relu(args: &T) -> T - where - T: Clone + PartialOrd + Zero, - { - if args > &T::zero() { - args.clone() - } else { - T::zero() - } - } -} - -impl Activate for ReLU -where - D: Dimension, - T: Clone + PartialOrd + Zero, -{ - fn activate(&self, x: &Array) -> Array { - x.mapv(|x| Self::relu(&x)) - } -} - -impl Objective for ReLU -where - D: Dimension, - T: Clone + One + PartialOrd + Zero, -{ - fn gradient(&self, args: &Array) -> Array { - args.mapv(|x| Self::derivative(x)) - } -} - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct Sigmoid; - -impl Sigmoid { - pub fn new() -> Self { - Self - } - - pub fn derivative(x: T) -> T - where - T: Float, - { - (-x).exp() / (T::one() + (-x).exp()).powi(2) - } - - pub fn sigmoid(x: T) -> T - where - T: Float, - { - T::one() / (T::one() + (-x).exp()) - } -} - -impl Activate for Sigmoid -where - D: Dimension, - T: Float, -{ - fn activate(&self, x: &Array) -> Array { - x.mapv(|x| Self::sigmoid(x)) - } -} - -impl Objective for Sigmoid -where - D: Dimension, - T: Float, -{ - fn gradient(&self, args: &Array) -> Array { - args.mapv(|x| Self::derivative(x)) - } -} - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct Softmax { - axis: Option, -} - -impl Softmax { - pub fn new(axis: Option) -> Self { - Self { axis } - } - - pub fn axis(&self) -> Option { - self.axis - } - - pub fn softmax(args: Array) -> Array - where - D: Dimension, - T: Float, - { - let denom = args.mapv(|x| x.exp()).sum(); - args.mapv(|x| x.exp() / denom) - } - - pub fn softmax_axis(&self, args: Array) -> Array - where - T: NdFloat, - D: Dimension + RemoveAxis, - { - let exp = args.mapv(|x| x.exp()); - if let Some(axis) = self.axis { - let denom = exp.sum_axis(Axis(axis)); - exp / denom - } else { - let denom = exp.sum(); - exp / denom - } - } -} - -impl Activate for Softmax -where - D: Dimension + RemoveAxis, - T: NdFloat, -{ - fn activate(&self, x: &Array) -> Array { - let exp = x.mapv(|x| x.exp()); - if let Some(axis) = self.axis { - let denom = exp.sum_axis(Axis(axis)); - exp / denom - } else { - let denom = exp.sum(); - exp / denom - } - } -} - -impl Objective for Softmax -where - D: Dimension + RemoveAxis, - T: NdFloat, -{ - fn gradient(&self, args: &Array) -> Array { - let exp = args.mapv(|x| x.exp()); - if let Some(axis) = self.axis { - let denom = exp.sum_axis(Axis(axis)); - exp / denom - } else { - let denom = exp.sum(); - exp / denom - } - } -} - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct Tanh; - -impl Tanh { - pub fn new() -> Self { - Self - } - - pub fn derivative(args: &T) -> T - where - T: Float, - { - T::one() - args.tanh().powi(2) - } - - pub fn tanh(args: &T) -> T - where - T: Float, - { - args.tanh() - } -} - -impl Activate for Tanh -where - D: Dimension, - T: Float, -{ - fn activate(&self, x: &Array) -> Array { - x.mapv(Float::tanh) - } -} - -impl Objective for Tanh -where - D: Dimension, - T: Float, -{ - fn gradient(&self, args: &Array) -> Array { - args.mapv(|x| Self::derivative(&x)) - } -} diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 88725948..af740aa3 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -3,8 +3,8 @@ Contrib: FL03 */ use super::{LayerKind, LayerParams, LayerPosition, LayerShape}; -use crate::prelude::{Activate, Forward, Linear, Parameterized, Params}; -use ndarray::prelude::{Array2, Ix2, NdFloat}; +use crate::prelude::{Activate, Forward, Linear, Neuron, Parameterized, Params}; +use ndarray::prelude::{Array2, Axis, Ix1, Ix2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; @@ -210,3 +210,35 @@ where Self::new(features, LayerPosition::input()) } } + +// impl IntoIterator for Layer +// where +// A: Activate, +// T: Float, +// { +// type Item = (Array1, Array0); +// type IntoIter = std::vec::IntoIter; + +// fn into_iter(self) -> Self::IntoIter { +// self.params().weights().axis_iter(Axis(0)).zip(self.params().bias().axis_iter(Axis(0))).map(|(w, b)| (w.to_owned(), b.to_owned())).collect::>().into_iter() +// } +// } + +impl IntoIterator for Layer +where + A: Activate + Activate + Default, + T: Float, +{ + type Item = Neuron; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.params() + .weights() + .axis_iter(Axis(0)) + .zip(self.params().bias().axis_iter(Axis(0))) + .map(|(w, b)| (w.to_owned(), b.to_owned()).into()) + .collect::>() + .into_iter() + } +} diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index 1fb1ac3e..d44fc6d5 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -14,18 +14,18 @@ pub(crate) mod sublayer; use crate::func::activate::{Activate, ActivateDyn}; use crate::prelude::Forward; use ndarray::prelude::{Array2, Ix2}; +// use ndarray::IntoDimension; use num::Float; pub type LayerDyn = Layer>; pub trait L: Forward> { - fn features(&self) -> &LayerShape; - - fn features_mut(&mut self) -> &mut LayerShape; - + fn features(&self) -> LayerShape; + fn name(&self) -> &str; fn params(&self) -> &LayerParams; + fn position(&self) -> LayerPosition; - fn params_mut(&mut self) -> &mut LayerParams; + fn is_biased(&self) -> bool; } pub trait LayerExt: L @@ -42,7 +42,7 @@ mod tests { use super::*; use crate::core::prelude::linarr; use crate::func::activate::Softmax; - use crate::prelude::Forward; + use crate::prelude::{Forward, ParameterizedExt}; use ndarray::prelude::Ix2; #[test] @@ -58,4 +58,17 @@ mod tests { assert_eq!(pred.dim(), (samples, outputs)); } + + #[test] + fn test_layer_iter() { + let (_samples, inputs, outputs) = (20, 5, 3); + let features = LayerShape::new(inputs, outputs); + + let layer = Layer::::from(features).init(true); + + for neuron in layer.into_iter() { + assert_eq!(neuron.features(), inputs); + assert_eq!(neuron.bias().dim(), ()); + } + } } diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index 1bd40ff3..3b3110a0 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -3,11 +3,10 @@ Contrib: FL03 */ //! # neurons -pub use self::{neuron::*, node::*, params::*, synapse::*, utils::*}; +pub use self::{neuron::*, node::*, synapse::*, utils::*}; pub(crate) mod neuron; pub(crate) mod node; -pub(crate) mod params; pub(crate) mod synapse; use crate::func::activate::Activate; @@ -61,9 +60,7 @@ mod tests { let data = array![[10.0, 10.0, 6.0, 1.0, 8.0]]; let weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; - let neuron = Neuron::new(5) - .with_rho(Softmax::default()) - .with_weights(weights.clone()); + let neuron = Neuron::::new(5).with_weights(weights.clone()); let linear = data.dot(&weights) + bias; let exp = softmax(&linear); diff --git a/ml/neural/src/neurons/neuron.rs b/ml/neural/src/neurons/neuron.rs index 9d1217fa..11ed789d 100644 --- a/ml/neural/src/neurons/neuron.rs +++ b/ml/neural/src/neurons/neuron.rs @@ -2,9 +2,9 @@ Appellation: neuron Contrib: FL03 */ -use crate::core::GenerateRandom; +use super::Node; use crate::func::activate::{Activate, Linear}; -use crate::prelude::{Biased, Forward, Weighted}; +use crate::prelude::{Forward, Parameterized, ParameterizedExt, Weighted}; use ndarray::prelude::{Array0, Array1, Array2, Ix1, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; @@ -17,9 +17,8 @@ where T: Float, { activation: A, - bias: Array0, features: usize, - weights: Array1, + node: Node, } impl Neuron @@ -27,22 +26,46 @@ where A: Activate, T: Float, { + pub fn activation(&self) -> &A { + &self.activation + } + + pub fn features(&self) -> usize { + self.features + } + + pub fn node(&self) -> &Node { + &self.node + } + + pub fn node_mut(&mut self) -> &mut Node { + &mut self.node + } + pub fn rho(&self) -> &A { &self.activation } pub fn with_bias(mut self, bias: Array0) -> Self { - self.bias = bias; + self.node = self.node.with_bias(bias); self } - pub fn with_rho(mut self, rho: A) -> Self { - self.activation = rho; + pub fn with_rho>(self, rho: B) -> Neuron { + Neuron { + activation: rho, + features: self.features, + node: self.node, + } + } + + pub fn with_node(mut self, node: Node) -> Self { + self.node = node; self } pub fn with_weights(mut self, weights: Array1) -> Self { - self.weights = weights; + self.node = self.node.with_weights(weights); self } } @@ -55,9 +78,8 @@ where pub fn new(features: usize) -> Self { Self { activation: A::default(), - bias: Array0::zeros(()), features, - weights: Array1::zeros(features), + node: Node::new(features), } } } @@ -71,7 +93,7 @@ where where G: Fn(&Array1) -> Array1, { - let grad = gradient(&self.weights); + let grad = gradient(self.node().weights()); self.update_with_gradient(gamma, &grad); } @@ -93,52 +115,75 @@ where } pub fn init_bias(mut self) -> Self { - let dk = (T::one() / T::from(self.features).unwrap()).sqrt(); - self.bias = Array0::uniform_between(dk, ()); + self.node = self.node.init_bias(); self } pub fn init_weight(mut self) -> Self { - let features = self.features; - let dk = (T::one() / T::from(features).unwrap()).sqrt(); - self.weights = Array1::uniform_between(dk, features); + self.node = self.node.init_weight(); self } } -impl Biased for Neuron +// impl Biased for Neuron +// where +// T: Float, +// A: Activate, +// { +// fn bias(&self) -> &Array0 { +// self.node.bias() +// } + +// fn bias_mut(&mut self) -> &mut Array0 { +// self.node.bias_mut() +// } + +// fn set_bias(&mut self, bias: Array0) { +// self.node.set_bias(bias); +// } +// } + +// impl Weighted for Neuron +// where +// T: Float, +// A: Activate, +// { +// fn weights(&self) -> &Array1 { +// self.node.weights() +// } + +// fn weights_mut(&mut self) -> &mut Array1 { +// self.node.weights_mut() +// } + +// fn set_weights(&mut self, weights: Array1) { +// self.node.set_weights(weights); +// } +// } + +impl Parameterized for Neuron where - T: NdFloat, A: Activate, + T: Float, { - fn bias(&self) -> &Array0 { - &self.bias - } + type Features = usize; - fn bias_mut(&mut self) -> &mut Array0 { - &mut self.bias - } + type Params = Node; - fn set_bias(&mut self, bias: Array0) { - self.bias = bias; + fn features(&self) -> &Self::Features { + &self.features } -} -impl Weighted for Neuron -where - T: NdFloat, - A: Activate, -{ - fn weights(&self) -> &Array1 { - &self.weights + fn features_mut(&mut self) -> &mut Self::Features { + &mut self.features } - fn weights_mut(&mut self) -> &mut Array1 { - &mut self.weights + fn params(&self) -> &Self::Params { + &self.node } - fn set_weights(&mut self, weights: Array1) { - self.weights = weights; + fn params_mut(&mut self) -> &mut Self::Params { + &mut self.node } } @@ -163,3 +208,69 @@ where self.rho().activate(&linstep) } } + +impl From<(Array1, Array0)> for Neuron +where + T: Float, + A: Activate + Default, +{ + fn from((weights, bias): (Array1, Array0)) -> Self { + Self { + activation: A::default(), + features: weights.len(), + node: Node::from((weights, bias)), + } + } +} + +impl From<(Array1, T)> for Neuron +where + T: NdFloat, + A: Activate + Default, +{ + fn from((weights, bias): (Array1, T)) -> Self { + Self { + activation: A::default(), + features: weights.len(), + node: Node::from((weights, bias)), + } + } +} + +impl From<(Array1, Array0, A)> for Neuron +where + T: Float, + A: Activate, +{ + fn from((weights, bias, activation): (Array1, Array0, A)) -> Self { + Self { + activation, + features: weights.len(), + node: Node::from((weights, bias)), + } + } +} + +impl From<(Array1, T, A)> for Neuron +where + T: NdFloat, + A: Activate, +{ + fn from((weights, bias, activation): (Array1, T, A)) -> Self { + Self { + activation, + features: weights.len(), + node: Node::from((weights, bias)), + } + } +} + +impl From> for (Array1, Array0) +where + T: Float, + A: Activate, +{ + fn from(neuron: Neuron) -> Self { + neuron.node().clone().into() + } +} diff --git a/ml/neural/src/neurons/node.rs b/ml/neural/src/neurons/node.rs index b76f481f..0faae4a5 100644 --- a/ml/neural/src/neurons/node.rs +++ b/ml/neural/src/neurons/node.rs @@ -152,3 +152,38 @@ where &mut self.weights } } + +impl From<(Array1, Array0)> for Node +where + T: Float, +{ + fn from((weights, bias): (Array1, Array0)) -> Self { + Self { + bias, + features: weights.len(), + weights, + } + } +} + +impl From<(Array1, T)> for Node +where + T: NdFloat, +{ + fn from((weights, bias): (Array1, T)) -> Self { + Self { + bias: Array0::ones(()) * bias, + features: weights.len(), + weights, + } + } +} + +impl From> for (Array1, Array0) +where + T: Float, +{ + fn from(node: Node) -> Self { + (node.weights, node.bias) + } +} diff --git a/ml/neural/src/neurons/params.rs b/ml/neural/src/neurons/params.rs deleted file mode 100644 index 2250ac74..00000000 --- a/ml/neural/src/neurons/params.rs +++ /dev/null @@ -1,167 +0,0 @@ -/* - Appellation: params - Contrib: FL03 -*/ -use crate::core::prelude::GenerateRandom; - -use crate::prelude::{Biased, Weighted}; -use ndarray::prelude::{Array0, Array1, Array2, Ix1, NdFloat}; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use num::{Float, FromPrimitive}; - -#[derive(Clone, Debug, PartialEq)] -pub struct NeuronParams -where - T: Float, -{ - bias: Array0, - pub features: usize, - weights: Array1, -} - -impl NeuronParams -where - T: Float, -{ - pub fn new(features: usize) -> Self { - Self { - bias: Array0::zeros(()), - features, - weights: Array1::zeros(features), - } - } - - pub fn features(&self) -> usize { - self.features - } - - pub fn features_mut(&mut self) -> &mut usize { - &mut self.features - } - - pub fn set_features(&mut self, features: usize) { - self.features = features; - } - - pub fn with_bias(mut self, bias: Array0) -> Self { - self.bias = bias.into(); - self - } - - pub fn with_features(mut self, features: usize) -> Self { - self.features = features; - self - } - - pub fn with_weights(mut self, weights: Array1) -> Self { - self.weights = weights; - self - } -} - -impl NeuronParams -where - T: Float + SampleUniform, -{ - pub fn init(mut self, biased: bool) -> Self { - if biased { - self = self.init_bias(); - } - self.init_weight() - } - - pub fn init_bias(mut self) -> Self { - let dk = (T::one() / T::from(self.features).unwrap()).sqrt(); - self.bias = Array0::uniform_between(dk, ()); - self - } - - pub fn init_weight(mut self) -> Self { - let features = self.features; - let dk = (T::one() / T::from(features).unwrap()).sqrt(); - self.weights = Array1::uniform_between(dk, features); - self - } -} - -impl NeuronParams -where - T: FromPrimitive + NdFloat, -{ - pub fn apply_gradient(&mut self, gamma: T, gradient: G) - where - G: Fn(&Array1) -> Array1, - { - let mut grad = gradient(self.weights()); - grad /= grad.mapv(|ws| ws.powi(2)).sum().sqrt(); - self.weights_mut().scaled_add(-gamma, &grad); - self.weights /= self.weights().mapv(|ws| ws.powi(2)).sum().sqrt(); - } - - pub fn linear(&self, data: &Array2) -> Array1 { - data.dot(&self.weights().t()) + self.bias() - } -} - -// impl Params for NeuronParams -// where -// T: Float, -// { -// fn bias(&self) -> &Array0 { -// &self.bias -// } - -// fn bias_mut(&mut self) -> &mut Array0 { -// &mut self.bias -// } - -// fn weights(&self) -> &Array1 { -// &self.weights -// } - -// fn weights_mut(&mut self) -> &mut Array1 { -// &mut self.weights -// } - -// fn set_bias(&mut self, bias: Array0) { -// self.bias = bias; -// } - -// fn set_weights(&mut self, weights: Array1) { -// self.weights = weights; -// } -// } - -impl Biased for NeuronParams -where - T: Float, -{ - fn bias(&self) -> &Array0 { - &self.bias - } - - fn bias_mut(&mut self) -> &mut Array0 { - &mut self.bias - } - - fn set_bias(&mut self, bias: Array0) { - self.bias = bias; - } -} - -impl Weighted for NeuronParams -where - T: Float, -{ - fn set_weights(&mut self, weights: Array1) { - self.weights = weights; - } - - fn weights(&self) -> &Array1 { - &self.weights - } - - fn weights_mut(&mut self) -> &mut Array1 { - &mut self.weights - } -} diff --git a/ml/neural/src/params/mod.rs b/ml/neural/src/params/mod.rs index 9d217392..1f967b19 100644 --- a/ml/neural/src/params/mod.rs +++ b/ml/neural/src/params/mod.rs @@ -95,6 +95,46 @@ where fn params_mut(&mut self) -> &mut Self::Params; } +pub trait ParameterizedExt: Parameterized +where + D: Dimension, + T: Float, + >::Params: Params + 'static, +{ + fn bias(&self) -> &Array { + Biased::bias(self.params()) + } + + fn bias_mut(&mut self) -> &mut Array { + Biased::bias_mut(self.params_mut()) + } + + fn weights(&self) -> &Array { + Weighted::weights(self.params()) + } + + fn weights_mut(&mut self) -> &mut Array { + Weighted::weights_mut(self.params_mut()) + } + + fn set_bias(&mut self, bias: Array) { + Biased::set_bias(self.params_mut(), bias) + } + + fn set_weights(&mut self, weights: Array) { + Weighted::set_weights(self.params_mut(), weights) + } +} + +impl ParameterizedExt for P +where + D: Dimension, + P: Parameterized, + T: Float, +

>::Params: Params + 'static, +{ +} + // impl Params for S // where // S: Parameterized, @@ -177,21 +217,6 @@ where // } // } -// impl Weighted for P -// where -// P: Parameterized, -// D: Dimension, -// T: Float, -// { -// fn weights(&self) -> &Array { -// self.params().weights() -// } - -// fn weights_mut(&mut self) -> &mut Array { -// self.params_mut().weights_mut() -// } -// } - pub(crate) mod utils {} #[cfg(test)] From 75918611716bec3f40b76ae238eef6dd825ae615 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 25 Nov 2023 15:30:13 -0600 Subject: [PATCH 068/118] update Signed-off-by: FL03 --- core/src/lib.rs | 3 +- core/src/specs.rs | 82 ++++++++++++------ core/src/vars/epsilon.rs | 4 + core/src/vars/mod.rs | 12 +++ ml/neural/src/layers/features.rs | 6 ++ ml/neural/src/layers/layer.rs | 13 --- ml/neural/src/layers/params.rs | 50 +++++------ ml/neural/src/models/mod.rs | 16 +++- ml/neural/src/models/model.rs | 32 +++++-- ml/neural/src/ops/mod.rs | 2 +- ml/neural/src/ops/norm.rs | 92 ++++++++++++-------- ml/neural/src/params/bias.rs | 95 ++++++++++++++++++++- ml/neural/src/params/group.rs | 88 +++++++++++++++++-- ml/optim/src/optimize/mod.rs | 4 + ml/transformers/src/codec/encode/encoder.rs | 4 +- 15 files changed, 377 insertions(+), 126 deletions(-) create mode 100644 core/src/vars/epsilon.rs create mode 100644 core/src/vars/mod.rs diff --git a/core/src/lib.rs b/core/src/lib.rs index 78d2edc4..b62764ae 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -14,14 +14,15 @@ pub mod errors; pub mod masks; pub mod states; pub mod step; +pub mod vars; pub mod prelude { - pub use crate::epochs::*; pub use crate::errors::*; pub use crate::masks::*; pub use crate::states::*; pub use crate::step::*; + // pub use crate::vars::*; pub use crate::primitives::*; pub use crate::specs::*; diff --git a/core/src/specs.rs b/core/src/specs.rs index da4618aa..fdc8002a 100644 --- a/core/src/specs.rs +++ b/core/src/specs.rs @@ -2,28 +2,48 @@ Appellation: specs Contrib: FL03 */ -use ndarray::prelude::{Array, Axis, Dimension}; +use ndarray::prelude::{Array, Axis, Dimension, Ix2}; use ndarray::IntoDimension; // use ndarray::linalg::Dot; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Uniform}; use ndarray_rand::RandomExt; use num::{Float, One, Zero}; +use std::ops; + +pub trait Arithmetic: + ops::Add + + ops::Div + + ops::Mul + + ops::Sub +{ +} + +impl Arithmetic for A where + A: ops::Add + + ops::Div + + ops::Mul + + ops::Sub +{ +} pub trait IntoAxis { fn into_axis(self) -> Axis; } -impl IntoAxis for usize { +impl IntoAxis for S +where + S: AsRef, +{ fn into_axis(self) -> Axis { - Axis(self) + Axis(*self.as_ref()) } } pub trait Apply { fn apply(&self, f: F) -> T where - F: FnOnce(&Self) -> T; + F: Fn(&Self) -> T; } pub trait ApplyTo { @@ -51,49 +71,61 @@ where } } -pub trait GenerateRandom +pub trait GenerateRandom where + D: Dimension, T: Float + SampleUniform, { - type Dim: Dimension; - fn bernoulli( - dim: impl IntoDimension, + dim: impl IntoDimension, p: Option, - ) -> Result, BernoulliError> { + ) -> Result, BernoulliError> { let dist = Bernoulli::new(p.unwrap_or(0.5))?; Ok(Array::random(dim.into_dimension(), dist)) } - fn uniform(axis: usize, dim: impl IntoDimension) -> Array { + fn uniform(axis: usize, dim: impl IntoDimension) -> Array { let dim = dim.into_dimension(); let dk = (T::one() / T::from(dim[axis]).unwrap()).sqrt(); Self::uniform_between(dk, dim) } - fn uniform_between(dk: T, dim: impl IntoDimension) -> Array { + fn uniform_between(dk: T, dim: impl IntoDimension) -> Array { Array::random(dim, Uniform::new(-dk, dk)) } } -impl GenerateRandom for Array +impl GenerateRandom for Array where T: Float + SampleUniform, D: Dimension, { - type Dim = D; +} - fn bernoulli( - dim: impl IntoDimension, - p: Option, - ) -> Result, BernoulliError> { - let dist = Bernoulli::new(p.unwrap_or(0.5))?; - Ok(Array::random(dim.into_dimension(), dist)) - } +pub trait MatrixOps: + Arithmetic, Array> + Sized +where + A: Dimension, + B: Dimension, +{ +} - fn uniform(axis: usize, dim: impl IntoDimension) -> Array { - let dim = dim.into_dimension(); - let k = (T::from(dim[axis]).unwrap()).sqrt(); - Array::random(dim, Uniform::new(-k, k)) - } +impl MatrixOps for Array +where + A: Dimension, + B: Dimension, + D: Dimension, + T: Float, + Self: Arithmetic, Array>, +{ +} + +impl MatrixOps for &Array +where + A: Dimension, + B: Dimension, + D: Dimension, + T: Float, + Self: Arithmetic, Array>, +{ } diff --git a/core/src/vars/epsilon.rs b/core/src/vars/epsilon.rs new file mode 100644 index 00000000..2055d87f --- /dev/null +++ b/core/src/vars/epsilon.rs @@ -0,0 +1,4 @@ +/* + Appellation: epsilon + Contrib: FL03 +*/ diff --git a/core/src/vars/mod.rs b/core/src/vars/mod.rs new file mode 100644 index 00000000..89a36959 --- /dev/null +++ b/core/src/vars/mod.rs @@ -0,0 +1,12 @@ +/* + Appellation: const + Contrib: FL03 +*/ +pub use self::{epsilon::*, utils::*}; + +pub(crate) mod epsilon; + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests {} diff --git a/ml/neural/src/layers/features.rs b/ml/neural/src/layers/features.rs index 8153e590..cd99a87f 100644 --- a/ml/neural/src/layers/features.rs +++ b/ml/neural/src/layers/features.rs @@ -105,6 +105,12 @@ impl std::fmt::Display for LayerShape { } } +// impl From for (usize,) { +// fn from(features: LayerShape) -> Self { +// (features.inputs,) +// } +// } + impl Features for LayerShape { fn inputs(&self) -> usize { self.inputs diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index af740aa3..a1481463 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -211,19 +211,6 @@ where } } -// impl IntoIterator for Layer -// where -// A: Activate, -// T: Float, -// { -// type Item = (Array1, Array0); -// type IntoIter = std::vec::IntoIter; - -// fn into_iter(self) -> Self::IntoIter { -// self.params().weights().axis_iter(Axis(0)).zip(self.params().bias().axis_iter(Axis(0))).map(|(w, b)| (w.to_owned(), b.to_owned())).collect::>().into_iter() -// } -// } - impl IntoIterator for Layer where A: Activate + Activate + Default, diff --git a/ml/neural/src/layers/params.rs b/ml/neural/src/layers/params.rs index ae843843..e37c3c81 100644 --- a/ml/neural/src/layers/params.rs +++ b/ml/neural/src/layers/params.rs @@ -4,8 +4,8 @@ */ use super::LayerShape; use crate::core::prelude::GenerateRandom; -use crate::prelude::{Biased, Weighted}; -use ndarray::prelude::{Array1, Array2, Ix2}; +use crate::prelude::{Biased, Node, Weighted}; +use ndarray::prelude::{Array1, Array2, Axis, Ix2}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; @@ -77,35 +77,6 @@ where } } -// impl Params for LayerParams -// where -// T: Float, -// { -// fn bias(&self) -> &Array1 { -// &self.bias -// } - -// fn bias_mut(&mut self) -> &mut Array1 { -// &mut self.bias -// } - -// fn weights(&self) -> &Array2 { -// &self.weights -// } - -// fn weights_mut(&mut self) -> &mut Array2 { -// &mut self.weights -// } - -// fn set_bias(&mut self, bias: Array1) { -// self.bias = bias; -// } - -// fn set_weights(&mut self, weights: Array2) { -// self.weights = weights; -// } -// } - impl Biased for LayerParams where T: Float, @@ -139,3 +110,20 @@ where &mut self.weights } } + +impl IntoIterator for LayerParams +where + T: Float, +{ + type Item = Node; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.weights() + .axis_iter(Axis(0)) + .zip(self.bias().axis_iter(Axis(0))) + .map(|(w, b)| (w.to_owned(), b.to_owned()).into()) + .collect::>() + .into_iter() + } +} diff --git a/ml/neural/src/models/mod.rs b/ml/neural/src/models/mod.rs index f58e1ba7..6dac2006 100644 --- a/ml/neural/src/models/mod.rs +++ b/ml/neural/src/models/mod.rs @@ -12,13 +12,25 @@ pub(crate) mod stack; use crate::prelude::Forward; use ndarray::prelude::Array2; +use num::Float; -pub trait Module: Forward, Output = Array2> { +pub trait Module: Forward, Output = Array2> +where + T: Float, +{ fn add_module(&mut self, module: impl Module); - fn layers(&self) -> &[impl Module]; + fn layers(&self) -> &[impl Forward, Output = Array2>]; fn name(&self) -> &str; + + fn forward(&self, input: Array2) -> Array2 { + let mut output = input; + for layer in self.layers() { + output = layer.forward(&output); + } + output + } } pub(crate) mod utils {} diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs index f1d09ec5..b1ca0903 100644 --- a/ml/neural/src/models/model.rs +++ b/ml/neural/src/models/model.rs @@ -3,7 +3,8 @@ Contrib: FL03 */ use super::ModelConfig; -use crate::layers::LayerDyn; +use crate::prelude::{Forward, ForwardDyn}; +use ndarray::prelude::Array2; use num::Float; pub struct Model @@ -11,7 +12,7 @@ where T: Float, { config: ModelConfig, - layers: Vec>, + layers: Vec>, } impl Model @@ -33,23 +34,44 @@ where &mut self.config } - pub fn layers(&self) -> &[LayerDyn] { + pub fn layers(&self) -> &[ForwardDyn] { &self.layers } - pub fn layers_mut(&mut self) -> &mut [LayerDyn] { + pub fn layers_mut(&mut self) -> &mut [ForwardDyn] { &mut self.layers } + + pub fn add_layer(&mut self, layer: ForwardDyn) { + self.layers.push(layer); + } } impl IntoIterator for Model where T: Float, { - type Item = LayerDyn; + type Item = ForwardDyn; type IntoIter = std::vec::IntoIter; fn into_iter(self) -> Self::IntoIter { self.layers.into_iter() } } + +impl Forward> for Model +where + T: Float, +{ + type Output = Array2; + + fn forward(&self, input: &Array2) -> Array2 { + let mut iter = self.layers().into_iter(); + + let mut output = iter.next().unwrap().forward(input); + for layer in iter { + output = layer.forward(&output); + } + output + } +} diff --git a/ml/neural/src/ops/mod.rs b/ml/neural/src/ops/mod.rs index 6a21e539..e7dcc92e 100644 --- a/ml/neural/src/ops/mod.rs +++ b/ml/neural/src/ops/mod.rs @@ -22,7 +22,7 @@ mod tests { let data: Array = Array::linspace(1., 4., 4) .into_shape((1, features)) .unwrap(); - let norm = LayerNorm::::new(features); + let norm = LayerNorm::::new((1, features)); let normed = norm.forward(&data); let rounded = normed.map(|x| x.round_to(4)); let exp = array![[-1.1619, -0.3873, 0.3873, 1.1619]]; diff --git a/ml/neural/src/ops/norm.rs b/ml/neural/src/ops/norm.rs index b982e63b..436346ef 100644 --- a/ml/neural/src/ops/norm.rs +++ b/ml/neural/src/ops/norm.rs @@ -2,75 +2,95 @@ Appellation: norm Contrib: FL03 */ +use crate::core::MatrixOps; use crate::prelude::Forward; -use ndarray::prelude::{Array, Array1, NdFloat}; -use ndarray::Dimension; +use ndarray::prelude::{Array, Axis, Dimension, Ix2, NdFloat}; +use ndarray::{IntoDimension, RemoveAxis}; use num::{Float, FromPrimitive}; use serde::{Deserialize, Serialize}; use std::ops::{Add, Mul}; +pub fn norm(x: &Array, axis: usize) -> Array +where + D: Dimension + RemoveAxis, + T: FromPrimitive + NdFloat, + Array: MatrixOps, +{ + let axis = Axis(axis); + let epsilon = T::from(1e-6).unwrap(); + // Calculate the mean and standard deviation of the activations along the feature axis. + let mean = x.mean_axis(axis.clone()).expect("mean_axis failed"); + + let std = x.std_axis(axis, T::one()); + (x.clone() - mean) / (std + epsilon) +} + +pub fn norm_and_scale( + x: &Array, + alpha: &Array, + beta: &Array, +) -> Array +where + D: Dimension, + T: FromPrimitive + NdFloat, + Array: + Add, Output = Array> + Mul, Output = Array>, +{ + let epsilon = T::from(1e-6).unwrap(); + // Calculate the mean and standard deviation of the activations along the feature axis. + let mean = x.mean().unwrap_or_else(T::zero); + // Normalize the activations. + let norm = (x - mean) / (x.std(T::one()) + epsilon); + + // Scale and shift the normalized activations with learnable parameters alpha and beta. + norm * alpha.clone() + beta.clone() +} + #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -pub struct LayerNorm +pub struct LayerNorm where T: Float, + D: Dimension, { - alpha: Array1, - beta: Array1, + alpha: Array, + beta: Array, } -impl LayerNorm +impl LayerNorm where T: Float, + D: Dimension + RemoveAxis, { - pub fn new(features: usize) -> Self { + pub fn new(dim: impl IntoDimension) -> Self { + let dim = dim.into_dimension(); Self { - alpha: Array1::ones(features), - beta: Array1::zeros(features), + alpha: Array::ones(dim.clone()), + beta: Array::zeros(dim), } } - pub fn alpha(&self) -> &Array1 { + pub fn alpha(&self) -> &Array { &self.alpha } - pub fn alpha_mut(&mut self) -> &mut Array1 { + pub fn alpha_mut(&mut self) -> &mut Array { &mut self.alpha } - pub fn beta(&self) -> &Array1 { + pub fn beta(&self) -> &Array { &self.beta } - pub fn beta_mut(&mut self) -> &mut Array1 { + pub fn beta_mut(&mut self) -> &mut Array { &mut self.beta } } -impl LayerNorm -where - T: FromPrimitive + NdFloat, -{ - pub fn norm_and_scale(&self, x: &Array) -> Array - where - D: Dimension, - Array: Add, Output = Array> + Mul, Output = Array>, - { - let epsilon = T::from(1e-6).unwrap(); - // Calculate the mean and standard deviation of the activations along the feature axis. - let mean = x.mean().unwrap_or_else(T::zero); - // Normalize the activations. - let norm = (x - mean) / (x.std(T::one()) + epsilon); - - // Scale and shift the normalized activations with learnable parameters alpha and beta. - norm * self.alpha().clone() + self.beta().clone() - } -} - -impl Forward> for LayerNorm +impl Forward> for LayerNorm where D: Dimension, T: FromPrimitive + NdFloat, - Array: Add, Output = Array> + Mul, Output = Array>, + Array: Add, Output = Array> + Mul, Output = Array>, { type Output = Array; @@ -82,6 +102,6 @@ where let norm = (data - mean) / (data.std(T::one()) + epsilon); // Scale and shift the normalized activations with learnable parameters alpha and beta. - norm * self.alpha().clone() + self.beta().clone() + norm * self.alpha.clone() + self.beta.clone() } } diff --git a/ml/neural/src/params/bias.rs b/ml/neural/src/params/bias.rs index 7e010129..d92fcb61 100644 --- a/ml/neural/src/params/bias.rs +++ b/ml/neural/src/params/bias.rs @@ -2,8 +2,10 @@ Appellation: bias Contrib: FL03 */ +use crate::core::prelude::GenerateRandom; use crate::generate_uniform_arr; -use ndarray::prelude::{Array, Array1, Dimension, Ix2, NdFloat}; +use ndarray::prelude::{Array, Array1, Dimension, Ix1, NdFloat}; +use ndarray::IntoDimension; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; @@ -16,15 +18,102 @@ pub struct Belief { pub features: usize, } -pub enum Biases +#[derive(Clone, Debug, Deserialize, EnumIs, PartialEq, Serialize, SmartDefault)] +pub enum Biases where D: Dimension, T: Float, { - Biased(Array), + Biased(Array), + #[default] Unbiased, } +impl Biases +where + D: Dimension, + T: Float, +{ + pub fn biased(bias: Array) -> Self { + Self::Biased(bias) + } +} + +impl Biases +where + D: Dimension, + T: Float + SampleUniform, +{ + pub fn init(self, dk: T, features: impl IntoDimension) -> Self { + Self::Biased(Array::uniform_between(dk, features)) + } + pub fn uniform(dk: T, features: impl IntoDimension) -> Self { + Self::Biased(Array::uniform_between(dk, features)) + } +} + +impl From> for Biases +where + D: Dimension, + T: Float, +{ + fn from(bias: Array) -> Self { + Self::Biased(bias) + } +} + +impl From>> for Biases +where + D: Dimension, + T: Float, +{ + fn from(bias: Option>) -> Self { + match bias { + Some(bias) => Self::Biased(bias), + None => Self::Unbiased, + } + } +} + +impl From> for Option> +where + D: Dimension, + T: Float, +{ + fn from(bias: Biases) -> Self { + match bias { + Biases::Biased(bias) => Some(bias), + Biases::Unbiased => None, + } + } +} + +impl From> for Array +where + D: Dimension, + T: Float, +{ + fn from(bias: Biases) -> Self { + match bias { + Biases::Biased(bias) => bias, + Biases::Unbiased => Array::zeros(D::zeros(D::NDIM.unwrap_or_default())), + } + } +} + +// impl<'a, T, D> From<&'a Biases> for ArrayView<'a, T, D> +// where +// D: Dimension + 'a, +// T: Float, +// { +// fn from(bias: &'a Biases) -> Self { +// match bias { +// Biases::Biased(bias) => bias.view(), +// Biases::Unbiased => ArrayView::empty(D::zeros(D::NDIM.unwrap_or_default())), +// } +// } +// } + #[derive(Clone, Debug, Deserialize, EnumIs, PartialEq, Serialize, SmartDefault)] pub enum Bias { Biased(Array1), diff --git a/ml/neural/src/params/group.rs b/ml/neural/src/params/group.rs index 69603844..878facdc 100644 --- a/ml/neural/src/params/group.rs +++ b/ml/neural/src/params/group.rs @@ -3,8 +3,11 @@ Contrib: FL03 */ use super::{Biased, Weighted}; -use ndarray::prelude::{Array, Axis, Dimension, Ix2}; +use crate::core::prelude::GenerateRandom; +use crate::prelude::Forward; +use ndarray::prelude::{Array, Axis, Dimension, Ix1, Ix2}; use ndarray::{IntoDimension, RemoveAxis}; +use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; #[derive(Clone, Debug, Eq, PartialEq)] @@ -12,9 +15,9 @@ pub struct ParamGroup where T: Float, D: Dimension, - ::Smaller: Dimension, { bias: Array, + features: D, weights: Array, } @@ -22,23 +25,72 @@ impl ParamGroup where T: Float, D: Dimension + RemoveAxis, - ::Smaller: Dimension, { pub fn new(dim: impl IntoDimension) -> Self { let dim = dim.into_dimension(); - let smaller = dim.clone().remove_axis(Axis(dim.ndim() - 1)); Self { - bias: Array::zeros(smaller), + bias: Array::zeros(dim.remove_axis(Axis(dim.ndim() - 1))), + features: dim.clone(), weights: Array::zeros(dim), } } } +impl ParamGroup +where + T: Float, + D: Dimension, +{ + pub fn features(&self) -> &D { + &self.features + } + + pub fn inputs(&self) -> usize { + self.weights.shape().last().unwrap().clone() + } + + pub fn outputs(&self) -> usize { + if self.features.ndim() == 1 { + return 1; + } + self.weights.shape().first().unwrap().clone() + } +} + +impl ParamGroup +where + T: Float + SampleUniform, + D: Dimension + RemoveAxis, +{ + pub fn init(mut self, biased: bool) -> Self { + if biased { + self = self.init_bias(); + } + self.init_weight() + } + + pub fn init_bias(mut self) -> Self { + let dk = (T::one() / T::from(self.inputs()).unwrap()).sqrt(); + self.bias = Array::uniform_between( + dk, + self.features() + .remove_axis(Axis(self.features().ndim() - 1)) + .clone(), + ); + self + } + + pub fn init_weight(mut self) -> Self { + let dk = (T::one() / T::from(self.inputs()).unwrap()).sqrt(); + self.weights = Array::uniform_between(dk, self.features().clone()); + self + } +} + impl Biased for ParamGroup where T: Float, D: Dimension + RemoveAxis, - ::Smaller: Dimension, { fn bias(&self) -> &Array { &self.bias @@ -56,7 +108,7 @@ where impl Weighted for ParamGroup where T: Float, - D: Dimension + RemoveAxis, + D: Dimension, ::Smaller: Dimension, { fn weights(&self) -> &Array { @@ -71,3 +123,25 @@ where self.weights = weights; } } + +impl Forward> for ParamGroup +where + T: Float + 'static, +{ + type Output = Array; + + fn forward(&self, data: &Array) -> Self::Output { + data.dot(self.weights()) + self.bias() + } +} + +impl Forward> for ParamGroup +where + T: Float + 'static, +{ + type Output = Array; + + fn forward(&self, data: &Array) -> Self::Output { + data.dot(self.weights()) + self.bias() + } +} diff --git a/ml/optim/src/optimize/mod.rs b/ml/optim/src/optimize/mod.rs index dd1f650c..95724319 100644 --- a/ml/optim/src/optimize/mod.rs +++ b/ml/optim/src/optimize/mod.rs @@ -12,6 +12,10 @@ pub trait Optimize { type Model; fn apply(&self, model: &mut Self::Model) -> &mut Self::Model; + + fn model(&self) -> &Self::Model; + + fn model_mut(&mut self) -> &mut Self::Model; } pub(crate) mod utils {} diff --git a/ml/transformers/src/codec/encode/encoder.rs b/ml/transformers/src/codec/encode/encoder.rs index b8c6c73f..61757b3f 100644 --- a/ml/transformers/src/codec/encode/encoder.rs +++ b/ml/transformers/src/codec/encode/encoder.rs @@ -24,8 +24,8 @@ impl Encoder { Self { attention, network, - norm_attention: LayerNorm::new(params.model), - norm_network: LayerNorm::new(params.model), + norm_attention: LayerNorm::new((params.model, params.model)), + norm_network: LayerNorm::new((params.model, params.model)), params, } } From 2a3b058c46314d9c0ed8796f65e6a95235905990 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 26 Nov 2023 12:02:37 -0600 Subject: [PATCH 069/118] update Signed-off-by: FL03 --- concision/examples/gradients.rs | 2 +- core/src/errors/error.rs | 10 +- ml/neural/src/arch/deep.rs | 29 ++- ml/neural/src/arch/mod.rs | 3 +- ml/neural/src/arch/shallow.rs | 16 +- ml/neural/src/func/activate/mod.rs | 34 ++- ml/neural/src/layers/{ => cmp}/features.rs | 71 +++--- ml/neural/src/layers/{ => cmp}/kinds.rs | 4 +- ml/neural/src/layers/cmp/mod.rs | 14 ++ ml/neural/src/layers/exp/layer.rs | 231 ++++++++++++++++++ ml/neural/src/layers/exp/mod.rs | 15 ++ ml/neural/src/layers/{ => exp}/sublayer.rs | 14 +- ml/neural/src/layers/exp/wrapper.rs | 15 ++ ml/neural/src/layers/layer.rs | 78 +++--- ml/neural/src/layers/mod.rs | 47 ++-- ml/neural/src/layers/params.rs | 71 +++++- ml/neural/src/{models => layers}/stack.rs | 62 +++-- ml/neural/src/models/mod.rs | 3 +- ml/neural/src/neurons/node.rs | 18 ++ ml/neural/src/nn/ffn/mod.rs | 29 +++ ml/neural/src/nn/ffn/model.rs | 4 + ml/neural/src/nn/graph/mod.rs | 17 ++ ml/neural/src/nn/graph/model.rs | 4 + ml/neural/src/nn/mod.rs | 3 + ml/neural/src/params/bias.rs | 10 +- ml/neural/src/params/group.rs | 34 +++ ml/neural/src/params/mod.rs | 20 +- ml/neural/src/prop/mod.rs | 2 +- ml/optim/examples/norm.rs | 4 +- ml/optim/src/grad/gradient.rs | 2 +- ml/optim/src/grad/mod.rs | 8 +- ml/optim/src/grad/sgd.rs | 3 +- ml/optim/src/optimize/mod.rs | 25 +- ml/optim/src/optimize/optimizer.rs | 35 ++- ml/optim/src/specs.rs | 10 +- ml/transformers/src/attention/multi/params.rs | 4 +- ml/transformers/src/attention/params/dim.rs | 4 +- .../src/attention/params/hyperparams.rs | 4 +- ml/transformers/src/codec/encode/encoder.rs | 2 +- ml/transformers/src/codec/encode/params.rs | 4 +- ml/transformers/src/ffn/mod.rs | 2 +- ml/transformers/src/ffn/network.rs | 58 +++-- ml/transformers/src/ffn/params.rs | 35 ++- ml/transformers/src/primitives.rs | 8 +- 44 files changed, 851 insertions(+), 217 deletions(-) rename ml/neural/src/layers/{ => cmp}/features.rs (81%) rename ml/neural/src/layers/{ => cmp}/kinds.rs (96%) create mode 100644 ml/neural/src/layers/cmp/mod.rs create mode 100644 ml/neural/src/layers/exp/layer.rs create mode 100644 ml/neural/src/layers/exp/mod.rs rename ml/neural/src/layers/{ => exp}/sublayer.rs (76%) create mode 100644 ml/neural/src/layers/exp/wrapper.rs rename ml/neural/src/{models => layers}/stack.rs (80%) create mode 100644 ml/neural/src/nn/ffn/mod.rs create mode 100644 ml/neural/src/nn/ffn/model.rs create mode 100644 ml/neural/src/nn/graph/mod.rs create mode 100644 ml/neural/src/nn/graph/model.rs diff --git a/concision/examples/gradients.rs b/concision/examples/gradients.rs index 4cd07a1a..649597b0 100644 --- a/concision/examples/gradients.rs +++ b/concision/examples/gradients.rs @@ -1,4 +1,4 @@ -use concision::prelude::{linarr, Forward, LayerShape, ParameterizedExt}; +use concision::prelude::{linarr, Features, Forward, LayerShape, ParameterizedExt}; use concision::neural::prelude::{Layer, Neuron, Objective, Sigmoid}; use concision::optim::grad::*; diff --git a/core/src/errors/error.rs b/core/src/errors/error.rs index 95861212..ea8bf8e5 100644 --- a/core/src/errors/error.rs +++ b/core/src/errors/error.rs @@ -22,16 +22,18 @@ use strum::{Display, EnumIs, EnumIter, EnumVariantNames}; Serialize, SmartDefault, )] +#[non_exhaustive] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] pub enum Errors { Async, Codec, Connection, + Custom(String), Data, Dimension, #[default] - Error(String), + Error, Execution, IO, Null, @@ -101,6 +103,12 @@ impl std::fmt::Display for Error { impl std::error::Error for Error {} +impl From> for Error { + fn from(err: Box) -> Self { + Self::new(Errors::Unknown, err.to_string()) + } +} + impl From for Error { fn from(err: Errors) -> Self { Self::new(err, String::new()) diff --git a/ml/neural/src/arch/deep.rs b/ml/neural/src/arch/deep.rs index 346da871..d5eb62c9 100644 --- a/ml/neural/src/arch/deep.rs +++ b/ml/neural/src/arch/deep.rs @@ -3,18 +3,17 @@ Contrib: FL03 */ use crate::func::activate::{Activate, Linear}; -use crate::models::Stack; -use crate::prelude::{Forward, Layer, Parameterized}; +use crate::prelude::{Features, Forward, Layer, Parameterized, Stack}; -use ndarray::prelude::{Array2, Ix2, NdFloat}; +use ndarray::prelude::{Array2, NdFloat}; use num::Float; pub struct DeepNetwork where T: Float, - I: Activate, - H: Activate, - O: Activate, + I: Activate, + H: Activate, + O: Activate, { pub input: Layer, pub hidden: Stack, @@ -24,9 +23,9 @@ where impl DeepNetwork where T: Float, - I: Activate, - H: Activate, - O: Activate, + I: Activate, + H: Activate, + O: Activate, { pub fn new(input: Layer, hidden: Stack, output: Layer) -> Self { Self { @@ -39,9 +38,9 @@ where impl DeepNetwork where T: Float, - I: Activate + Clone, - H: Activate + Clone, - O: Activate + Clone, + I: Activate + Clone, + H: Activate + Clone, + O: Activate + Clone, { pub fn validate_dims(&self) -> bool { self.hidden.validate_shapes() @@ -53,9 +52,9 @@ where impl Forward> for DeepNetwork where T: NdFloat, - I: Activate + Clone, - H: Activate + Clone, - O: Activate + Clone, + I: Activate + Clone, + H: Activate + Clone, + O: Activate + Clone, { type Output = Array2; diff --git a/ml/neural/src/arch/mod.rs b/ml/neural/src/arch/mod.rs index 78e2629d..0da86531 100644 --- a/ml/neural/src/arch/mod.rs +++ b/ml/neural/src/arch/mod.rs @@ -31,8 +31,7 @@ pub(crate) mod utils {} mod tests { use super::*; use crate::core::prelude::linarr; - use crate::models::stack::Stack; - use crate::prelude::{Forward, Layer, LayerShape, Sigmoid}; + use crate::prelude::{Forward, Layer, LayerShape, Sigmoid, Stack}; use ndarray::prelude::Ix2; #[test] diff --git a/ml/neural/src/arch/shallow.rs b/ml/neural/src/arch/shallow.rs index d88dcbfc..352bdb2d 100644 --- a/ml/neural/src/arch/shallow.rs +++ b/ml/neural/src/arch/shallow.rs @@ -3,16 +3,16 @@ Contrib: FL03 */ use crate::func::activate::{Activate, Linear}; -use crate::prelude::{Forward, Layer, Parameterized}; +use crate::prelude::{Features, Forward, Layer, Parameterized}; -use ndarray::prelude::{Array2, Ix2, NdFloat}; +use ndarray::prelude::{Array2, NdFloat}; use num::Float; pub struct ShallowNetwork where T: Float, - I: Activate, - O: Activate, + I: Activate, + O: Activate, { pub input: Layer, pub output: Layer, @@ -21,8 +21,8 @@ where impl ShallowNetwork where T: Float, - I: Activate, - O: Activate, + I: Activate, + O: Activate, { pub fn new(input: Layer, output: Layer) -> Self { Self { input, output } @@ -44,8 +44,8 @@ where impl Forward> for ShallowNetwork where T: NdFloat, - I: Activate, - O: Activate, + I: Activate, + O: Activate, { type Output = Array2; diff --git a/ml/neural/src/func/activate/mod.rs b/ml/neural/src/func/activate/mod.rs index 49426cea..0c7b1d93 100644 --- a/ml/neural/src/func/activate/mod.rs +++ b/ml/neural/src/func/activate/mod.rs @@ -12,11 +12,39 @@ pub(crate) mod binary; pub(crate) mod linear; pub(crate) mod nl; -pub type ActivationFn = fn(T) -> T; +// use crate::core::prelude::ShapeResult; +use ndarray::prelude::{Array, ArrayD, Dimension, Ix2, IxDyn}; -pub type ActivateDyn = Box>; +pub type ActivationFn = Box) -> Array>; -use ndarray::prelude::{Array, Dimension, Ix2}; +pub type ActivateDyn = Box>; + +pub trait Rho { + fn rho(&self, args: &T) -> T; +} + +impl Rho for F +where + F: Fn(&T) -> T, +{ + fn rho(&self, args: &T) -> T { + self.call((args,)) + } +} + +pub trait A +where + T: Clone, +{ + // fn activate(&self, args: &Array) -> ShapeResult> { + // let shape = args.shape(); + // let res = self.activate_dyn(&args.into_dyn()); + // let res: Array = res.to_shape(shape)?.to_owned(); + // Ok(res) + // } + + fn activate_dyn(&self, args: &ArrayD) -> ArrayD; +} pub trait Activate where diff --git a/ml/neural/src/layers/features.rs b/ml/neural/src/layers/cmp/features.rs similarity index 81% rename from ml/neural/src/layers/features.rs rename to ml/neural/src/layers/cmp/features.rs index cd99a87f..7d6e0ae0 100644 --- a/ml/neural/src/layers/features.rs +++ b/ml/neural/src/layers/cmp/features.rs @@ -2,13 +2,17 @@ Appellation: features Contrib: FL03 */ -use ndarray::prelude::Ix2; +use ndarray::prelude::{Dimension, Ix2}; use ndarray::IntoDimension; use serde::{Deserialize, Serialize}; pub trait Features { fn inputs(&self) -> usize; + fn network(&self) -> usize { + self.inputs() * self.outputs() + } + fn outputs(&self) -> usize; fn in_by_out(&self) -> (usize, usize) { @@ -24,7 +28,40 @@ pub trait Features { } } -pub trait FeaturesExt: Features + IntoDimension { +impl Features for usize { + fn inputs(&self) -> usize { + *self + } + + fn outputs(&self) -> usize { + 1 + } +} + +impl Features for (usize, usize) { + fn inputs(&self) -> usize { + self.1 + } + + fn outputs(&self) -> usize { + self.0 + } +} + +impl Features for [usize; 2] { + fn inputs(&self) -> usize { + self[1] + } + + fn outputs(&self) -> usize { + self[0] + } +} + +pub trait FeaturesExt: Features + IntoDimension +where + D: Dimension, +{ fn new(inputs: usize, outputs: usize) -> Self; fn single(inputs: usize) -> Self @@ -81,22 +118,6 @@ impl LayerShape { pub fn uniform_scale(&self) -> T { (T::one() / T::from(self.inputs()).unwrap()).sqrt() } - - pub fn inputs(&self) -> usize { - self.inputs - } - - pub fn outputs(&self) -> usize { - self.outputs - } - - pub fn in_by_out(&self) -> (usize, usize) { - (self.inputs, self.outputs) - } - - pub fn out_by_in(&self) -> (usize, usize) { - (self.outputs, self.inputs) - } } impl std::fmt::Display for LayerShape { @@ -105,12 +126,6 @@ impl std::fmt::Display for LayerShape { } } -// impl From for (usize,) { -// fn from(features: LayerShape) -> Self { -// (features.inputs,) -// } -// } - impl Features for LayerShape { fn inputs(&self) -> usize { self.inputs @@ -122,14 +137,14 @@ impl Features for LayerShape { } impl IntoDimension for LayerShape { - type Dim = ndarray::Ix2; + type Dim = Ix2; fn into_dimension(self) -> Self::Dim { ndarray::Ix2(self.outputs, self.inputs) } } -impl From for ndarray::Ix2 { +impl From for Ix2 { fn from(features: LayerShape) -> Self { ndarray::Ix2(features.outputs, features.inputs) } @@ -150,8 +165,8 @@ impl From for [usize; 2] { impl From<[usize; 2]> for LayerShape { fn from(features: [usize; 2]) -> Self { Self { - inputs: features[0], - outputs: features[1], + inputs: features[1], + outputs: features[0], } } } diff --git a/ml/neural/src/layers/kinds.rs b/ml/neural/src/layers/cmp/kinds.rs similarity index 96% rename from ml/neural/src/layers/kinds.rs rename to ml/neural/src/layers/cmp/kinds.rs index 668ae7ac..1031669d 100644 --- a/ml/neural/src/layers/kinds.rs +++ b/ml/neural/src/layers/cmp/kinds.rs @@ -29,7 +29,7 @@ use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; pub enum LayerKind { #[default] Input = 0, - Hidden(usize), + Hidden, Output, } @@ -51,7 +51,7 @@ impl LayerPosition { } pub fn hidden(idx: usize) -> Self { - Self::new(idx, LayerKind::Hidden(idx)) + Self::new(idx, LayerKind::Hidden) } pub fn output(idx: usize) -> Self { diff --git a/ml/neural/src/layers/cmp/mod.rs b/ml/neural/src/layers/cmp/mod.rs new file mode 100644 index 00000000..5bb0b5dd --- /dev/null +++ b/ml/neural/src/layers/cmp/mod.rs @@ -0,0 +1,14 @@ +/* + Appellation: cmp + Contrib: FL03 +*/ +//! # Layers +pub use self::{features::*, kinds::*, utils::*}; + +pub(crate) mod features; +pub(crate) mod kinds; + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests {} diff --git a/ml/neural/src/layers/exp/layer.rs b/ml/neural/src/layers/exp/layer.rs new file mode 100644 index 00000000..5bead9d7 --- /dev/null +++ b/ml/neural/src/layers/exp/layer.rs @@ -0,0 +1,231 @@ +/* + Appellation: layer + Contrib: FL03 +*/ +use crate::layers::{LayerKind, LayerParams, LayerPosition, LayerShape}; +use crate::prelude::{Activate, Features, Forward, Linear, Neuron, Parameterized, Params}; +use ndarray::prelude::{Array2, Axis, Ix1, Ix2, NdFloat}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use num::Float; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct Layer +where + A: Activate, + T: Float, +{ + activator: A, + pub features: LayerShape, + name: String, + params: LayerParams, + position: LayerPosition, +} + +impl Layer +where + A: Default + Activate, + T: Float, +{ + pub fn new(features: LayerShape, position: LayerPosition) -> Self { + Self { + activator: A::default(), + features, + name: String::new(), + params: LayerParams::new(features), + position, + } + } + + pub fn input(features: LayerShape) -> Self { + Self::new(features, LayerPosition::input()) + } + + pub fn hidden(features: LayerShape, index: usize) -> Self { + Self::new(features, LayerPosition::hidden(index)) + } + + pub fn output(features: LayerShape, index: usize) -> Self { + Self::new(features, LayerPosition::output(index)) + } +} + +impl Layer +where + A: Activate, + T: Float, +{ + pub fn activator(&self) -> &A { + &self.activator + } + + pub fn index(&self) -> usize { + self.position().index() + } + + pub fn kind(&self) -> &LayerKind { + self.position().kind() + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn position(&self) -> &LayerPosition { + &self.position + } + + pub fn set_name(&mut self, name: impl ToString) { + self.name = name.to_string(); + } + + pub fn update_position(&mut self, idx: usize, output: bool) { + self.position = if idx == 0 { + LayerPosition::input() + } else if output { + LayerPosition::output(idx) + } else { + LayerPosition::hidden(idx) + }; + } + + pub fn validate_layer(&self, other: &Self) -> bool { + let pos = self.position().index().abs_diff(other.position().index()); + if pos == 1 { + if self.position().index() > other.position().index() { + return self.features().inputs() == other.features().outputs(); + } else { + return self.features().outputs() == other.features().inputs(); + } + } + false + } + + pub fn with_name(mut self, name: impl ToString) -> Self { + self.name = name.to_string(); + self + } +} + +impl Layer +where + A: Activate + Clone + 'static, + T: Float, +{ + pub fn as_dyn(&self) -> Layer>> { + Layer { + activator: Box::new(self.activator.clone()), + features: self.features.clone(), + name: self.name.clone(), + params: self.params.clone(), + position: self.position.clone(), + } + } +} + +impl Layer +where + A: Activate, + T: Float + 'static, +{ + pub fn update_with_gradient(&mut self, gamma: T, grad: &Array2) { + self.params.weights_mut().scaled_add(-gamma, grad); + } +} + +impl Layer +where + A: Activate, + T: NdFloat, +{ + pub fn linear(&self, args: &Array2) -> Array2 { + args.dot(&self.params.weights().t()) + self.params.bias() + } +} + +impl Layer +where + A: Activate, + T: Float + SampleUniform, +{ + pub fn init(mut self, biased: bool) -> Self { + self.params = self.params.init(biased); + self + } +} + +impl Forward> for Layer +where + A: Activate, + T: NdFloat, +{ + type Output = Array2; + + fn forward(&self, args: &Array2) -> Self::Output { + self.activator.activate(&self.linear(args)) + } +} + +impl Parameterized for Layer +where + A: Activate, + T: Float, +{ + type Features = LayerShape; + type Params = LayerParams; + + fn features(&self) -> &LayerShape { + &self.features + } + + fn features_mut(&mut self) -> &mut LayerShape { + &mut self.features + } + + fn params(&self) -> &LayerParams { + &self.params + } + + fn params_mut(&mut self) -> &mut LayerParams { + &mut self.params + } +} + +impl PartialOrd for Layer +where + A: Activate + PartialEq, + T: Float, +{ + fn partial_cmp(&self, other: &Self) -> Option { + self.position.partial_cmp(&other.position) + } +} + +impl From for Layer +where + A: Activate + Default, + T: Float, +{ + fn from(features: LayerShape) -> Self { + Self::new(features, LayerPosition::input()) + } +} + +impl IntoIterator for Layer +where + A: Activate + Activate + Default, + T: Float, +{ + type Item = Neuron; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.params() + .weights() + .axis_iter(Axis(0)) + .zip(self.params().bias().axis_iter(Axis(0))) + .map(|(w, b)| (w.to_owned(), b.to_owned()).into()) + .collect::>() + .into_iter() + } +} diff --git a/ml/neural/src/layers/exp/mod.rs b/ml/neural/src/layers/exp/mod.rs new file mode 100644 index 00000000..589ab107 --- /dev/null +++ b/ml/neural/src/layers/exp/mod.rs @@ -0,0 +1,15 @@ +/* + Appellation: exp + Contrib: FL03 +*/ +//! # Experimental Layers +pub use self::{layer::*, sublayer::*, wrapper::*, utils::*}; + +pub(crate) mod layer; +pub(crate) mod sublayer; +pub(crate) mod wrapper; + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests {} diff --git a/ml/neural/src/layers/sublayer.rs b/ml/neural/src/layers/exp/sublayer.rs similarity index 76% rename from ml/neural/src/layers/sublayer.rs rename to ml/neural/src/layers/exp/sublayer.rs index 7b26a321..4dc297c5 100644 --- a/ml/neural/src/layers/sublayer.rs +++ b/ml/neural/src/layers/exp/sublayer.rs @@ -2,19 +2,17 @@ Appellation: sublayers Contrib: FL03 */ -use super::Layer; -use crate::func::activate::{Activate, Linear}; -use crate::ops::LayerNorm; -use crate::prelude::Forward; +use crate::layers::Layer; +use crate::prelude::{Activate, Forward, LayerNorm, Linear}; -use ndarray::prelude::{Array2, Ix2, NdFloat}; +use ndarray::prelude::{Array2, NdFloat}; use num::{Float, FromPrimitive}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Sublayer where - A: Activate, + A: Activate, T: Float, { layer: Layer, @@ -23,7 +21,7 @@ where impl Sublayer where - A: Activate, + A: Activate, T: Float, { pub fn new(layer: Layer, norm: LayerNorm) -> Self { @@ -33,7 +31,7 @@ where impl Forward> for Sublayer where - A: Activate, + A: Activate, T: FromPrimitive + NdFloat, { type Output = Array2; diff --git a/ml/neural/src/layers/exp/wrapper.rs b/ml/neural/src/layers/exp/wrapper.rs new file mode 100644 index 00000000..4cfcd89f --- /dev/null +++ b/ml/neural/src/layers/exp/wrapper.rs @@ -0,0 +1,15 @@ +/* + Appellation: sublayers + Contrib: FL03 +*/ +use crate::layers::LayerParams; + + +use ndarray::prelude::Array2; +use num::Float; + +pub trait Wrapper where T: Float { + fn apply(&self, data: &Array2) -> Array2; + + fn params(&self) -> &LayerParams; +} diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index a1481463..63446c5d 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -3,8 +3,9 @@ Contrib: FL03 */ use super::{LayerKind, LayerParams, LayerPosition, LayerShape}; -use crate::prelude::{Activate, Forward, Linear, Neuron, Parameterized, Params}; -use ndarray::prelude::{Array2, Axis, Ix1, Ix2, NdFloat}; +use crate::func::activate::{Activate, Linear}; +use crate::prelude::{Features, Forward, Neuron, Node, Parameterized, Params}; +use ndarray::prelude::{Array2, Ix1, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; @@ -12,7 +13,7 @@ use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Layer where - A: Activate, + A: Activate, T: Float, { activator: A, @@ -24,7 +25,7 @@ where impl Layer where - A: Default + Activate, + A: Default + Activate, T: Float, { pub fn new(features: LayerShape, position: LayerPosition) -> Self { @@ -52,7 +53,7 @@ where impl Layer where - A: Activate, + A: Activate, T: Float, { pub fn activator(&self) -> &A { @@ -79,6 +80,13 @@ where self.name = name.to_string(); } + pub fn set_node(&mut self, idx: usize, neuron: &Neuron) + where + A: Activate, + { + self.params.set_node(idx, neuron.node().clone()); + } + pub fn update_position(&mut self, idx: usize, output: bool) { self.position = if idx == 0 { LayerPosition::input() @@ -89,16 +97,11 @@ where }; } - pub fn validate_layer(&self, other: &Self) -> bool { - let pos = self.position().index().abs_diff(other.position().index()); - if pos == 1 { - if self.position().index() > other.position().index() { - return self.features().inputs() == other.features().outputs(); - } else { - return self.features().outputs() == other.features().inputs(); - } + pub fn validate_layer(&self, other: &Self, next: bool) -> bool { + if next { + return self.features().inputs() == other.features().outputs(); } - false + self.features().outputs() == other.features().inputs() } pub fn with_name(mut self, name: impl ToString) -> Self { @@ -109,10 +112,10 @@ where impl Layer where - A: Activate + Clone + 'static, + A: Activate + Clone + 'static, T: Float, { - pub fn as_dyn(&self) -> Layer>> { + pub fn as_dyn(&self) -> Layer>> { Layer { activator: Box::new(self.activator.clone()), features: self.features.clone(), @@ -125,7 +128,7 @@ where impl Layer where - A: Activate, + A: Activate, T: Float + 'static, { pub fn update_with_gradient(&mut self, gamma: T, grad: &Array2) { @@ -135,7 +138,7 @@ where impl Layer where - A: Activate, + A: Activate, T: NdFloat, { pub fn linear(&self, args: &Array2) -> Array2 { @@ -145,7 +148,7 @@ where impl Layer where - A: Activate, + A: Activate, T: Float + SampleUniform, { pub fn init(mut self, biased: bool) -> Self { @@ -156,7 +159,7 @@ where impl Forward> for Layer where - A: Activate, + A: Activate, T: NdFloat, { type Output = Array2; @@ -168,7 +171,7 @@ where impl Parameterized for Layer where - A: Activate, + A: Activate, T: Float, { type Features = LayerShape; @@ -193,7 +196,7 @@ where impl PartialOrd for Layer where - A: Activate + PartialEq, + A: Activate + PartialEq, T: Float, { fn partial_cmp(&self, other: &Self) -> Option { @@ -203,7 +206,7 @@ where impl From for Layer where - A: Activate + Default, + A: Activate + Default, T: Float, { fn from(features: LayerShape) -> Self { @@ -213,19 +216,30 @@ where impl IntoIterator for Layer where - A: Activate + Activate + Default, + A: Activate + Default, T: Float, { - type Item = Neuron; + type Item = Node; type IntoIter = std::vec::IntoIter; fn into_iter(self) -> Self::IntoIter { - self.params() - .weights() - .axis_iter(Axis(0)) - .zip(self.params().bias().axis_iter(Axis(0))) - .map(|(w, b)| (w.to_owned(), b.to_owned()).into()) - .collect::>() - .into_iter() + self.params.into_iter() + } +} + +impl FromIterator> for Layer +where + A: Activate + Default, + T: Float, +{ + fn from_iter>>(nodes: I) -> Self { + let params = LayerParams::from_iter(nodes); + Self { + activator: A::default(), + features: *params.features(), + name: String::new(), + params, + position: LayerPosition::input(), + } } } diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index d44fc6d5..296307b7 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -3,23 +3,28 @@ Contrib: FL03 */ //! # Layers -pub use self::{features::*, kinds::*, layer::*, params::*, sublayer::*, utils::*}; +pub use self::{cmp::*, layer::*, params::*, stack::*, utils::*}; -pub(crate) mod features; -pub(crate) mod kinds; +pub(crate) mod cmp; pub(crate) mod layer; pub(crate) mod params; -pub(crate) mod sublayer; +pub(crate) mod stack; + +pub mod exp; use crate::func::activate::{Activate, ActivateDyn}; -use crate::prelude::Forward; -use ndarray::prelude::{Array2, Ix2}; +use crate::prelude::Node; +use ndarray::prelude::Ix2; // use ndarray::IntoDimension; use num::Float; -pub type LayerDyn = Layer>; +pub type LayerDyn = Layer>; -pub trait L: Forward> { +pub trait L: IntoIterator> +where + A: Activate, + T: Float, +{ fn features(&self) -> LayerShape; fn name(&self) -> &str; fn params(&self) -> &LayerParams; @@ -28,12 +33,12 @@ pub trait L: Forward> { fn is_biased(&self) -> bool; } -pub trait LayerExt: L -where - T: Float, -{ - type Rho: Activate; -} +// pub trait LayerExt: L +// where +// T: Float, +// { +// type Rho: Activate; +// } pub(crate) mod utils {} @@ -42,7 +47,7 @@ mod tests { use super::*; use crate::core::prelude::linarr; use crate::func::activate::Softmax; - use crate::prelude::{Forward, ParameterizedExt}; + use crate::prelude::{Biased, Forward, Node, Parameterized}; use ndarray::prelude::Ix2; #[test] @@ -57,6 +62,12 @@ mod tests { let pred = layer.forward(&args); assert_eq!(pred.dim(), (samples, outputs)); + + let nodes = (0..outputs) + .map(|_| Node::::new(inputs).init(true)) + .collect::>(); + let layer = Layer::::from_iter(nodes); + assert_eq!(layer.features(), &features); } #[test] @@ -66,9 +77,9 @@ mod tests { let layer = Layer::::from(features).init(true); - for neuron in layer.into_iter() { - assert_eq!(neuron.features(), inputs); - assert_eq!(neuron.bias().dim(), ()); + for node in layer.into_iter() { + assert_eq!(node.features(), inputs); + assert_eq!(node.bias().dim(), ()); } } } diff --git a/ml/neural/src/layers/params.rs b/ml/neural/src/layers/params.rs index e37c3c81..7c2d9e15 100644 --- a/ml/neural/src/layers/params.rs +++ b/ml/neural/src/layers/params.rs @@ -4,8 +4,8 @@ */ use super::LayerShape; use crate::core::prelude::GenerateRandom; -use crate::prelude::{Biased, Node, Weighted}; -use ndarray::prelude::{Array1, Array2, Axis, Ix2}; +use crate::prelude::{Biased, Features, Forward, Node, Weighted}; +use ndarray::prelude::{Array1, Array2, Axis, Ix2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; @@ -29,11 +29,6 @@ where } } - pub fn reset(&mut self) { - self.bias = Array1::zeros(self.features.outputs()); - self.weights = Array2::zeros(self.features.out_by_in()); - } - pub fn features(&self) -> &LayerShape { &self.features } @@ -42,6 +37,16 @@ where &mut self.features } + pub fn set_node(&mut self, idx: usize, node: Node) { + self.bias_mut() + .index_axis_mut(Axis(0), idx) + .assign(&node.bias()); + + self.weights_mut() + .index_axis_mut(Axis(0), idx) + .assign(&node.weights()); + } + pub fn with_bias(mut self, bias: Array1) -> Self { self.bias = bias; self @@ -53,6 +58,16 @@ where } } +impl LayerParams +where + T: NdFloat, +{ + pub fn reset(&mut self) { + self.bias *= T::zero(); + self.weights *= T::zero(); + } +} + impl LayerParams where T: Float + SampleUniform, @@ -111,6 +126,30 @@ where } } +impl Features for LayerParams +where + T: Float, +{ + fn inputs(&self) -> usize { + self.features.inputs() + } + + fn outputs(&self) -> usize { + self.features.outputs() + } +} + +impl Forward> for LayerParams +where + T: NdFloat, +{ + type Output = Array2; + + fn forward(&self, input: &Array2) -> Array2 { + input.dot(&self.weights) + &self.bias + } +} + impl IntoIterator for LayerParams where T: Float, @@ -127,3 +166,21 @@ where .into_iter() } } + +impl FromIterator> for LayerParams +where + T: Float, +{ + fn from_iter>>(nodes: I) -> Self { + let nodes = nodes.into_iter().collect::>(); + let mut iter = nodes.iter(); + let node = iter.next().unwrap(); + let shape = LayerShape::new(node.features(), nodes.len()); + let mut params = LayerParams::new(shape); + params.set_node(0, node.clone()); + for (i, node) in iter.into_iter().enumerate() { + params.set_node(i + 1, node.clone()); + } + params + } +} diff --git a/ml/neural/src/models/stack.rs b/ml/neural/src/layers/stack.rs similarity index 80% rename from ml/neural/src/models/stack.rs rename to ml/neural/src/layers/stack.rs index 1e3036b2..4a98bea9 100644 --- a/ml/neural/src/models/stack.rs +++ b/ml/neural/src/layers/stack.rs @@ -2,21 +2,25 @@ Appellation: stack Contrib: FL03 */ -use crate::layers::Layer; -use crate::prelude::{Activate, LayerShape, Linear, Parameterized}; -use ndarray::prelude::Ix2; +use crate::layers::{Layer, LayerShape}; +use crate::prelude::{Activate, Features, Linear, Parameterized}; use num::Float; use serde::{Deserialize, Serialize}; use std::ops; -pub trait HiddenLayers {} +pub trait Layers: IntoIterator> +where + A: Activate, + T: Float, +{ +} /// A [Stack] is a collection of [Layer]s, typically used to construct the hidden /// layers of a deep neural network. #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] pub struct Stack where - A: Activate, + A: Activate, T: Float, { children: Vec>, @@ -24,7 +28,7 @@ where impl Stack where - A: Activate + Default, + A: Activate + Default, T: Float, { pub fn build_layers(mut self, shapes: impl IntoIterator) -> Self { @@ -38,7 +42,7 @@ where impl Stack where - A: Activate + Clone + Default, + A: Activate + Clone + Default, T: Float + crate::core::prelude::SampleUniform, { pub fn init_layers(mut self, biased: bool) -> Self { @@ -51,7 +55,7 @@ where impl Stack where - A: Activate, + A: Activate, T: Float, { pub fn new() -> Self { @@ -112,7 +116,7 @@ where impl AsRef<[Layer]> for Stack where - A: Activate, + A: Activate, T: Float, { fn as_ref(&self) -> &[Layer] { @@ -122,7 +126,7 @@ where impl AsMut<[Layer]> for Stack where - A: Activate, + A: Activate, T: Float, { fn as_mut(&mut self) -> &mut [Layer] { @@ -132,7 +136,7 @@ where impl IntoIterator for Stack where - A: Activate, + A: Activate, T: Float, { type Item = Layer; @@ -145,7 +149,7 @@ where impl FromIterator> for Stack where - A: Activate, + A: Activate, T: Float, { fn from_iter>>(iter: I) -> Self { @@ -157,17 +161,17 @@ where impl From>> for Stack where - A: Activate, + A: Activate, T: Float, { - fn from(layers: Vec>) -> Self { - Self { children: layers } + fn from(children: Vec>) -> Self { + Self { children } } } impl From> for Stack where - A: Activate, + A: Activate, T: Float, { fn from(layer: Layer) -> Self { @@ -179,7 +183,7 @@ where impl ops::Index for Stack where - A: Activate, + A: Activate, T: Float, { type Output = Layer; @@ -191,7 +195,7 @@ where impl ops::IndexMut for Stack where - A: Activate, + A: Activate, T: Float, { fn index_mut(&mut self, index: usize) -> &mut Self::Output { @@ -199,6 +203,28 @@ where } } +impl ops::Index> for Stack +where + A: Activate, + T: Float, +{ + type Output = [Layer]; + + fn index(&self, index: ops::Range) -> &Self::Output { + &self.children[index] + } +} + +impl ops::IndexMut> for Stack +where + A: Activate, + T: Float, +{ + fn index_mut(&mut self, index: ops::Range) -> &mut Self::Output { + &mut self.children[index] + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/ml/neural/src/models/mod.rs b/ml/neural/src/models/mod.rs index 6dac2006..217f8457 100644 --- a/ml/neural/src/models/mod.rs +++ b/ml/neural/src/models/mod.rs @@ -4,11 +4,10 @@ */ //! # Model //! -pub use self::{config::*, model::*, stack::*, utils::*}; +pub use self::{config::*, model::*, utils::*}; pub(crate) mod config; pub(crate) mod model; -pub(crate) mod stack; use crate::prelude::Forward; use ndarray::prelude::Array2; diff --git a/ml/neural/src/neurons/node.rs b/ml/neural/src/neurons/node.rs index 0faae4a5..25d9e955 100644 --- a/ml/neural/src/neurons/node.rs +++ b/ml/neural/src/neurons/node.rs @@ -110,6 +110,7 @@ where impl Forward> for Node where + Self: Biased, T: FromPrimitive + NdFloat, { type Output = Array1; @@ -153,6 +154,23 @@ where } } +impl FromIterator for Node +where + T: Float, +{ + fn from_iter(iter: I) -> Self + where + I: IntoIterator, + { + let weights = Array1::::from_iter(iter); + Self { + bias: Array0::zeros(()), + features: weights.len(), + weights, + } + } +} + impl From<(Array1, Array0)> for Node where T: Float, diff --git a/ml/neural/src/nn/ffn/mod.rs b/ml/neural/src/nn/ffn/mod.rs new file mode 100644 index 00000000..f5a8d48d --- /dev/null +++ b/ml/neural/src/nn/ffn/mod.rs @@ -0,0 +1,29 @@ +/* + Appellation: ffn + Contrib: FL03 +*/ +//! # Feed Forward Neural Network +//! +pub use self::{model::*, utils::*}; + +pub(crate) mod model; + +use ndarray::prelude::{Array, Array2, Dimension, Ix2}; +use num::Float; + +pub trait FeedForward +where + D: Dimension, + T: Float, +{ + type Opt; + + fn backward(&mut self, args: &Array2, targets: &Array2, opt: &Self::Opt) -> Array2; + + fn forward(&self, args: &Array2) -> Array; +} + +pub(crate) mod utils {} + +#[cfg(tets)] +mod tests {} diff --git a/ml/neural/src/nn/ffn/model.rs b/ml/neural/src/nn/ffn/model.rs new file mode 100644 index 00000000..e2a0d764 --- /dev/null +++ b/ml/neural/src/nn/ffn/model.rs @@ -0,0 +1,4 @@ +/* + Appellation: model + Contrib: FL03 +*/ diff --git a/ml/neural/src/nn/graph/mod.rs b/ml/neural/src/nn/graph/mod.rs new file mode 100644 index 00000000..c61bc342 --- /dev/null +++ b/ml/neural/src/nn/graph/mod.rs @@ -0,0 +1,17 @@ +/* + Appellation: graph + Contrib: FL03 +*/ +//! # Graph Neural Network +//! +pub use self::{model::*, utils::*}; + +pub(crate) mod model; + + + + +pub(crate) mod utils {} + +#[cfg(tets)] +mod tests {} diff --git a/ml/neural/src/nn/graph/model.rs b/ml/neural/src/nn/graph/model.rs new file mode 100644 index 00000000..e2a0d764 --- /dev/null +++ b/ml/neural/src/nn/graph/model.rs @@ -0,0 +1,4 @@ +/* + Appellation: model + Contrib: FL03 +*/ diff --git a/ml/neural/src/nn/mod.rs b/ml/neural/src/nn/mod.rs index 5d743331..83ddfaff 100644 --- a/ml/neural/src/nn/mod.rs +++ b/ml/neural/src/nn/mod.rs @@ -8,6 +8,9 @@ pub use self::{position::*, sequential::*, utils::*}; pub(crate) mod position; pub(crate) mod sequential; +pub mod ffn; +pub mod graph; + use crate::layers::Layer; use crate::Trainable; use num::Float; diff --git a/ml/neural/src/params/bias.rs b/ml/neural/src/params/bias.rs index d92fcb61..1fa1ae67 100644 --- a/ml/neural/src/params/bias.rs +++ b/ml/neural/src/params/bias.rs @@ -13,9 +13,13 @@ use smart_default::SmartDefault; use std::ops; use strum::EnumIs; -pub struct Belief { - pub bias: Bias, - pub features: usize, +pub struct Belief +where + D: Dimension, + T: Float, +{ + pub bias: Array, + pub features: D, } #[derive(Clone, Debug, Deserialize, EnumIs, PartialEq, Serialize, SmartDefault)] diff --git a/ml/neural/src/params/group.rs b/ml/neural/src/params/group.rs index 878facdc..6b297c88 100644 --- a/ml/neural/src/params/group.rs +++ b/ml/neural/src/params/group.rs @@ -9,6 +9,7 @@ use ndarray::prelude::{Array, Axis, Dimension, Ix1, Ix2}; use ndarray::{IntoDimension, RemoveAxis}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; +use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Eq, PartialEq)] pub struct ParamGroup @@ -145,3 +146,36 @@ where data.dot(self.weights()) + self.bias() } } + +impl<'a, T, D> Deserialize<'a> for ParamGroup +where + T: Deserialize<'a> + Float, + D: Deserialize<'a> + Dimension, + ::Smaller: Deserialize<'a> + Dimension, +{ + fn deserialize(deserializer: Der) -> Result + where + Der: serde::Deserializer<'a>, + { + let (bias, features, weights) = Deserialize::deserialize(deserializer)?; + Ok(Self { + bias, + features, + weights, + }) + } +} + +impl Serialize for ParamGroup +where + T: Float + Serialize, + D: Dimension + RemoveAxis + Serialize, + ::Smaller: Dimension + Serialize, +{ + fn serialize(&self, serializer: Ser) -> Result + where + Ser: serde::Serializer, + { + (self.bias(), self.features(), self.weights()).serialize(serializer) + } +} diff --git a/ml/neural/src/params/mod.rs b/ml/neural/src/params/mod.rs index 1f967b19..9ad2c6bf 100644 --- a/ml/neural/src/params/mod.rs +++ b/ml/neural/src/params/mod.rs @@ -25,7 +25,6 @@ pub trait Biased where D: Dimension, T: Float, - Self: Weighted, { /// Returns an owned reference to the bias of the layer. fn bias(&self) -> &Array; @@ -67,7 +66,7 @@ where fn set_weights(&mut self, weights: Array); } -pub trait ParamsExt: Biased +pub trait ParamsExt: Biased + Weighted where Array: Dot, Output = Array>, D: Dimension, @@ -84,7 +83,7 @@ where T: Float, { type Features: IntoDimension; - type Params: Biased; + type Params: Biased + Weighted; fn features(&self) -> &Self::Features; @@ -171,32 +170,31 @@ where impl Params for P where D: Dimension, - P: Biased, T: Float, - ::Smaller: Dimension, + Self: Biased + Weighted + Sized, { fn bias(&self) -> &Array { - self.bias() + Biased::bias(self) } fn bias_mut(&mut self) -> &mut Array { - self.bias_mut() + Biased::bias_mut(self) } fn weights(&self) -> &Array { - self.weights() + Weighted::weights(self) } fn weights_mut(&mut self) -> &mut Array { - self.weights_mut() + Weighted::weights_mut(self) } fn set_bias(&mut self, bias: Array) { - self.set_bias(bias) + Biased::set_bias(self, bias) } fn set_weights(&mut self, weights: Array) { - self.set_weights(weights) + Weighted::set_weights(self, weights) } } diff --git a/ml/neural/src/prop/mod.rs b/ml/neural/src/prop/mod.rs index 6ddf7a7f..f1b081a6 100644 --- a/ml/neural/src/prop/mod.rs +++ b/ml/neural/src/prop/mod.rs @@ -18,7 +18,7 @@ pub type ForwardDyn = Box, Output = Ar pub trait Backward: Forward { type Optim; - fn backward(&mut self, data: &T, opt: Self::Optim); + fn backward(&mut self, data: &T, opt: &Self::Optim); } pub trait Forward { diff --git a/ml/optim/examples/norm.rs b/ml/optim/examples/norm.rs index e5cc5dbe..827b9a59 100644 --- a/ml/optim/examples/norm.rs +++ b/ml/optim/examples/norm.rs @@ -1,4 +1,4 @@ -use concision_neural::prelude::LayerShape; +use concision_neural::prelude::{Features, LayerShape}; use concision_optim::prelude::Norm; use ndarray::prelude::Array; @@ -15,7 +15,7 @@ fn main() -> anyhow::Result<()> { Ok(()) } -pub fn sample_norm(features: LayerShape, samples: usize) -> anyhow::Result<()> { +pub fn sample_norm(features: impl Features, samples: usize) -> anyhow::Result<()> { let n = samples * features.inputs(); let args = Array::linspace(1., n as f64, n) .into_shape((samples, features.inputs())) diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs index f85bccc5..151ffb0b 100644 --- a/ml/optim/src/grad/gradient.rs +++ b/ml/optim/src/grad/gradient.rs @@ -44,7 +44,7 @@ impl Grad where T: NdFloat, { - pub fn step(&mut self, x: &Array2, y: &Array1) -> anyhow::Result { + pub fn step(&mut self, data: &Array2, targets: &Array1) -> anyhow::Result { let cost = T::zero(); Ok(cost) } diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index 26426384..514f0a75 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -101,7 +101,7 @@ mod tests { use super::*; use crate::core::prelude::linarr; use crate::neural::func::activate::{Linear, Objective, Sigmoid}; - use crate::neural::prelude::{Layer, LayerShape, Parameterized, Weighted}; + use crate::neural::prelude::{Features, Layer, LayerShape, Parameterized, Weighted}; use ndarray::prelude::{Array1, Array2}; fn test_grad(args: &Array2) -> Array2 { @@ -110,8 +110,7 @@ mod tests { #[test] fn descent() { - let (_samples, inputs) = (20, 5); - let outputs = 1; + let (_samples, inputs, outputs) = (20, 5, 1); let (epochs, gamma) = (10, 0.001); @@ -130,8 +129,7 @@ mod tests { #[test] fn test_gradient() { - let (samples, inputs) = (20, 5); - let outputs = 1; + let (samples, inputs, outputs) = (20, 5, 1); let (epochs, gamma) = (10, 0.001); diff --git a/ml/optim/src/grad/sgd.rs b/ml/optim/src/grad/sgd.rs index 3b00fdff..d3e5e379 100644 --- a/ml/optim/src/grad/sgd.rs +++ b/ml/optim/src/grad/sgd.rs @@ -5,7 +5,8 @@ //! # Stochastic Gradient Descent (SGD) //! //! -use crate::neural::prelude::{Activate, Forward, Layer, Parameterized, Weighted}; + +use crate::neural::prelude::{Activate, Features, Forward, Layer, Parameterized, Weighted}; // use crate::prelude::ObjectiveFn; use ndarray::prelude::{s, Array1, Array2, Axis, Ix2, NdFloat}; use ndarray_stats::DeviationExt; diff --git a/ml/optim/src/optimize/mod.rs b/ml/optim/src/optimize/mod.rs index 95724319..c43fb8bf 100644 --- a/ml/optim/src/optimize/mod.rs +++ b/ml/optim/src/optimize/mod.rs @@ -8,14 +8,29 @@ pub use self::{optimizer::*, utils::*}; pub(crate) mod optimizer; -pub trait Optimize { - type Model; +use crate::neural::prelude::Forward; +use ndarray::prelude::{Array, Array2, Dimension, Ix2}; +use num::Float; - fn apply(&self, model: &mut Self::Model) -> &mut Self::Model; +pub trait Optimize { + type Model: Forward, Output = Array2>; - fn model(&self) -> &Self::Model; + fn name(&self) -> &str; - fn model_mut(&mut self) -> &mut Self::Model; + fn optimize(&mut self, model: &mut Self::Model, args: &Array2, targets: &Array2); +} + +pub trait Gradient +where + D: Dimension, + T: Float, +{ + fn update(&mut self, gamma: T, params: &mut Array, gradients: &Array) + where + T: 'static, + { + params.scaled_add(-gamma, gradients); + } } pub(crate) mod utils {} diff --git a/ml/optim/src/optimize/optimizer.rs b/ml/optim/src/optimize/optimizer.rs index 70b403ef..6b6df9bb 100644 --- a/ml/optim/src/optimize/optimizer.rs +++ b/ml/optim/src/optimize/optimizer.rs @@ -3,15 +3,40 @@ Contrib: FL03 */ use crate::neural::prelude::Params; -use ndarray::prelude::Dimension; +use ndarray::prelude::Array2; +use num::Float; -pub struct Optimizer { +pub trait Optimizer { + fn name(&self) -> &str; +} + +pub struct OptimizerStep +where + T: Float, +{ + data: Array2, params: Vec>, + targets: Array2, } -impl Optimizer { - pub fn new() -> Self { - Self { params: Vec::new() } +impl OptimizerStep +where + T: Float, +{ + pub fn new(data: Array2, targets: Array2) -> Self { + Self { + data, + params: Vec::new(), + targets, + } + } + + pub fn zeros(inputs: usize, outputs: usize, samples: usize) -> Self { + Self { + data: Array2::zeros((samples, inputs)), + params: Vec::new(), + targets: Array2::zeros((samples, outputs)), + } } pub fn params(&self) -> &[Box] { diff --git a/ml/optim/src/specs.rs b/ml/optim/src/specs.rs index 4f3e018d..75414202 100644 --- a/ml/optim/src/specs.rs +++ b/ml/optim/src/specs.rs @@ -2,9 +2,17 @@ Appellation: specs Contrib: FL03 */ -use ndarray::prelude::{Array, Dimension}; +use ndarray::prelude::{Array, Ix2, Dimension}; use num::Float; +pub trait ApplyGradient +where + D: Dimension, + T: Float, +{ + fn apply_gradient(&mut self, gamma: T, gradients: &Array); +} + pub trait Gradient where T: Float, diff --git a/ml/transformers/src/attention/multi/params.rs b/ml/transformers/src/attention/multi/params.rs index ed508a7c..b6f955b3 100644 --- a/ml/transformers/src/attention/multi/params.rs +++ b/ml/transformers/src/attention/multi/params.rs @@ -2,7 +2,7 @@ Appellation: params Contrib: FL03 */ -use crate::{HEADS, MODEL_SIZE}; +use crate::{HEADS, MODEL}; use serde::{Deserialize, Serialize}; #[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] @@ -31,7 +31,7 @@ impl MultiHeadParams { impl Default for MultiHeadParams { fn default() -> Self { - Self::new(HEADS, MODEL_SIZE) + Self::new(HEADS, MODEL) } } diff --git a/ml/transformers/src/attention/params/dim.rs b/ml/transformers/src/attention/params/dim.rs index b00e8d93..157fed1a 100644 --- a/ml/transformers/src/attention/params/dim.rs +++ b/ml/transformers/src/attention/params/dim.rs @@ -10,7 +10,7 @@ //! - `batch`: The batch size //! - `heads`: The number of attention heads //! - `model`: The dimension of the model (embedding size) -use crate::{HEADS, MODEL_SIZE, QUERY_SIZE}; +use crate::{HEADS, MODEL, QUERY_SIZE}; use ndarray::prelude::{Ix2, Ix3, Ix4}; use ndarray::IntoDimension; use serde::{Deserialize, Serialize}; @@ -60,7 +60,7 @@ impl BaseShape { } pub fn std(batch: usize, seq: usize) -> Self { - Self::new(batch, seq, MODEL_SIZE) + Self::new(batch, seq, MODEL) } pub fn batch(&self) -> usize { diff --git a/ml/transformers/src/attention/params/hyperparams.rs b/ml/transformers/src/attention/params/hyperparams.rs index 658dd0a0..4c636d93 100644 --- a/ml/transformers/src/attention/params/hyperparams.rs +++ b/ml/transformers/src/attention/params/hyperparams.rs @@ -15,7 +15,7 @@ //! use super::dim::{BaseShape, HeadShape, MultiShape}; -use crate::{HEADS, MODEL_SIZE, SAMPLES}; +use crate::{HEADS, MODEL, SAMPLES}; use serde::{Deserialize, Serialize}; #[derive( @@ -42,7 +42,7 @@ impl AttentionParameters { } pub fn std(batch: usize, seq: usize) -> Self { - Self::new(batch, HEADS, MODEL_SIZE, SAMPLES, seq) + Self::new(batch, HEADS, MODEL, SAMPLES, seq) } pub fn batch_size(&self) -> usize { diff --git a/ml/transformers/src/codec/encode/encoder.rs b/ml/transformers/src/codec/encode/encoder.rs index 61757b3f..55442edd 100644 --- a/ml/transformers/src/codec/encode/encoder.rs +++ b/ml/transformers/src/codec/encode/encoder.rs @@ -20,7 +20,7 @@ pub struct Encoder { impl Encoder { pub fn new(params: EncoderParams) -> Self { let attention = MultiHeadAttention::new(params.heads, params.model); - let network = FFN::new(params.model, None); + let network = FFN::new(params.model, crate::NETWORK); Self { attention, network, diff --git a/ml/transformers/src/codec/encode/params.rs b/ml/transformers/src/codec/encode/params.rs index b1cb416d..d7d84f35 100644 --- a/ml/transformers/src/codec/encode/params.rs +++ b/ml/transformers/src/codec/encode/params.rs @@ -2,7 +2,7 @@ Appellation: params Contrib: FL03 */ -use crate::{HEADS, MODEL_SIZE}; +use crate::{HEADS, MODEL}; use serde::{Deserialize, Serialize}; #[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] @@ -31,6 +31,6 @@ impl EncoderParams { impl Default for EncoderParams { fn default() -> Self { - Self::new(HEADS, MODEL_SIZE) + Self::new(HEADS, MODEL) } } diff --git a/ml/transformers/src/ffn/mod.rs b/ml/transformers/src/ffn/mod.rs index d5fd0d86..7dd10d33 100644 --- a/ml/transformers/src/ffn/mod.rs +++ b/ml/transformers/src/ffn/mod.rs @@ -26,7 +26,7 @@ mod tests { let x = linarr::((samples, model)).unwrap(); let _y = linarr::((samples, model)).unwrap(); - let ffn = FFN::new(model, Some(network)); + let ffn = FFN::new(model, network); // assert!(network.validate_dims()); let pred = ffn.forward(&x); diff --git a/ml/transformers/src/ffn/network.rs b/ml/transformers/src/ffn/network.rs index ec90e99c..05096ff0 100644 --- a/ml/transformers/src/ffn/network.rs +++ b/ml/transformers/src/ffn/network.rs @@ -3,35 +3,59 @@ Contrib: FL03 */ use super::FFNParams; -use crate::neural::func::activate::{Activate, ReLU}; -use crate::neural::prelude::{Forward, Layer}; -use ndarray::prelude::Array2; +use crate::neural::prelude::{Forward, Layer, ReLU}; +use crate::prelude::{MODEL, NETWORK}; +use ndarray::prelude::{Array2, NdFloat}; +use num::Float; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct FFN { - input: Layer, - output: Layer, +pub struct FFN +where + T: Float, +{ + input: Layer, + output: Layer, pub params: FFNParams, } -impl FFN { - pub fn new(model: usize, network: Option) -> Self { - let network = network.unwrap_or(crate::NETWORK_SIZE); - let features = network / model; +impl FFN +where + T: Float, +{ + pub fn new(model: usize, network: usize) -> Self { Self { - input: Layer::input((model, features).into()), - output: Layer::output((features, model).into(), 1), + input: Layer::input((model, network).into()), + output: Layer::output((network, model).into(), 1), params: FFNParams::new(model, network), } } + + pub fn input(&self) -> &Layer { + &self.input + } + + pub fn output(&self) -> &Layer { + &self.output + } +} + +impl Default for FFN +where + T: Float, +{ + fn default() -> Self { + Self::new(MODEL, NETWORK) + } } -impl Forward> for FFN { - type Output = Array2; +impl Forward> for FFN +where + T: NdFloat, +{ + type Output = Array2; - fn forward(&self, data: &Array2) -> Self::Output { - self.output - .forward(&ReLU::default().activate(&self.input.forward(data))) + fn forward(&self, data: &Array2) -> Self::Output { + self.output.forward(&ReLU(&self.input.forward(data))) } } diff --git a/ml/transformers/src/ffn/params.rs b/ml/transformers/src/ffn/params.rs index 83f388a3..da4c10a0 100644 --- a/ml/transformers/src/ffn/params.rs +++ b/ml/transformers/src/ffn/params.rs @@ -2,6 +2,10 @@ Appellation: params Contrib: FL03 */ +use crate::neural::prelude::Features; +use crate::prelude::{MODEL, NETWORK}; +use ndarray::prelude::Ix2; +use ndarray::IntoDimension; use serde::{Deserialize, Serialize}; #[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] @@ -15,20 +19,39 @@ impl FFNParams { Self { model, network } } - pub fn model_size(&self) -> usize { + pub fn model(&self) -> usize { self.model } - pub fn network_size(&self) -> usize { + pub fn network(&self) -> usize { self.network } + + pub fn features(&self) -> usize { + self.network / self.model + } } impl Default for FFNParams { fn default() -> Self { - Self { - model: crate::MODEL_SIZE, - network: crate::NETWORK_SIZE, - } + Self::new(MODEL, NETWORK) + } +} + +impl Features for FFNParams { + fn inputs(&self) -> usize { + self.model + } + + fn outputs(&self) -> usize { + self.network + } +} + +impl IntoDimension for FFNParams { + type Dim = Ix2; + + fn into_dimension(self) -> Ix2 { + (self.model(), self.network()).into_dimension() } } diff --git a/ml/transformers/src/primitives.rs b/ml/transformers/src/primitives.rs index 851c0fd0..d004ca57 100644 --- a/ml/transformers/src/primitives.rs +++ b/ml/transformers/src/primitives.rs @@ -10,9 +10,9 @@ pub(crate) mod constants { /// The default number of heads in the multi-head attention layer pub const HEADS: usize = 8; /// The default dimension of the model (embedding size) - pub const MODEL_SIZE: usize = 512; - /// The default dimension of the feed-forward network - pub const NETWORK_SIZE: usize = 2048; + pub const MODEL: usize = 512; + /// The default number of parameters in the feed-forward network + pub const NETWORK: usize = 2048; /// The default number of samples to draw from the attention distribution pub const SAMPLES: usize = 10000; } @@ -23,7 +23,7 @@ pub(crate) mod statics { lazy_static! { /// The default dimensions of the query, key, and value tensors w/r/2 a single head - pub static ref QUERY_SIZE: usize = MODEL_SIZE / HEADS; + pub static ref QUERY_SIZE: usize = MODEL / HEADS; } } From 41a6698fe9993ff4d0ca039e24e7800835e456b2 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 26 Nov 2023 12:37:24 -0600 Subject: [PATCH 070/118] update Signed-off-by: FL03 --- data/src/datasets/group.rs | 75 +++++++++++++++++++++++++++++ data/src/datasets/mod.rs | 3 +- ml/neural/src/layers/exp/mod.rs | 2 +- ml/neural/src/layers/exp/wrapper.rs | 6 ++- ml/neural/src/nn/ffn/mod.rs | 4 ++ ml/neural/src/nn/graph/mod.rs | 3 -- ml/neural/src/params/mod.rs | 14 +++--- ml/optim/src/grad/mod.rs | 3 +- ml/optim/src/optimize/mod.rs | 7 ++- ml/optim/src/optimize/optimizer.rs | 11 ++++- ml/optim/src/specs.rs | 2 +- 11 files changed, 112 insertions(+), 18 deletions(-) create mode 100644 data/src/datasets/group.rs diff --git a/data/src/datasets/group.rs b/data/src/datasets/group.rs new file mode 100644 index 00000000..fcbb8af4 --- /dev/null +++ b/data/src/datasets/group.rs @@ -0,0 +1,75 @@ +/* + Appellation: group + Contrib: FL03 +*/ +use ndarray::prelude::{Array, Dimension, Ix2}; +use ndarray::IntoDimension; +use num::Float; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DataGroup +where + T: Float, + D: Dimension, +{ + data: Array, + targets: Array, +} + +impl DataGroup +where + T: Float, + D: Dimension, +{ + pub fn new(data: Array, targets: Array) -> Self { + Self { data, targets } + } + + pub fn zeros(ds: impl IntoDimension, ts: impl IntoDimension) -> Self { + Self::new(Array::zeros(ds), Array::zeros(ts)) + } + + pub fn inputs(&self) -> usize { + self.data.shape().last().unwrap().clone() + } + + pub fn samples(&self) -> usize { + self.data.shape().first().unwrap().clone() + } + + pub fn data(&self) -> &Array { + &self.data + } + + pub fn targets(&self) -> &Array { + &self.targets + } +} + +impl<'a, T, D> Deserialize<'a> for DataGroup +where + T: Deserialize<'a> + Float, + D: Deserialize<'a> + Dimension, +{ + fn deserialize(deserializer: Der) -> Result + where + Der: serde::Deserializer<'a>, + { + let (data, targets) = Deserialize::deserialize(deserializer)?; + Ok(Self::new(data, targets)) + } +} + +impl Serialize for DataGroup +where + T: Float + Serialize, + D: Dimension + Serialize, +{ + fn serialize(&self, serializer: Ser) -> Result + where + Ser: serde::Serializer, + { + (self.data(), self.targets()).serialize(serializer) + } +} diff --git a/data/src/datasets/mod.rs b/data/src/datasets/mod.rs index f88ec742..00597a07 100644 --- a/data/src/datasets/mod.rs +++ b/data/src/datasets/mod.rs @@ -3,8 +3,9 @@ Contrib: FL03 */ //! # Dataset -pub use self::{dataset::*, utils::*}; +pub use self::{dataset::*, group::*, utils::*}; pub(crate) mod dataset; +pub(crate) mod group; pub(crate) mod utils {} diff --git a/ml/neural/src/layers/exp/mod.rs b/ml/neural/src/layers/exp/mod.rs index 589ab107..3890a3ec 100644 --- a/ml/neural/src/layers/exp/mod.rs +++ b/ml/neural/src/layers/exp/mod.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ //! # Experimental Layers -pub use self::{layer::*, sublayer::*, wrapper::*, utils::*}; +pub use self::{layer::*, sublayer::*, utils::*, wrapper::*}; pub(crate) mod layer; pub(crate) mod sublayer; diff --git a/ml/neural/src/layers/exp/wrapper.rs b/ml/neural/src/layers/exp/wrapper.rs index 4cfcd89f..0a8f2c7e 100644 --- a/ml/neural/src/layers/exp/wrapper.rs +++ b/ml/neural/src/layers/exp/wrapper.rs @@ -4,11 +4,13 @@ */ use crate::layers::LayerParams; - use ndarray::prelude::Array2; use num::Float; -pub trait Wrapper where T: Float { +pub trait Wrapper +where + T: Float, +{ fn apply(&self, data: &Array2) -> Array2; fn params(&self) -> &LayerParams; diff --git a/ml/neural/src/nn/ffn/mod.rs b/ml/neural/src/nn/ffn/mod.rs index f5a8d48d..3e1a209b 100644 --- a/ml/neural/src/nn/ffn/mod.rs +++ b/ml/neural/src/nn/ffn/mod.rs @@ -11,6 +11,8 @@ pub(crate) mod model; use ndarray::prelude::{Array, Array2, Dimension, Ix2}; use num::Float; + + pub trait FeedForward where D: Dimension, @@ -21,6 +23,8 @@ where fn backward(&mut self, args: &Array2, targets: &Array2, opt: &Self::Opt) -> Array2; fn forward(&self, args: &Array2) -> Array; + + } pub(crate) mod utils {} diff --git a/ml/neural/src/nn/graph/mod.rs b/ml/neural/src/nn/graph/mod.rs index c61bc342..30ce3a11 100644 --- a/ml/neural/src/nn/graph/mod.rs +++ b/ml/neural/src/nn/graph/mod.rs @@ -8,9 +8,6 @@ pub use self::{model::*, utils::*}; pub(crate) mod model; - - - pub(crate) mod utils {} #[cfg(tets)] diff --git a/ml/neural/src/params/mod.rs b/ml/neural/src/params/mod.rs index 9ad2c6bf..b2b1b20d 100644 --- a/ml/neural/src/params/mod.rs +++ b/ml/neural/src/params/mod.rs @@ -83,7 +83,7 @@ where T: Float, { type Features: IntoDimension; - type Params: Biased + Weighted; + type Params; fn features(&self) -> &Self::Features; @@ -101,27 +101,27 @@ where >::Params: Params + 'static, { fn bias(&self) -> &Array { - Biased::bias(self.params()) + Params::bias(self.params()) } fn bias_mut(&mut self) -> &mut Array { - Biased::bias_mut(self.params_mut()) + Params::bias_mut(self.params_mut()) } fn weights(&self) -> &Array { - Weighted::weights(self.params()) + Params::weights(self.params()) } fn weights_mut(&mut self) -> &mut Array { - Weighted::weights_mut(self.params_mut()) + Params::weights_mut(self.params_mut()) } fn set_bias(&mut self, bias: Array) { - Biased::set_bias(self.params_mut(), bias) + Params::set_bias(self.params_mut(), bias) } fn set_weights(&mut self, weights: Array) { - Weighted::set_weights(self.params_mut(), weights) + Params::set_weights(self.params_mut(), weights) } } diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index 514f0a75..fa1af69d 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -26,7 +26,7 @@ pub struct DescentParams { } pub(crate) mod utils { - use crate::neural::prelude::{Forward, Parameterized, Weighted}; + use crate::neural::prelude::{Forward, Parameterized, Params}; use ndarray::prelude::{Array, Array1, Array2, Dimension, Ix2, NdFloat}; use ndarray_stats::DeviationExt; use num::{FromPrimitive, Signed}; @@ -41,6 +41,7 @@ pub(crate) mod utils { where A: Forward, Output = Array2> + Parameterized, T: FromPrimitive + NdFloat + Signed, + >::Params: Params + 'static, { let (_samples, _inputs) = data.dim(); let pred = model.forward(data); diff --git a/ml/optim/src/optimize/mod.rs b/ml/optim/src/optimize/mod.rs index c43fb8bf..01bcc45b 100644 --- a/ml/optim/src/optimize/mod.rs +++ b/ml/optim/src/optimize/mod.rs @@ -17,7 +17,12 @@ pub trait Optimize { fn name(&self) -> &str; - fn optimize(&mut self, model: &mut Self::Model, args: &Array2, targets: &Array2); + // fn optimize(&mut self, model: &mut Self::Model, args: &Array2, targets: &Array2) -> T { + // let gradients = model.backward(args, targets); + // let loss = model.loss(args, targets); + // self.update(model, &gradients); + // loss + // } } pub trait Gradient diff --git a/ml/optim/src/optimize/optimizer.rs b/ml/optim/src/optimize/optimizer.rs index 6b6df9bb..1f34cfeb 100644 --- a/ml/optim/src/optimize/optimizer.rs +++ b/ml/optim/src/optimize/optimizer.rs @@ -6,8 +6,17 @@ use crate::neural::prelude::Params; use ndarray::prelude::Array2; use num::Float; -pub trait Optimizer { +pub trait Optimizer +where + T: Float, +{ fn name(&self) -> &str; + + fn step( + &mut self, + data: &Array2, + targets: &Array2, + ) -> impl Fn(&mut Box>) -> T; } pub struct OptimizerStep diff --git a/ml/optim/src/specs.rs b/ml/optim/src/specs.rs index 75414202..8b6d7bd7 100644 --- a/ml/optim/src/specs.rs +++ b/ml/optim/src/specs.rs @@ -2,7 +2,7 @@ Appellation: specs Contrib: FL03 */ -use ndarray::prelude::{Array, Ix2, Dimension}; +use ndarray::prelude::{Array, Dimension, Ix2}; use num::Float; pub trait ApplyGradient From bc05211fe55aac3eefa7a02b794025a149a89682 Mon Sep 17 00:00:00 2001 From: FL03 Date: Tue, 28 Nov 2023 12:08:28 -0600 Subject: [PATCH 071/118] update Signed-off-by: FL03 --- concision/examples/gradients.rs | 4 +- ml/neural/src/layers/exp/layer.rs | 146 ++++++++++------------- ml/neural/src/layers/layer.rs | 8 ++ ml/neural/src/nn/ffn/mod.rs | 11 +- ml/neural/src/nn/{graph => gnn}/mod.rs | 17 +++ ml/neural/src/nn/{graph => gnn}/model.rs | 0 ml/neural/src/nn/mod.rs | 19 ++- ml/neural/src/params/group.rs | 19 ++- ml/optim/src/grad/descent.rs | 107 +++++++++-------- 9 files changed, 194 insertions(+), 137 deletions(-) rename ml/neural/src/nn/{graph => gnn}/mod.rs (54%) rename ml/neural/src/nn/{graph => gnn}/model.rs (100%) diff --git a/concision/examples/gradients.rs b/concision/examples/gradients.rs index 649597b0..a04cda6d 100644 --- a/concision/examples/gradients.rs +++ b/concision/examples/gradients.rs @@ -41,9 +41,9 @@ pub fn sample_descent( ) -> anyhow::Result<()> { // Generate some example data let x = linarr((samples, features.inputs()))?; - let y = linarr(samples)?; + let y = linarr((samples, features.outputs()))?; - let model = Neuron::new(features.inputs()).init_weight(); + let model = Layer::from(features).init(false); println!( "Targets:\n\n{:?}\nPredictions:\n\n{:?}\n", &y, diff --git a/ml/neural/src/layers/exp/layer.rs b/ml/neural/src/layers/exp/layer.rs index 5bead9d7..0a9a1a13 100644 --- a/ml/neural/src/layers/exp/layer.rs +++ b/ml/neural/src/layers/exp/layer.rs @@ -1,10 +1,11 @@ /* - Appellation: layer + Appellation: model Contrib: FL03 */ -use crate::layers::{LayerKind, LayerParams, LayerPosition, LayerShape}; -use crate::prelude::{Activate, Features, Forward, Linear, Neuron, Parameterized, Params}; -use ndarray::prelude::{Array2, Axis, Ix1, Ix2, NdFloat}; +use crate::func::activate::{Activate, Linear}; +use crate::layers::{LayerParams, LayerShape}; +use crate::prelude::{Features, Forward, Neuron, Node, Parameterized, Params}; +use ndarray::prelude::{Array2, Ix1, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; @@ -12,93 +13,59 @@ use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Layer where - A: Activate, + A: Activate, T: Float, { activator: A, pub features: LayerShape, name: String, params: LayerParams, - position: LayerPosition, } impl Layer where - A: Default + Activate, + A: Default + Activate, T: Float, { - pub fn new(features: LayerShape, position: LayerPosition) -> Self { + pub fn new(features: LayerShape) -> Self { Self { activator: A::default(), features, name: String::new(), params: LayerParams::new(features), - position, } } - - pub fn input(features: LayerShape) -> Self { - Self::new(features, LayerPosition::input()) - } - - pub fn hidden(features: LayerShape, index: usize) -> Self { - Self::new(features, LayerPosition::hidden(index)) - } - - pub fn output(features: LayerShape, index: usize) -> Self { - Self::new(features, LayerPosition::output(index)) - } } impl Layer where - A: Activate, + A: Activate, T: Float, { pub fn activator(&self) -> &A { &self.activator } - pub fn index(&self) -> usize { - self.position().index() - } - - pub fn kind(&self) -> &LayerKind { - self.position().kind() - } - pub fn name(&self) -> &str { &self.name } - pub fn position(&self) -> &LayerPosition { - &self.position - } - pub fn set_name(&mut self, name: impl ToString) { self.name = name.to_string(); } - pub fn update_position(&mut self, idx: usize, output: bool) { - self.position = if idx == 0 { - LayerPosition::input() - } else if output { - LayerPosition::output(idx) - } else { - LayerPosition::hidden(idx) - }; - } - - pub fn validate_layer(&self, other: &Self) -> bool { - let pos = self.position().index().abs_diff(other.position().index()); - if pos == 1 { - if self.position().index() > other.position().index() { - return self.features().inputs() == other.features().outputs(); - } else { - return self.features().outputs() == other.features().inputs(); - } + pub fn set_node(&mut self, idx: usize, neuron: &Neuron) + where + A: Activate, + { + self.params.set_node(idx, neuron.node().clone()); + } + + pub fn validate_layer(&self, other: &Self, next: bool) -> bool { + if next { + return self.features().inputs() == other.features().outputs(); } - false + self.features().outputs() == other.features().inputs() } pub fn with_name(mut self, name: impl ToString) -> Self { @@ -109,25 +76,32 @@ where impl Layer where - A: Activate + Clone + 'static, + A: Activate + Clone + 'static, T: Float, { - pub fn as_dyn(&self) -> Layer>> { + pub fn as_dyn(&self) -> Layer>> { Layer { activator: Box::new(self.activator.clone()), features: self.features.clone(), name: self.name.clone(), params: self.params.clone(), - position: self.position.clone(), } } } impl Layer where - A: Activate, + A: Activate, T: Float + 'static, { + pub fn apply_gradient(&mut self, gamma: T, gradient: F) + where + F: Fn(&Array2) -> Array2, + { + let grad = gradient(&self.params.weights()); + self.params.weights_mut().scaled_add(-gamma, &grad); + } + pub fn update_with_gradient(&mut self, gamma: T, grad: &Array2) { self.params.weights_mut().scaled_add(-gamma, grad); } @@ -135,7 +109,7 @@ where impl Layer where - A: Activate, + A: Activate, T: NdFloat, { pub fn linear(&self, args: &Array2) -> Array2 { @@ -145,7 +119,7 @@ where impl Layer where - A: Activate, + A: Activate, T: Float + SampleUniform, { pub fn init(mut self, biased: bool) -> Self { @@ -156,7 +130,7 @@ where impl Forward> for Layer where - A: Activate, + A: Activate, T: NdFloat, { type Output = Array2; @@ -168,7 +142,7 @@ where impl Parameterized for Layer where - A: Activate, + A: Activate, T: Float, { type Features = LayerShape; @@ -191,41 +165,51 @@ where } } -impl PartialOrd for Layer -where - A: Activate + PartialEq, - T: Float, -{ - fn partial_cmp(&self, other: &Self) -> Option { - self.position.partial_cmp(&other.position) - } -} +// impl PartialOrd for Layer +// where +// A: Activate + PartialEq, +// T: Float, +// { +// fn partial_cmp(&self, other: &Self) -> Option { +// self.position.partial_cmp(&other.position) +// } +// } impl From for Layer where - A: Activate + Default, + A: Activate + Default, T: Float, { fn from(features: LayerShape) -> Self { - Self::new(features, LayerPosition::input()) + Self::new(features) } } impl IntoIterator for Layer where - A: Activate + Activate + Default, + A: Activate + Default, T: Float, { - type Item = Neuron; + type Item = Node; type IntoIter = std::vec::IntoIter; fn into_iter(self) -> Self::IntoIter { - self.params() - .weights() - .axis_iter(Axis(0)) - .zip(self.params().bias().axis_iter(Axis(0))) - .map(|(w, b)| (w.to_owned(), b.to_owned()).into()) - .collect::>() - .into_iter() + self.params.into_iter() + } +} + +impl FromIterator> for Layer +where + A: Activate + Default, + T: Float, +{ + fn from_iter>>(nodes: I) -> Self { + let params = LayerParams::from_iter(nodes); + Self { + activator: A::default(), + features: *params.features(), + name: String::new(), + params, + } } } diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 63446c5d..6349da49 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -131,6 +131,14 @@ where A: Activate, T: Float + 'static, { + pub fn apply_gradient(&mut self, gamma: T, gradient: F) + where + F: Fn(&Array2) -> Array2, + { + let grad = gradient(&self.params.weights()); + self.params.weights_mut().scaled_add(-gamma, &grad); + } + pub fn update_with_gradient(&mut self, gamma: T, grad: &Array2) { self.params.weights_mut().scaled_add(-gamma, grad); } diff --git a/ml/neural/src/nn/ffn/mod.rs b/ml/neural/src/nn/ffn/mod.rs index 3e1a209b..e85fddf4 100644 --- a/ml/neural/src/nn/ffn/mod.rs +++ b/ml/neural/src/nn/ffn/mod.rs @@ -11,7 +11,12 @@ pub(crate) mod model; use ndarray::prelude::{Array, Array2, Dimension, Ix2}; use num::Float; - +pub trait Optimizer +where + T: Float, +{ + fn step(&mut self, grad: &Array2) -> Array2; +} pub trait FeedForward where @@ -20,11 +25,11 @@ where { type Opt; + fn apply_gradients(&mut self, gamma: &T, grad: &Array); + fn backward(&mut self, args: &Array2, targets: &Array2, opt: &Self::Opt) -> Array2; fn forward(&self, args: &Array2) -> Array; - - } pub(crate) mod utils {} diff --git a/ml/neural/src/nn/graph/mod.rs b/ml/neural/src/nn/gnn/mod.rs similarity index 54% rename from ml/neural/src/nn/graph/mod.rs rename to ml/neural/src/nn/gnn/mod.rs index 30ce3a11..b872f4f9 100644 --- a/ml/neural/src/nn/graph/mod.rs +++ b/ml/neural/src/nn/gnn/mod.rs @@ -8,6 +8,23 @@ pub use self::{model::*, utils::*}; pub(crate) mod model; +use num::Float; + + + +pub trait GNN +where + T: Float, +{ + type G; + + fn depth(&self) -> usize { + self.layers().len() + } + + fn layers(&self) -> &[Self::G]; +} + pub(crate) mod utils {} #[cfg(tets)] diff --git a/ml/neural/src/nn/graph/model.rs b/ml/neural/src/nn/gnn/model.rs similarity index 100% rename from ml/neural/src/nn/graph/model.rs rename to ml/neural/src/nn/gnn/model.rs diff --git a/ml/neural/src/nn/mod.rs b/ml/neural/src/nn/mod.rs index 83ddfaff..a55fb8cd 100644 --- a/ml/neural/src/nn/mod.rs +++ b/ml/neural/src/nn/mod.rs @@ -9,10 +9,12 @@ pub(crate) mod position; pub(crate) mod sequential; pub mod ffn; -pub mod graph; +pub mod gnn; +use crate::core::BoxResult; use crate::layers::Layer; use crate::Trainable; +use ndarray::prelude::{Array, Dimension, Ix2}; use num::Float; pub trait NeuralNet: Trainable @@ -41,4 +43,19 @@ where fn hidden_layers(&self) -> &[Layer]; } +pub trait Compile {} + +pub trait Train {} + +pub trait Predict where D: Dimension, T: Float, { + type Output; + + fn predict(&self, input: &Array) -> BoxResult; + + fn predict_batch(&self, input: &[Array]) -> BoxResult> { + let res = input.iter().map(|x| self.predict(x).expect("")).collect(); + Ok(res) + } +} + pub(crate) mod utils {} diff --git a/ml/neural/src/params/group.rs b/ml/neural/src/params/group.rs index 6b297c88..d1304421 100644 --- a/ml/neural/src/params/group.rs +++ b/ml/neural/src/params/group.rs @@ -4,7 +4,7 @@ */ use super::{Biased, Weighted}; use crate::core::prelude::GenerateRandom; -use crate::prelude::Forward; +use crate::prelude::{Forward, Node}; use ndarray::prelude::{Array, Axis, Dimension, Ix1, Ix2}; use ndarray::{IntoDimension, RemoveAxis}; use ndarray_rand::rand_distr::uniform::SampleUniform; @@ -179,3 +179,20 @@ where (self.bias(), self.features(), self.weights()).serialize(serializer) } } + +impl IntoIterator for ParamGroup +where + T: Float, +{ + type Item = Node; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.weights() + .axis_iter(Axis(0)) + .zip(self.bias().axis_iter(Axis(0))) + .map(|(w, b)| (w.to_owned(), b.to_owned()).into()) + .collect::>() + .into_iter() + } +} diff --git a/ml/optim/src/grad/descent.rs b/ml/optim/src/grad/descent.rs index b351039b..3d14d5d7 100644 --- a/ml/optim/src/grad/descent.rs +++ b/ml/optim/src/grad/descent.rs @@ -2,119 +2,128 @@ Appellation: grad Contrib: FL03 */ -use crate::neural::prelude::{Forward, Neuron}; -use crate::prelude::Norm; -use ndarray::prelude::{Array1, Array2}; +use crate::neural::prelude::{Forward, Layer,}; +use ndarray::prelude::{Array1, Array2, NdFloat}; use ndarray_stats::DeviationExt; +use num::{Float, Signed}; #[derive(Clone)] -pub struct GradientDescent { - pub gamma: f64, - model: Neuron, +pub struct GradientDescent where T: Float { + pub gamma: T, + model: Layer, } -impl GradientDescent { - pub fn new(gamma: f64, model: Neuron) -> Self { +impl GradientDescent where T: Float { + pub fn new(gamma: T, model: Layer) -> Self { Self { gamma, model } } - pub fn gamma(&self) -> f64 { + pub fn gamma(&self) -> T { self.gamma } - pub fn gamma_mut(&mut self) -> &mut f64 { + pub fn gamma_mut(&mut self) -> &mut T { &mut self.gamma } - pub fn model(&self) -> &Neuron { + pub fn model(&self) -> &Layer { &self.model } - pub fn model_mut(&mut self) -> &mut Neuron { + pub fn model_mut(&mut self) -> &mut Layer { &mut self.model } - pub fn set_gamma(&mut self, gamma: f64) { + pub fn set_gamma(&mut self, gamma: T) { self.gamma = gamma; } - pub fn set_model(&mut self, model: Neuron) { + pub fn set_model(&mut self, model: Layer) { self.model = model; } - pub fn with_gamma(mut self, gamma: f64) -> Self { + pub fn with_gamma(mut self, gamma: T) -> Self { self.gamma = gamma; self } - pub fn with_model(mut self, model: Neuron) -> Self { + pub fn with_model(mut self, model: Layer) -> Self { self.model = model; self } +} + +impl GradientDescent where T: NdFloat + Signed { + + pub fn gradient( &mut self, - data: &Array2, - targets: &Array1, - grad: impl Fn(&Array1) -> Array1, - ) -> anyhow::Result { + data: &Array2, + targets: &Array2, + grad: impl Fn(&Array2) -> Array2, + ) -> anyhow::Result { + let lr = self.gamma(); - let (samples, _inputs) = data.dim(); + let ns = T::from(data.shape()[0]).unwrap(); let pred = self.model.forward(data); + let scale = T::from(2).unwrap() * ns; + let errors = &pred - targets; let dz = errors * grad(&pred); - let dw = data.t().dot(&dz) / (2.0 * samples as f64); + let dw = data.t().dot(&dz) / scale; - self.model_mut().update_with_gradient(lr, &dw); + self.model_mut().update_with_gradient(lr, &dw.t().to_owned()); let loss = targets.mean_sq_err(&self.model().forward(data))?; - Ok(loss) - } - - pub fn step(&mut self, data: &Array2, targets: &Array1) -> anyhow::Result { - // let pred = self.model.forward(data); - let gradient = |p: &Array1| { - let error = targets - &data.dot(&(p / p.l2())); - let scale = -1.0 / (2.0 * data.len() as f64); - let grad = scale * error.dot(data); - - &grad / grad.l2() - }; - self.model.apply_gradient(self.gamma, &gradient); - - let loss = targets.mean_sq_err(&self.model.forward(data))?; - Ok(loss) + Ok(T::from(loss).unwrap()) } } #[cfg(test)] mod tests { use super::*; - use ndarray::prelude::{Array, Array1, Array2}; + use crate::neural::prelude::{LayerShape, Objective, Sigmoid}; + use ndarray::prelude::{Array, Array2}; - fn sample_data(samples: usize, inputs: usize) -> (Array2, Array1) { - let n = samples * inputs; - let x = Array::linspace(1., n as f64, n) + fn sample_data(inputs: usize, outputs: usize, samples: usize) -> (Array2, Array2) { + let m = samples * inputs; + let n = samples * outputs; + let x = Array::linspace(1., m as f64, m) .into_shape((samples, inputs)) .unwrap(); - let y = Array::linspace(1., samples as f64, samples) - .into_shape(samples) + let y = Array::linspace(1., n as f64, n) + .into_shape((samples, outputs)) .unwrap(); (x, y) } #[test] fn test_descent() { - let (samples, inputs) = (20, 5); + let (samples, inputs, outputs) = (20, 5, 3); let (_epochs, gamma) = (1, 0.01); // Generate some example data - let (x, y) = sample_data(samples, inputs); + let (x, y) = sample_data(inputs, outputs, samples); + let features = LayerShape::new(inputs, outputs); + let model = Layer::from(features).init(true); - let model = Neuron::new(inputs).init_weight(); let mut grad = GradientDescent::new(gamma, model); - let _s = grad.step(&x, &y); + let l1 = { + let tmp = grad.gradient(&x, &y, |xs| Sigmoid::default().gradient(xs)); + assert!(tmp.is_ok()); + tmp.unwrap() + }; + + let l2 = { + let tmp = grad.gradient(&x, &y, |xs| Sigmoid::default().gradient(xs)); + assert!(tmp.is_ok()); + tmp.unwrap() + }; + + assert!(l1 > l2); + } } From 58de8ac5f4e5a4d27d5347212bb32bcfcf15f34c Mon Sep 17 00:00:00 2001 From: FL03 Date: Thu, 30 Nov 2023 11:06:26 -0600 Subject: [PATCH 072/118] update Signed-off-by: FL03 --- ml/neural/src/layers/cmp/features.rs | 6 ++ ml/neural/src/layers/layer.rs | 25 +++++++ ml/neural/src/layers/mod.rs | 33 +++++++-- ml/neural/src/layers/stack.rs | 78 +++++++++++++++++++++ ml/neural/src/nn/ffn/mlp.rs | 101 +++++++++++++++++++++++++++ ml/neural/src/nn/ffn/mod.rs | 7 +- ml/neural/src/nn/ffn/model.rs | 11 +++ ml/neural/src/nn/gnn/mod.rs | 2 - ml/neural/src/nn/mod.rs | 6 +- ml/optim/src/grad/descent.rs | 25 ++++--- ml/optim/src/grad/mod.rs | 17 +++-- 11 files changed, 283 insertions(+), 28 deletions(-) create mode 100644 ml/neural/src/nn/ffn/mlp.rs diff --git a/ml/neural/src/layers/cmp/features.rs b/ml/neural/src/layers/cmp/features.rs index 7d6e0ae0..f25f7b59 100644 --- a/ml/neural/src/layers/cmp/features.rs +++ b/ml/neural/src/layers/cmp/features.rs @@ -111,6 +111,12 @@ impl LayerShape { Self { inputs, outputs } } + pub fn from_dimension(shape: impl IntoDimension) -> Self { + let dim = shape.into_dimension(); + let (outputs, inputs) = dim.into_pattern(); + Self::new(inputs, outputs) + } + pub fn neuron(inputs: usize) -> Self { Self::new(inputs, 1) } diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 6349da49..44b48dff 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -165,6 +165,20 @@ where } } +impl Features for Layer +where + A: Activate, + T: Float, +{ + fn inputs(&self) -> usize { + self.features.inputs() + } + + fn outputs(&self) -> usize { + self.features.outputs() + } +} + impl Forward> for Layer where A: Activate, @@ -212,6 +226,17 @@ where } } +// impl From for Layer +// where +// A: Activate + Default, +// S: IntoDimension +// T: Float, +// { +// fn from(features: LayerShape) -> Self { +// Self::new(features, LayerPosition::input()) +// } +// } + impl From for Layer where A: Activate + Default, diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index 296307b7..cbc8f014 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -12,9 +12,8 @@ pub(crate) mod stack; pub mod exp; -use crate::func::activate::{Activate, ActivateDyn}; -use crate::prelude::Node; -use ndarray::prelude::Ix2; +use crate::prelude::{Activate, ActivateDyn, Forward, Node}; +use ndarray::prelude::{Array2, Ix2}; // use ndarray::IntoDimension; use num::Float; @@ -33,6 +32,14 @@ where fn is_biased(&self) -> bool; } +pub trait FFNLayer: Forward> + L +where + A: Activate, + T: Float, +{ + fn forward(&self, args: &Node) -> Node; +} + // pub trait LayerExt: L // where // T: Float, @@ -40,7 +47,25 @@ where // type Rho: Activate; // } -pub(crate) mod utils {} +pub(crate) mod utils { + use crate::prelude::{Activate, Features, Layer}; + use num::Float; + + pub fn validate_layers(layers: I) -> bool + where + A: Activate, + T: Float, + I: IntoIterator>, + { + let layers = layers.into_iter().collect::>(); + let depth = layers.len(); + let mut dim = true; + for (i, layer) in layers[..(depth - 1)].into_iter().enumerate() { + dim = dim && layer.inputs() == layers[i + 1].outputs(); + } + dim + } +} #[cfg(test)] mod tests { diff --git a/ml/neural/src/layers/stack.rs b/ml/neural/src/layers/stack.rs index 4a98bea9..65e47bca 100644 --- a/ml/neural/src/layers/stack.rs +++ b/ml/neural/src/layers/stack.rs @@ -4,6 +4,7 @@ */ use crate::layers::{Layer, LayerShape}; use crate::prelude::{Activate, Features, Linear, Parameterized}; +// use ndarray::{IntoDimension, Ix2}; use num::Float; use serde::{Deserialize, Serialize}; use std::ops; @@ -15,6 +16,23 @@ where { } +pub trait StackExt +where + A: Activate, + T: Float, + Self: Clone + Layers + IntoIterator>, +{ + fn validate_params(&self) -> bool { + let layers = self.clone().into_iter().collect::>(); + let depth = layers.len(); + let mut dim = true; + for (i, layer) in layers[..(depth - 1)].into_iter().enumerate() { + dim = dim && layer.features().inputs() == layers[i + 1].features().outputs(); + } + dim + } +} + /// A [Stack] is a collection of [Layer]s, typically used to construct the hidden /// layers of a deep neural network. #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] @@ -31,7 +49,24 @@ where A: Activate + Default, T: Float, { + // pub fn create>(shapes: impl IntoIterator) -> Self { + // let shapes = shapes.into_iter().map(|s| s.into_dimension()); + // let mut children = Vec::new(); + // for (inputs, outputs) in shapes { + // children.push(Layer::::from(LayerShape::new(inputs, outputs))); + // } + + // } + // pub fn build_layers(mut self, shapes: impl IntoIterator) -> Self { + // // let shapes = shapes.into_iter().map(|s| (s.inputs(), s.outputs())); + // for (inputs, outputs) in shapes.into_iter().map(|s| (s.inputs(), s.outputs())) { + // self.children + // .push(Layer::::from(LayerShape::new(inputs, outputs))); + // } + // self + // } pub fn build_layers(mut self, shapes: impl IntoIterator) -> Self { + // let shapes = shapes.into_iter().map(|s| (s.inputs(), s.outputs())); for (inputs, outputs) in shapes.into_iter() { self.children .push(Layer::::from(LayerShape::new(inputs, outputs))); @@ -97,6 +132,14 @@ where self.children.len() } + pub fn validate_params(&self) -> bool { + let mut dim = true; + for (i, layer) in self[..(self.len() - 1)].iter().enumerate() { + dim = dim && layer.features().outputs() == self[i + 1].features().inputs(); + } + dim + } + pub fn validate_shapes(&self) -> bool { let mut dim = true; for (i, layer) in self.children[..(self.len() - 1)].iter().enumerate() { @@ -225,6 +268,40 @@ where } } +impl ops::Index> for Stack +where + A: Activate, + T: Float, +{ + type Output = [Layer]; + + fn index(&self, index: ops::RangeFrom) -> &Self::Output { + &self.children[index] + } +} + +impl ops::IndexMut> for Stack +where + A: Activate, + T: Float, +{ + fn index_mut(&mut self, index: ops::RangeFrom) -> &mut Self::Output { + &mut self.children[index] + } +} + +impl ops::Index> for Stack +where + A: Activate, + T: Float, +{ + type Output = [Layer]; + + fn index(&self, index: ops::RangeTo) -> &Self::Output { + &self.children[index] + } +} + #[cfg(test)] mod tests { use super::*; @@ -239,6 +316,7 @@ mod tests { let stack = Stack::::new() .build_layers(shapes) .init_layers(true); + // assert!(stack.validate_params()); assert!(stack.validate_shapes()); for (layer, shape) in stack.layers().iter().zip(&shapes) { diff --git a/ml/neural/src/nn/ffn/mlp.rs b/ml/neural/src/nn/ffn/mlp.rs new file mode 100644 index 00000000..35b82803 --- /dev/null +++ b/ml/neural/src/nn/ffn/mlp.rs @@ -0,0 +1,101 @@ +/* + Appellation: mlp + Contrib: FL03 +*/ +//! # Multi-Layer Perceptron +//! + +use crate::func::activate::{Activate, Linear}; +use crate::layers::{Layer, LayerShape, Stack}; +use crate::prelude::{Features, Forward, Parameterized}; + +use ndarray::prelude::{Array2, Ix2, NdFloat}; +use ndarray::IntoDimension; +use num::Float; + +pub struct MLP +where + T: Float, + I: Activate, + H: Activate, + O: Activate, +{ + pub input: Layer, + pub hidden: Stack, + pub output: Layer, +} + +impl MLP +where + T: Float, + I: Activate, + H: Activate, + O: Activate, +{ + pub fn new(input: Layer, hidden: Stack, output: Layer) -> Self { + Self { + input, + hidden, + output, + } + } + + pub fn input(&self) -> &Layer { + &self.input + } + + pub fn hidden(&self) -> &Stack { + &self.hidden + } + + pub fn output(&self) -> &Layer { + &self.output + } +} +impl MLP +where + T: Float, + I: Activate + Default, + H: Activate + Default, + O: Activate + Default, +{ + // pub fn create>(inputs: Sh, hidden: impl IntoIterator, outputs: Sh) -> Self { + // let input = LayerShape::from_dimension(inputs); + // let hidden = Stack::new().build_layers(hidden); + // let output = LayerShape::from_dimension(outputs); + + // Self::new(Layer::from(input), hidden, Layer::new(output)) + // } +} + +impl MLP +where + T: Float, + I: Activate + Clone, + H: Activate + Clone, + O: Activate + Clone, +{ + pub fn validate_dims(&self) -> bool { + self.hidden.validate_shapes() + && self.input.features().outputs() == self.hidden.first().unwrap().features().inputs() + && self.output.features().inputs() == self.hidden.last().unwrap().features().outputs() + } +} + +impl Forward> for MLP +where + T: NdFloat, + I: Activate + Clone, + H: Activate + Clone, + O: Activate + Clone, +{ + type Output = Array2; + + fn forward(&self, args: &Array2) -> Self::Output { + let mut out = self.input.forward(args); + for layer in self.hidden.clone().into_iter() { + out = layer.forward(&out); + } + self.output.forward(&out) + } +} diff --git a/ml/neural/src/nn/ffn/mod.rs b/ml/neural/src/nn/ffn/mod.rs index e85fddf4..d63d503f 100644 --- a/ml/neural/src/nn/ffn/mod.rs +++ b/ml/neural/src/nn/ffn/mod.rs @@ -4,8 +4,9 @@ */ //! # Feed Forward Neural Network //! -pub use self::{model::*, utils::*}; +pub use self::{mlp::*, model::*, utils::*}; +pub(crate) mod mlp; pub(crate) mod model; use ndarray::prelude::{Array, Array2, Dimension, Ix2}; @@ -25,9 +26,7 @@ where { type Opt; - fn apply_gradients(&mut self, gamma: &T, grad: &Array); - - fn backward(&mut self, args: &Array2, targets: &Array2, opt: &Self::Opt) -> Array2; + fn backward(&mut self, args: &Array2, targets: &Array, opt: &Self::Opt) -> Array2; fn forward(&self, args: &Array2) -> Array; } diff --git a/ml/neural/src/nn/ffn/model.rs b/ml/neural/src/nn/ffn/model.rs index e2a0d764..feaf351e 100644 --- a/ml/neural/src/nn/ffn/model.rs +++ b/ml/neural/src/nn/ffn/model.rs @@ -2,3 +2,14 @@ Appellation: model Contrib: FL03 */ +use crate::prelude::Activate; +use ndarray::prelude::Array2; +use num::Float; + +pub trait Model +where + A: Activate, + T: Float, +{ + fn forward(&self, args: &Array2) -> Array2; +} diff --git a/ml/neural/src/nn/gnn/mod.rs b/ml/neural/src/nn/gnn/mod.rs index b872f4f9..644ca496 100644 --- a/ml/neural/src/nn/gnn/mod.rs +++ b/ml/neural/src/nn/gnn/mod.rs @@ -10,8 +10,6 @@ pub(crate) mod model; use num::Float; - - pub trait GNN where T: Float, diff --git a/ml/neural/src/nn/mod.rs b/ml/neural/src/nn/mod.rs index a55fb8cd..81de33ec 100644 --- a/ml/neural/src/nn/mod.rs +++ b/ml/neural/src/nn/mod.rs @@ -47,7 +47,11 @@ pub trait Compile {} pub trait Train {} -pub trait Predict where D: Dimension, T: Float, { +pub trait Predict +where + D: Dimension, + T: Float, +{ type Output; fn predict(&self, input: &Array) -> BoxResult; diff --git a/ml/optim/src/grad/descent.rs b/ml/optim/src/grad/descent.rs index 3d14d5d7..e213582b 100644 --- a/ml/optim/src/grad/descent.rs +++ b/ml/optim/src/grad/descent.rs @@ -2,18 +2,24 @@ Appellation: grad Contrib: FL03 */ -use crate::neural::prelude::{Forward, Layer,}; +use crate::neural::prelude::{Forward, Layer}; use ndarray::prelude::{Array1, Array2, NdFloat}; use ndarray_stats::DeviationExt; use num::{Float, Signed}; #[derive(Clone)] -pub struct GradientDescent where T: Float { +pub struct GradientDescent +where + T: Float, +{ pub gamma: T, model: Layer, } -impl GradientDescent where T: Float { +impl GradientDescent +where + T: Float, +{ pub fn new(gamma: T, model: Layer) -> Self { Self { gamma, model } } @@ -51,19 +57,18 @@ impl GradientDescent where T: Float { self.model = model; self } - } -impl GradientDescent where T: NdFloat + Signed { - - +impl GradientDescent +where + T: NdFloat + Signed, +{ pub fn gradient( &mut self, data: &Array2, targets: &Array2, grad: impl Fn(&Array2) -> Array2, ) -> anyhow::Result { - let lr = self.gamma(); let ns = T::from(data.shape()[0]).unwrap(); let pred = self.model.forward(data); @@ -74,7 +79,8 @@ impl GradientDescent where T: NdFloat + Signed { let dz = errors * grad(&pred); let dw = data.t().dot(&dz) / scale; - self.model_mut().update_with_gradient(lr, &dw.t().to_owned()); + self.model_mut() + .update_with_gradient(lr, &dw.t().to_owned()); let loss = targets.mean_sq_err(&self.model().forward(data))?; Ok(T::from(loss).unwrap()) @@ -124,6 +130,5 @@ mod tests { }; assert!(l1 > l2); - } } diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index fa1af69d..8d23b3c0 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -27,21 +27,24 @@ pub struct DescentParams { pub(crate) mod utils { use crate::neural::prelude::{Forward, Parameterized, Params}; - use ndarray::prelude::{Array, Array1, Array2, Dimension, Ix2, NdFloat}; + use ndarray::linalg::Dot; + use ndarray::prelude::{Array, Array1, Array2, Dimension, NdFloat}; use ndarray_stats::DeviationExt; use num::{FromPrimitive, Signed}; - pub fn gradient( + pub fn gradient( gamma: T, model: &mut A, data: &Array2, - targets: &Array2, - grad: impl Fn(&Array2) -> Array2, + targets: &Array, + grad: impl Fn(&Array) -> Array, ) -> f64 where - A: Forward, Output = Array2> + Parameterized, + A: Forward, Output = Array> + Parameterized, + D: Dimension, T: FromPrimitive + NdFloat + Signed, - >::Params: Params + 'static, + >::Params: Params + 'static, + Array2: Dot, Output = Array>, { let (_samples, _inputs) = data.dim(); let pred = model.forward(data); @@ -52,7 +55,7 @@ pub(crate) mod utils { // compute the gradient of the objective function w.r.t. the model's weights let dz = &errors * grad(&pred); // compute the gradient of the objective function w.r.t. the model's weights - let dw = data.t().dot(&dz) / ns; + let dw = data.t().to_owned().dot(&dz) / ns; // compute the gradient of the objective function w.r.t. the model's bias // let db = dz.sum_axis(Axis(0)) / ns; // // Apply the gradients to the model's learnable parameters From 94c4fc981820f4e3631fb02702bdf122db8b9de2 Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 1 Dec 2023 13:34:56 -0600 Subject: [PATCH 073/118] update Signed-off-by: FL03 --- Cargo.toml | 1 + core/src/specs.rs | 16 +- ml/neural/src/errors/error.rs | 53 ++++ ml/neural/src/errors/mod.rs | 9 + ml/neural/src/func/block.rs | 32 ++ ml/neural/src/func/loss/mod.rs | 10 +- ml/neural/src/func/loss/regress.rs | 38 ++- ml/neural/src/func/mod.rs | 40 ++- ml/neural/src/{ => func}/prop/mod.rs | 0 ml/neural/src/{ => func}/prop/modes.rs | 0 ml/neural/src/{ => func}/prop/propagation.rs | 0 ml/neural/src/layers/cmp/features.rs | 2 +- ml/neural/src/layers/exp/config.rs | 13 + ml/neural/src/layers/exp/mod.rs | 3 +- ml/neural/src/layers/exp/wrapper.rs | 17 +- ml/neural/src/layers/mod.rs | 1 - ml/neural/src/layers/params.rs | 24 +- ml/neural/src/layers/stack.rs | 20 +- ml/neural/src/lib.rs | 6 +- ml/neural/src/models/config.rs | 12 + ml/neural/src/models/mod.rs | 19 +- ml/neural/src/models/model.rs | 31 +- ml/neural/src/models/stack.rs | 292 +++++++++++++++++++ ml/neural/src/neurons/mod.rs | 3 +- ml/neural/src/neurons/node.rs | 12 + ml/neural/src/neurons/perceptron.rs | 35 +++ ml/neural/src/nn/ffn/mlp.rs | 30 +- ml/neural/src/nn/ffn/mod.rs | 32 +- ml/neural/src/nn/mod.rs | 26 +- ml/optim/src/grad/gradient.rs | 41 ++- 30 files changed, 733 insertions(+), 85 deletions(-) create mode 100644 ml/neural/src/errors/error.rs create mode 100644 ml/neural/src/errors/mod.rs create mode 100644 ml/neural/src/func/block.rs rename ml/neural/src/{ => func}/prop/mod.rs (100%) rename ml/neural/src/{ => func}/prop/modes.rs (100%) rename ml/neural/src/{ => func}/prop/propagation.rs (100%) create mode 100644 ml/neural/src/layers/exp/config.rs create mode 100644 ml/neural/src/models/stack.rs create mode 100644 ml/neural/src/neurons/perceptron.rs diff --git a/Cargo.toml b/Cargo.toml index 116fe71e..5e2db1bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ concision-core = { path = "core", version = "0.1.12" } computare = { features = ["full"], branch = "v0.1.0", git = "https://github.com/FL03/computare", version = "0.1.0" } anyhow = "1" +itertools = { features = [], version = "0.12" } lazy_static = "1" linfa = { features = [], version = "0.7" } ndarray = { features = ["serde-1"], version = "0.15" } diff --git a/core/src/specs.rs b/core/src/specs.rs index fdc8002a..7d01a3db 100644 --- a/core/src/specs.rs +++ b/core/src/specs.rs @@ -41,9 +41,21 @@ where } pub trait Apply { - fn apply(&self, f: F) -> T + fn apply(&self, f: F) -> Self where - F: Fn(&Self) -> T; + F: Fn(&T) -> T; +} + +impl Apply for Array +where + D: Dimension, +{ + fn apply(&self, f: F) -> Self + where + F: Fn(&T) -> T, + { + self.map(f) + } } pub trait ApplyTo { diff --git a/ml/neural/src/errors/error.rs b/ml/neural/src/errors/error.rs new file mode 100644 index 00000000..ea2128a5 --- /dev/null +++ b/ml/neural/src/errors/error.rs @@ -0,0 +1,53 @@ +/* + Appellation: error + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use smart_default::SmartDefault; +use strum::{Display, EnumIs, EnumIter, EnumVariantNames}; + +#[derive( + Clone, + Debug, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, + SmartDefault, +)] +#[non_exhaustive] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum Errors { + Async, + Codec, + Connection, + Custom(String), + Data, + Dimension, + #[default] + Error, + Execution, + IO, + Null, + Parse, + Process, + Runtime, + Syntax, + Unknown, +} + +pub enum NetworkError {} + +pub enum ActivationError {} + +pub enum LayerError {} + +pub enum ModelError {} diff --git a/ml/neural/src/errors/mod.rs b/ml/neural/src/errors/mod.rs new file mode 100644 index 00000000..267dace8 --- /dev/null +++ b/ml/neural/src/errors/mod.rs @@ -0,0 +1,9 @@ +/* + Appellation: errors + Contrib: FL03 +*/ +pub use self::{error::*, utils::*}; + +pub(crate) mod error; + +pub(crate) mod utils {} diff --git a/ml/neural/src/func/block.rs b/ml/neural/src/func/block.rs new file mode 100644 index 00000000..02d6669c --- /dev/null +++ b/ml/neural/src/func/block.rs @@ -0,0 +1,32 @@ +/* + Appellation: block + Contrib: FL03 +*/ +use crate::func::activate::{Activate, Linear, ReLU, Softmax}; +use num::Float; +use std::marker::PhantomData; + +pub trait FnBlock { + fn apply(&self, data: T) -> T; +} + +#[derive(Clone)] +pub struct FuncBlock { + method: Vec T>, +} + +pub struct FFNBlock +where + I: Activate, + H: Activate, + O: Activate, + T: Float, +{ + _args: PhantomData, + input: I, + hidden: H, + output: O, +} + +#[cfg(test)] +mod tests {} diff --git a/ml/neural/src/func/loss/mod.rs b/ml/neural/src/func/loss/mod.rs index 1a7fb86f..b6b535cc 100644 --- a/ml/neural/src/func/loss/mod.rs +++ b/ml/neural/src/func/loss/mod.rs @@ -19,11 +19,17 @@ use ndarray::prelude::{Array, Array1, Array2, Dimension, NdFloat}; use num::{Float, FromPrimitive}; use std::ops; -pub trait Loss +pub trait Loss { + type Output; + + fn loss(&self, pred: &T, target: &T) -> Self::Output; +} + +pub trait Losses where T: Float, { - fn loss(&self, pred: &Array, target: &Array1) -> T; + fn loss(&self, pred: &Array, target: &Array) -> Array; } // pub type LinearWeightGradient = fn() diff --git a/ml/neural/src/func/loss/regress.rs b/ml/neural/src/func/loss/regress.rs index 65f8fb49..61deac0c 100644 --- a/ml/neural/src/func/loss/regress.rs +++ b/ml/neural/src/func/loss/regress.rs @@ -3,8 +3,7 @@ Contrib: FL03 */ use super::Loss; -use ndarray::prelude::{Array, Array1, NdFloat}; -use ndarray::Dimension; +use ndarray::prelude::{Array, Dimension, NdFloat}; use num::Float; use std::ops; @@ -36,11 +35,14 @@ where } } -impl Loss for HuberLoss +impl Loss> for HuberLoss where - T: Float + ops::AddAssign, + D: Dimension, + T: NdFloat, { - fn loss(&self, pred: &Array, target: &Array1) -> T { + type Output = T; + + fn loss(&self, pred: &Array, target: &Array) -> Self::Output { let half = T::from(0.5).unwrap(); let mut loss = T::zero(); for (x, y) in pred.iter().cloned().zip(target.iter().cloned()) { @@ -59,11 +61,14 @@ where pub struct MeanAbsoluteError; -impl Loss for MeanAbsoluteError +impl Loss> for MeanAbsoluteError where + D: Dimension, T: Float + ops::AddAssign + ops::DivAssign, { - fn loss(&self, pred: &Array, target: &Array1) -> T { + type Output = T; + + fn loss(&self, pred: &Array, target: &Array) -> Self::Output { let mut res = T::zero(); for (p, t) in pred.iter().cloned().zip(target.iter().cloned()) { res += (p - t).abs(); @@ -75,16 +80,19 @@ where pub struct MeanSquaredError; -impl Loss for MeanSquaredError +impl Loss> for MeanSquaredError where + D: Dimension, T: NdFloat, { - fn loss(&self, pred: &Array, target: &Array1) -> T { - let mut res = T::zero(); - for (p, t) in pred.iter().cloned().zip(target.iter().cloned()) { - res += (p - t).powi(2); - } - res /= T::from(pred.len()).unwrap(); - res + type Output = T; + + fn loss(&self, pred: &Array, target: &Array) -> Self::Output { + let res = pred + .iter() + .cloned() + .zip(target.iter().cloned()) + .fold(T::zero(), |i, (p, t)| i + (p - t).powi(2)); + res / T::from(pred.len()).unwrap() } } diff --git a/ml/neural/src/func/mod.rs b/ml/neural/src/func/mod.rs index fb953059..7df9245c 100644 --- a/ml/neural/src/func/mod.rs +++ b/ml/neural/src/func/mod.rs @@ -13,14 +13,50 @@ //! ## Loss //! //! The loss functions are implemented as structs that implement the `Fn` trait. -pub use self::{rms::*, utils::*}; +pub use self::{block::*, rms::*, utils::*}; pub mod activate; pub mod loss; +pub mod prop; +pub(crate) mod block; pub(crate) mod rms; -pub(crate) mod utils {} +pub trait Lin { + type Output; + + fn linear(&self, args: &T) -> Self::Output; +} + +pub(crate) mod utils { + use ndarray::linalg::Dot; + use ndarray::prelude::{Array, Dimension}; + use num::Float; + use std::ops; + + pub fn lin( + args: &Array, + weights: &Array, + bias: &Array, + ) -> Array + where + A: Dimension, + D: Dimension, + O: Dimension, + T: Float, + Array: Dot, Output = Array>, + Array: ops::Add, Output = Array>, + { + args.dot(weights) + bias.clone() + } + + pub fn slope_intercept(args: T, slope: T, intercept: T) -> T + where + T: ops::Add + ops::Mul, + { + args * slope + intercept + } +} #[cfg(test)] mod tests { diff --git a/ml/neural/src/prop/mod.rs b/ml/neural/src/func/prop/mod.rs similarity index 100% rename from ml/neural/src/prop/mod.rs rename to ml/neural/src/func/prop/mod.rs diff --git a/ml/neural/src/prop/modes.rs b/ml/neural/src/func/prop/modes.rs similarity index 100% rename from ml/neural/src/prop/modes.rs rename to ml/neural/src/func/prop/modes.rs diff --git a/ml/neural/src/prop/propagation.rs b/ml/neural/src/func/prop/propagation.rs similarity index 100% rename from ml/neural/src/prop/propagation.rs rename to ml/neural/src/func/prop/propagation.rs diff --git a/ml/neural/src/layers/cmp/features.rs b/ml/neural/src/layers/cmp/features.rs index f25f7b59..daf249a9 100644 --- a/ml/neural/src/layers/cmp/features.rs +++ b/ml/neural/src/layers/cmp/features.rs @@ -82,7 +82,7 @@ where // } pub trait FromFeatures { - fn from_features(features: LayerShape) -> Self; + fn from_features(features: Sh) -> Self; } pub trait IntoFeatures { diff --git a/ml/neural/src/layers/exp/config.rs b/ml/neural/src/layers/exp/config.rs new file mode 100644 index 00000000..6db6a7ab --- /dev/null +++ b/ml/neural/src/layers/exp/config.rs @@ -0,0 +1,13 @@ +/* + Appellation: config + Contrib: FL03 +*/ +use crate::layers::{LayerKind, LayerShape}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct LayerConfig { + pub features: LayerShape, + kind: LayerKind, + name: String, +} diff --git a/ml/neural/src/layers/exp/mod.rs b/ml/neural/src/layers/exp/mod.rs index 3890a3ec..86c283d2 100644 --- a/ml/neural/src/layers/exp/mod.rs +++ b/ml/neural/src/layers/exp/mod.rs @@ -3,8 +3,9 @@ Contrib: FL03 */ //! # Experimental Layers -pub use self::{layer::*, sublayer::*, utils::*, wrapper::*}; +pub use self::{config::*, layer::*, sublayer::*, utils::*, wrapper::*}; +pub(crate) mod config; pub(crate) mod layer; pub(crate) mod sublayer; pub(crate) mod wrapper; diff --git a/ml/neural/src/layers/exp/wrapper.rs b/ml/neural/src/layers/exp/wrapper.rs index 0a8f2c7e..ee824e1a 100644 --- a/ml/neural/src/layers/exp/wrapper.rs +++ b/ml/neural/src/layers/exp/wrapper.rs @@ -2,16 +2,29 @@ Appellation: sublayers Contrib: FL03 */ -use crate::layers::LayerParams; +use crate::func::activate::Activate; +use crate::layers::Layer; use ndarray::prelude::Array2; use num::Float; +pub trait Wrap { + type Output; + + fn wrap(&self, obj: T) -> Self::Output; +} + pub trait Wrapper where T: Float, { fn apply(&self, data: &Array2) -> Array2; - fn params(&self) -> &LayerParams; + fn wrap(&self, layer: Layer) + where + A: Activate; + + fn wrapper(&self) -> &Self; + + fn wrapper_mut(&mut self) -> &mut Self; } diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index cbc8f014..fe06f625 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -27,7 +27,6 @@ where fn features(&self) -> LayerShape; fn name(&self) -> &str; fn params(&self) -> &LayerParams; - fn position(&self) -> LayerPosition; fn is_biased(&self) -> bool; } diff --git a/ml/neural/src/layers/params.rs b/ml/neural/src/layers/params.rs index 7c2d9e15..9e7f1c47 100644 --- a/ml/neural/src/layers/params.rs +++ b/ml/neural/src/layers/params.rs @@ -58,6 +58,15 @@ where } } +impl LayerParams +where + T: Float + 'static, +{ + pub fn update_with_gradient(&mut self, gamma: T, gradient: &Array2) { + self.weights_mut().scaled_add(-gamma, gradient); + } +} + impl LayerParams where T: NdFloat, @@ -139,14 +148,25 @@ where } } +impl Forward> for LayerParams +where + T: NdFloat, +{ + type Output = Array1; + + fn forward(&self, input: &Array1) -> Self::Output { + input.dot(self.weights()) + self.bias() + } +} + impl Forward> for LayerParams where T: NdFloat, { type Output = Array2; - fn forward(&self, input: &Array2) -> Array2 { - input.dot(&self.weights) + &self.bias + fn forward(&self, input: &Array2) -> Self::Output { + input.dot(self.weights()) + self.bias() } } diff --git a/ml/neural/src/layers/stack.rs b/ml/neural/src/layers/stack.rs index 65e47bca..0ea51b61 100644 --- a/ml/neural/src/layers/stack.rs +++ b/ml/neural/src/layers/stack.rs @@ -4,23 +4,15 @@ */ use crate::layers::{Layer, LayerShape}; use crate::prelude::{Activate, Features, Linear, Parameterized}; -// use ndarray::{IntoDimension, Ix2}; use num::Float; use serde::{Deserialize, Serialize}; use std::ops; -pub trait Layers: IntoIterator> -where - A: Activate, - T: Float, -{ -} - pub trait StackExt where A: Activate, T: Float, - Self: Clone + Layers + IntoIterator>, + Self: Clone + IntoIterator>, { fn validate_params(&self) -> bool { let layers = self.clone().into_iter().collect::>(); @@ -65,6 +57,16 @@ where // } // self // } + pub fn square(layers: usize, inputs: usize, outputs: usize) -> Self { + let mut children = Vec::with_capacity(layers); + children.push(Layer::::from(LayerShape::new(inputs, outputs))); + + for _ in 1..layers { + children.push(Layer::::from(LayerShape::new(outputs, outputs))); + } + Self::from_iter(children) + } + pub fn build_layers(mut self, shapes: impl IntoIterator) -> Self { // let shapes = shapes.into_iter().map(|s| (s.inputs(), s.outputs())); for (inputs, outputs) in shapes.into_iter() { diff --git a/ml/neural/src/lib.rs b/ml/neural/src/lib.rs index 72ab27b0..95b592b8 100644 --- a/ml/neural/src/lib.rs +++ b/ml/neural/src/lib.rs @@ -14,6 +14,7 @@ pub(crate) mod specs; pub(crate) mod utils; pub mod arch; +pub mod errors; pub mod func; pub mod layers; pub mod models; @@ -21,19 +22,18 @@ pub mod neurons; pub mod nn; pub mod ops; pub mod params; -pub mod prop; pub(crate) use concision_core as core; pub mod prelude { pub use crate::arch::*; - pub use crate::func::{activate::*, loss::*}; + pub use crate::errors::*; + pub use crate::func::{activate::*, loss::*, prop::*, rms::*}; pub use crate::layers::*; pub use crate::neurons::*; pub use crate::nn::*; pub use crate::ops::*; pub use crate::params::*; - pub use crate::prop::*; pub use crate::primitives::*; pub use crate::specs::*; diff --git a/ml/neural/src/models/config.rs b/ml/neural/src/models/config.rs index c592d89a..64d24648 100644 --- a/ml/neural/src/models/config.rs +++ b/ml/neural/src/models/config.rs @@ -2,7 +2,19 @@ Appellation: params Contrib: FL03 */ +use serde::{Deserialize, Serialize}; +#[derive(Clone, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] pub struct ModelConfig { pub layers: usize, } + +impl ModelConfig { + pub fn new(layers: usize) -> Self { + Self { layers } + } + + pub fn layers(&self) -> usize { + self.layers + } +} diff --git a/ml/neural/src/models/mod.rs b/ml/neural/src/models/mod.rs index 217f8457..6d5b0416 100644 --- a/ml/neural/src/models/mod.rs +++ b/ml/neural/src/models/mod.rs @@ -4,10 +4,11 @@ */ //! # Model //! -pub use self::{config::*, model::*, utils::*}; +pub use self::{config::*, model::*, stack::*, utils::*}; pub(crate) mod config; pub(crate) mod model; +pub(crate) mod stack; use crate::prelude::Forward; use ndarray::prelude::Array2; @@ -19,17 +20,15 @@ where { fn add_module(&mut self, module: impl Module); - fn layers(&self) -> &[impl Forward, Output = Array2>]; + fn compile(&mut self); + /// Returns a collection of all proceeding [Module]s in the network + fn children(&self) -> &Vec>; - fn name(&self) -> &str; + fn children_mut(&mut self) -> &mut Vec>; + /// Returns a collection of all [Module]s in the network + fn modules(&self) -> Vec<&impl Module>; - fn forward(&self, input: Array2) -> Array2 { - let mut output = input; - for layer in self.layers() { - output = layer.forward(&output); - } - output - } + fn name(&self) -> &str; } pub(crate) mod utils {} diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs index b1ca0903..8e81ff9b 100644 --- a/ml/neural/src/models/model.rs +++ b/ml/neural/src/models/model.rs @@ -2,17 +2,18 @@ Appellation: model Contrib: FL03 */ -use super::ModelConfig; -use crate::prelude::{Forward, ForwardDyn}; -use ndarray::prelude::Array2; +use super::{ModelConfig, ModelParams}; +use crate::prelude::{Forward, LayerParams}; +use ndarray::prelude::{Array2, NdFloat}; use num::Float; +#[derive(Clone, Debug)] pub struct Model where T: Float, { config: ModelConfig, - layers: Vec>, + params: ModelParams, } impl Model @@ -22,7 +23,7 @@ where pub fn new(config: ModelConfig) -> Self { Self { config, - layers: Vec::new(), + params: ModelParams::new(), } } @@ -34,16 +35,12 @@ where &mut self.config } - pub fn layers(&self) -> &[ForwardDyn] { - &self.layers + pub fn params(&self) -> &ModelParams { + &self.params } - pub fn layers_mut(&mut self) -> &mut [ForwardDyn] { - &mut self.layers - } - - pub fn add_layer(&mut self, layer: ForwardDyn) { - self.layers.push(layer); + pub fn params_mut(&mut self) -> &mut ModelParams { + &mut self.params } } @@ -51,22 +48,22 @@ impl IntoIterator for Model where T: Float, { - type Item = ForwardDyn; + type Item = LayerParams; type IntoIter = std::vec::IntoIter; fn into_iter(self) -> Self::IntoIter { - self.layers.into_iter() + self.params.into_iter() } } impl Forward> for Model where - T: Float, + T: NdFloat, { type Output = Array2; fn forward(&self, input: &Array2) -> Array2 { - let mut iter = self.layers().into_iter(); + let mut iter = self.clone().into_iter(); let mut output = iter.next().unwrap().forward(input); for layer in iter { diff --git a/ml/neural/src/models/stack.rs b/ml/neural/src/models/stack.rs new file mode 100644 index 00000000..822cf98b --- /dev/null +++ b/ml/neural/src/models/stack.rs @@ -0,0 +1,292 @@ +/* + Appellation: stack + Contrib: FL03 +*/ +use crate::prelude::{Features, LayerParams, LayerShape}; +use ndarray::prelude::{Dimension, Ix2}; +use ndarray::IntoDimension; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use num::Float; +use serde::{Deserialize, Serialize}; +use std::ops; + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +pub struct ModelParams +where + T: Float, +{ + children: Vec>, +} + +impl ModelParams +where + T: Float, +{ + pub fn new() -> Self { + Self { + children: Vec::new(), + } + } + + pub fn with_capacity(depth: usize) -> Self { + Self { + children: Vec::with_capacity(depth), + } + } + + pub fn with_shapes(shapes: impl IntoIterator) -> Self + where + Sh: IntoDimension, + { + let mut children = Vec::new(); + for (inputs, outputs) in shapes + .into_iter() + .map(|s| s.into_dimension().into_pattern()) + { + let features = LayerShape::new(inputs, outputs); + children.push(LayerParams::new(features)); + } + Self { children } + } + + pub fn is_empty(&self) -> bool { + self.children.is_empty() + } + + pub fn build_layers(mut self, shapes: impl IntoIterator) -> Self { + // let shapes = shapes.into_iter().map(|s| (s.inputs(), s.outputs())); + for (inputs, outputs) in shapes.into_iter() { + let features = LayerShape::new(inputs, outputs); + self.children.push(LayerParams::new(features)); + } + self + } + + pub fn len(&self) -> usize { + self.children.len() + } + + pub fn params(&self) -> &[LayerParams] { + &self.children + } + + pub fn params_mut(&mut self) -> &mut [LayerParams] { + &mut self.children + } + + pub fn pop(&mut self) -> Option> { + self.children.pop() + } + + pub fn push(&mut self, params: LayerParams) { + self.children.push(params); + } + + pub fn validate_shapes(&self) -> bool { + let mut dim = true; + for (i, layer) in self.children[..(self.len() - 1)].iter().enumerate() { + dim = dim && layer.features().outputs() == self.children[i + 1].features().inputs(); + } + dim + } +} + +impl ModelParams +where + T: Float + SampleUniform, +{ + pub fn init_layers(mut self, biased: bool) -> Self { + self.children + .iter_mut() + .for_each(|l| *l = l.clone().init(biased)); + self + } +} + +impl AsRef<[LayerParams]> for ModelParams +where + T: Float, +{ + fn as_ref(&self) -> &[LayerParams] { + &self.children + } +} + +impl AsMut<[LayerParams]> for ModelParams +where + T: Float, +{ + fn as_mut(&mut self) -> &mut [LayerParams] { + &mut self.children + } +} + +impl FromIterator> for ModelParams +where + T: Float, +{ + fn from_iter(iter: I) -> Self + where + I: IntoIterator>, + { + Self { + children: iter.into_iter().collect(), + } + } +} + +impl IntoIterator for ModelParams +where + T: Float, +{ + type Item = LayerParams; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.children.into_iter() + } +} + +impl ops::Index for ModelParams +where + T: Float, +{ + type Output = LayerParams; + + fn index(&self, index: usize) -> &Self::Output { + &self.children[index] + } +} + +impl ops::IndexMut for ModelParams +where + T: Float, +{ + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.children[index] + } +} + +impl ops::Index> for ModelParams +where + T: Float, +{ + type Output = [LayerParams]; + + fn index(&self, index: ops::Range) -> &Self::Output { + &self.children[index] + } +} + +impl ops::IndexMut> for ModelParams +where + T: Float, +{ + fn index_mut(&mut self, index: ops::Range) -> &mut Self::Output { + &mut self.children[index] + } +} + +impl ops::Index> for ModelParams +where + T: Float, +{ + type Output = [LayerParams]; + + fn index(&self, index: ops::RangeFrom) -> &Self::Output { + &self.children[index] + } +} + +impl ops::IndexMut> for ModelParams +where + T: Float, +{ + fn index_mut(&mut self, index: ops::RangeFrom) -> &mut Self::Output { + &mut self.children[index] + } +} + +impl ops::Index for ModelParams +where + T: Float, +{ + type Output = [LayerParams]; + + fn index(&self, index: ops::RangeFull) -> &Self::Output { + &self.children[index] + } +} + +impl ops::IndexMut for ModelParams +where + T: Float, +{ + fn index_mut(&mut self, index: ops::RangeFull) -> &mut Self::Output { + &mut self.children[index] + } +} + +impl ops::Index> for ModelParams +where + T: Float, +{ + type Output = [LayerParams]; + + fn index(&self, index: ops::RangeInclusive) -> &Self::Output { + &self.children[index] + } +} + +impl ops::IndexMut> for ModelParams +where + T: Float, +{ + fn index_mut(&mut self, index: ops::RangeInclusive) -> &mut Self::Output { + &mut self.children[index] + } +} + +impl ops::Index> for ModelParams +where + T: Float, +{ + type Output = [LayerParams]; + + fn index(&self, index: ops::RangeTo) -> &Self::Output { + &self.children[index] + } +} + +impl ops::IndexMut> for ModelParams +where + T: Float, +{ + fn index_mut(&mut self, index: ops::RangeTo) -> &mut Self::Output { + &mut self.children[index] + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::prelude::LayerShape; + + #[test] + fn test_model_params() { + let (inputs, outputs) = (5, 3); + + let shapes = [(inputs, outputs), (outputs, outputs), (outputs, 1)]; + + let params = ModelParams::::new() + .build_layers(shapes) + .init_layers(true); + + // validate the dimensions of the model params + assert!(params.validate_shapes()); + + for (layer, shape) in params.into_iter().zip(&shapes) { + assert_eq!(layer.features(), &LayerShape::new(shape.0, shape.1)); + } + } +} diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index 3b3110a0..65d51dbf 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -3,10 +3,11 @@ Contrib: FL03 */ //! # neurons -pub use self::{neuron::*, node::*, synapse::*, utils::*}; +pub use self::{neuron::*, node::*, perceptron::*, synapse::*, utils::*}; pub(crate) mod neuron; pub(crate) mod node; +pub(crate) mod perceptron; pub(crate) mod synapse; use crate::func::activate::Activate; diff --git a/ml/neural/src/neurons/node.rs b/ml/neural/src/neurons/node.rs index 25d9e955..860d6761 100644 --- a/ml/neural/src/neurons/node.rs +++ b/ml/neural/src/neurons/node.rs @@ -108,6 +108,18 @@ where } } +impl Forward> for Node +where + Self: Biased, + T: FromPrimitive + NdFloat, +{ + type Output = T; + + fn forward(&self, data: &Array1) -> Self::Output { + data.dot(&self.weights().t()) + self.bias().first().unwrap().clone() + } +} + impl Forward> for Node where Self: Biased, diff --git a/ml/neural/src/neurons/perceptron.rs b/ml/neural/src/neurons/perceptron.rs new file mode 100644 index 00000000..9a71a5c9 --- /dev/null +++ b/ml/neural/src/neurons/perceptron.rs @@ -0,0 +1,35 @@ +/* + Appellation: perceptron + Contrib: FL03 +*/ +use super::Node; +use crate::prelude::Forward; +use ndarray::prelude::{Array1, Array2, NdFloat}; +use num::{Float, FromPrimitive}; + +pub struct Perceptron +where + T: Float, +{ + node: Node, +} + +impl Perceptron +where + T: Float, +{ + pub fn new(node: Node) -> Self { + Self { node } + } +} + +impl Forward> for Perceptron +where + T: FromPrimitive + NdFloat, +{ + type Output = Array1; + + fn forward(&self, args: &Array2) -> Self::Output { + self.node.forward(args) + } +} diff --git a/ml/neural/src/nn/ffn/mlp.rs b/ml/neural/src/nn/ffn/mlp.rs index 35b82803..4a6d0351 100644 --- a/ml/neural/src/nn/ffn/mlp.rs +++ b/ml/neural/src/nn/ffn/mlp.rs @@ -5,7 +5,7 @@ //! # Multi-Layer Perceptron //! -use crate::func::activate::{Activate, Linear}; +use crate::func::activate::{Activate, Linear, ReLU, Softmax}; use crate::layers::{Layer, LayerShape, Stack}; use crate::prelude::{Features, Forward, Parameterized}; @@ -13,7 +13,7 @@ use ndarray::prelude::{Array2, Ix2, NdFloat}; use ndarray::IntoDimension; use num::Float; -pub struct MLP +pub struct MLP where T: Float, I: Activate, @@ -59,7 +59,31 @@ where H: Activate + Default, O: Activate + Default, { - // pub fn create>(inputs: Sh, hidden: impl IntoIterator, outputs: Sh) -> Self { + pub fn create(hidden: usize, inputs: usize, outputs: usize) -> Self + where + Sh: IntoDimension, + { + let input_shape = LayerShape::new(inputs, outputs); + let shape = LayerShape::new(outputs, outputs); + + let input = Layer::input(input_shape); + let stack = Stack::square(hidden, outputs, outputs); + let output = Layer::output(shape, hidden + 1); + Self::new(input, stack, output) + } + + pub fn from_features( + inputs: (usize, usize), + hidden: impl IntoIterator, + outputs: (usize, usize), + ) -> Self { + let input = Layer::input(LayerShape::from(inputs)); + let stack = Stack::new().build_layers(hidden); + let output = Layer::output(outputs.into(), stack.len() + 1); + Self::new(input, stack, output) + } + + // pub fn from_shapes(inputs: Sh, hidden: impl IntoIterator, outputs: Sh) -> Self where Sh: IntoDimension { // let input = LayerShape::from_dimension(inputs); // let hidden = Stack::new().build_layers(hidden); // let output = LayerShape::from_dimension(outputs); diff --git a/ml/neural/src/nn/ffn/mod.rs b/ml/neural/src/nn/ffn/mod.rs index d63d503f..7d4d8ae0 100644 --- a/ml/neural/src/nn/ffn/mod.rs +++ b/ml/neural/src/nn/ffn/mod.rs @@ -34,4 +34,34 @@ where pub(crate) mod utils {} #[cfg(tets)] -mod tests {} +mod tests { + use super::*; + use crate::core::prelude::linarr; + use crate::func::activate::{ReLU, Softmax}; + use crate::prelude::{Forward, Layer, LayerShape, Stack}; + use ndarray::prelude::Ix2; + + #[test] + fn test_mlp() { + let samples = 20; + let (inputs, outputs) = (5, 3); + let shapes = [(outputs, 4), (4, 4), (4, inputs)]; + + // sample data + let x = linarr::((samples, inputs)).unwrap(); + let _y = linarr::((samples, outputs)).unwrap(); + + // layers + let hidden = Stack::::new() + .build_layers(shapes) + .init_layers(true); + let input = Layer::::from(LayerShape::new(inputs, outputs)).init(false); + let output = Layer::::from(LayerShape::new(inputs, outputs)).init(false); + + let network = MLP::new(input, hidden, output); + assert!(network.validate_dims()); + + let pred = network.forward(&x); + assert_eq!(&pred.dim(), &(samples, outputs)); + } +} diff --git a/ml/neural/src/nn/mod.rs b/ml/neural/src/nn/mod.rs index 81de33ec..c730a11d 100644 --- a/ml/neural/src/nn/mod.rs +++ b/ml/neural/src/nn/mod.rs @@ -14,7 +14,7 @@ pub mod gnn; use crate::core::BoxResult; use crate::layers::Layer; use crate::Trainable; -use ndarray::prelude::{Array, Dimension, Ix2}; +use ndarray::prelude::{Array, Axis, Dimension, Ix2}; use num::Float; pub trait NeuralNet: Trainable @@ -45,7 +45,29 @@ where pub trait Compile {} -pub trait Train {} +pub trait Train +where + T: Float, +{ + fn train(&mut self, input: &Array, target: &Array) -> BoxResult; + + fn train_batch( + &mut self, + batch_size: usize, + input: &Array, + target: &Array, + ) -> BoxResult + where + T: std::iter::Sum, + { + let res = input + .axis_chunks_iter(Axis(0), batch_size) + .zip(target.axis_chunks_iter(Axis(0), batch_size)) + .map(|(x, y)| self.train(&x.to_owned(), &y.to_owned()).expect("")) + .sum(); + Ok(res) + } +} pub trait Predict where diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs index 151ffb0b..db7e825a 100644 --- a/ml/optim/src/grad/gradient.rs +++ b/ml/optim/src/grad/gradient.rs @@ -2,17 +2,19 @@ Appellation: grad Contrib: FL03 */ -use crate::neural::prelude::Params; -use ndarray::prelude::{Array1, Array2, NdFloat}; -use num::Float; +use crate::neural::models::ModelParams; +use crate::neural::prelude::{Forward, LayerParams}; +use ndarray::prelude::{Array2, NdFloat}; +use ndarray_stats::DeviationExt; +use num::{Float, Signed}; pub struct Grad where T: Float, { gamma: T, - params: Vec>, - objective: fn(&T) -> T, + params: Vec>, + objective: fn(&Array2) -> Array2, } impl Grad @@ -27,25 +29,42 @@ where &mut self.gamma } - pub fn objective(&self) -> fn(&T) -> T { + pub fn objective(&self) -> fn(&Array2) -> Array2 { self.objective } - pub fn params(&self) -> &Vec> { + pub fn model(&self) -> &[LayerParams] { &self.params } - pub fn params_mut(&mut self) -> &mut Vec> { + pub fn model_mut(&mut self) -> &mut [LayerParams] { &mut self.params } } impl Grad where - T: NdFloat, + T: NdFloat + Signed, { - pub fn step(&mut self, data: &Array2, targets: &Array1) -> anyhow::Result { - let cost = T::zero(); + pub fn step(&mut self, data: &Array2, targets: &Array2) -> anyhow::Result { + let grad = self.objective(); + let layers = self.model().len(); + let lr = self.gamma(); + let ns = T::from(data.shape()[0]).unwrap(); + + let mut cost = T::zero(); + let params = self.params.clone(); + + for (i, layer) in self.params[1..].iter_mut().enumerate() { + let pred = params[i - 1].forward(data); + let errors = &pred - targets; + let dz = errors * grad(&pred); + let dw = data.t().dot(&dz) / ns; + layer.update_with_gradient(lr, &dw.t().to_owned()); + let loss = targets.mean_sq_err(&pred)?; + cost += T::from(loss).unwrap(); + } + Ok(cost) } } From 9eaa56d8daf6e2ed207db0ccfade341061c38c47 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 2 Dec 2023 12:28:47 -0600 Subject: [PATCH 074/118] Add new modules and structs Signed-off-by: FL03 --- concision/examples/gradients.rs | 57 +------ core/src/lib.rs | 1 + core/src/params/kinds.rs | 34 ++++ core/src/params/mod.rs | 59 +++++++ core/src/params/param.rs | 84 +++++++++ core/src/specs.rs | 8 +- ml/neural/src/arch/mod.rs | 12 +- ml/neural/src/arch/shallow.rs | 51 +++++- ml/neural/src/errors/error.rs | 93 ++++++++-- ml/neural/src/func/activate/activator.rs | 161 ++++++++++++++++-- ml/neural/src/func/activate/mod.rs | 41 ++--- ml/neural/src/func/activate/nl/relu.rs | 4 +- ml/neural/src/func/activate/nl/sigmoid.rs | 4 +- ml/neural/src/func/activate/nl/softmax.rs | 4 +- ml/neural/src/func/activate/nl/tanh.rs | 4 +- ml/neural/src/func/loss/mod.rs | 40 ++++- ml/neural/src/func/loss/regress.rs | 59 ++++++- ml/neural/src/func/mod.rs | 9 + ml/neural/src/func/prop/mod.rs | 2 +- ml/neural/src/layers/cmp/features.rs | 93 +--------- ml/neural/src/layers/cmp/mod.rs | 93 ++++++++++ ml/neural/src/layers/exp/config.rs | 76 ++++++++- ml/neural/src/layers/exp/layer.rs | 76 +++------ ml/neural/src/layers/layer.rs | 91 +++++----- ml/neural/src/models/mod.rs | 12 +- ml/neural/src/models/{stack.rs => params.rs} | 0 ml/neural/src/nn/ffn/mlp.rs | 8 +- ml/neural/src/nn/ffn/mod.rs | 22 --- ml/neural/src/nn/ffn/model.rs | 49 +++++- ml/neural/src/nn/kinds.rs | 157 +++++++++++++++++ ml/neural/src/nn/mod.rs | 76 +-------- ml/neural/src/nn/position.rs | 6 +- ml/neural/src/specs.rs | 49 +++--- ml/optim/examples/sgd.rs | 2 +- ml/optim/src/grad/descent.rs | 14 +- ml/optim/src/grad/gradient.rs | 29 ++-- ml/optim/src/grad/mod.rs | 21 ++- ml/optim/src/grad/sgd.rs | 2 +- ml/optim/src/lib.rs | 8 +- ml/optim/src/optimize/mod.rs | 44 ----- ml/optim/src/optimize/optimizer.rs | 67 -------- ml/optim/src/optimizer.rs | 33 ++++ ml/optim/src/params/mod.rs | 50 ++++++ ml/optim/src/specs.rs | 56 ++---- .../src/attention/multi/attention.rs | 2 +- ml/transformers/src/ffn/network.rs | 4 +- 46 files changed, 1214 insertions(+), 653 deletions(-) create mode 100644 core/src/params/kinds.rs create mode 100644 core/src/params/mod.rs create mode 100644 core/src/params/param.rs rename ml/neural/src/models/{stack.rs => params.rs} (100%) create mode 100644 ml/neural/src/nn/kinds.rs delete mode 100644 ml/optim/src/optimize/mod.rs delete mode 100644 ml/optim/src/optimize/optimizer.rs create mode 100644 ml/optim/src/optimizer.rs create mode 100644 ml/optim/src/params/mod.rs diff --git a/concision/examples/gradients.rs b/concision/examples/gradients.rs index a04cda6d..f1edec27 100644 --- a/concision/examples/gradients.rs +++ b/concision/examples/gradients.rs @@ -1,7 +1,6 @@ -use concision::prelude::{linarr, Features, Forward, LayerShape, ParameterizedExt}; - -use concision::neural::prelude::{Layer, Neuron, Objective, Sigmoid}; -use concision::optim::grad::*; +use concision::neural::prelude::{Layer, Sigmoid}; +use concision::optim::grad::gradient; +use concision::prelude::{linarr, Features, Forward, LayerShape}; use ndarray::prelude::Array1; @@ -11,56 +10,13 @@ fn main() -> anyhow::Result<()> { let features = LayerShape::new(inputs, outputs); - let (epochs, gamma) = (1000000, 0.05); - - // basic_descent(epochs, features, gamma)?; - - // sample_descent(epochs, features, gamma, samples)?; + let (epochs, gamma) = (1000, 0.005); sample_gradient(epochs, features, gamma, samples)?; Ok(()) } -pub fn basic_descent(epochs: usize, features: LayerShape, gamma: f64) -> anyhow::Result<()> { - let mut model = Neuron::::new(features.inputs()).init_weight(); - - println!( - "{:?}", - gradient_descent(model.weights_mut(), epochs, gamma, |w| Sigmoid::new() - .gradient(w)) - ); - Ok(()) -} - -pub fn sample_descent( - epochs: usize, - features: LayerShape, - gamma: f64, - samples: usize, -) -> anyhow::Result<()> { - // Generate some example data - let x = linarr((samples, features.inputs()))?; - let y = linarr((samples, features.outputs()))?; - - let model = Layer::from(features).init(false); - println!( - "Targets:\n\n{:?}\nPredictions:\n\n{:?}\n", - &y, - model.forward(&x) - ); - - let mut grad = GradientDescent::new(gamma, model); - let mut losses = Array1::zeros(epochs); - for e in 0..epochs { - let cost = grad.gradient(&x, &y, |w| Sigmoid::new().gradient(w))?; - losses[e] = cost; - } - println!("Losses:\n\n{:?}\n", &losses); - println!("Trained:\n\n{:?}", grad.model().forward(&x)); - Ok(()) -} - pub fn sample_gradient( epochs: usize, features: LayerShape, @@ -71,7 +27,7 @@ pub fn sample_gradient( let x = linarr((samples, features.inputs()))?; let y = linarr((samples, features.outputs()))?; - let mut model = Layer::::input(features).init(false); + let mut model = Layer::::from(features).init(false); println!( "Targets (dim):\t{:?}\nPredictions:\n\n{:?}\n", &y.shape(), @@ -80,7 +36,8 @@ pub fn sample_gradient( let mut losses = Array1::zeros(epochs); for e in 0..epochs { - let cost = gradient(gamma, &mut model, &x, &y, |w| Sigmoid::new().gradient(w)); + let cost = gradient(gamma, &mut model, &x, &y, Sigmoid); + // let cost = model.grad(gamma, &x, &y); losses[e] = cost; } println!("Losses:\n\n{:?}\n", &losses); diff --git a/core/src/lib.rs b/core/src/lib.rs index b62764ae..5da3975f 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -12,6 +12,7 @@ pub(crate) mod utils; pub mod epochs; pub mod errors; pub mod masks; +pub mod params; pub mod states; pub mod step; pub mod vars; diff --git a/core/src/params/kinds.rs b/core/src/params/kinds.rs new file mode 100644 index 00000000..6e1bf00a --- /dev/null +++ b/core/src/params/kinds.rs @@ -0,0 +1,34 @@ +/* + Appellation: kinds + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; + +#[derive( + Clone, + Debug, + Default, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[non_exhaustive] +#[repr(usize)] +#[serde(rename_all = "lowercase", tag = "kind")] +#[strum(serialize_all = "lowercase")] +pub enum ParamKind { + #[default] + Bias, + Weight, + Other(String), +} diff --git a/core/src/params/mod.rs b/core/src/params/mod.rs new file mode 100644 index 00000000..5816ee68 --- /dev/null +++ b/core/src/params/mod.rs @@ -0,0 +1,59 @@ +/* + Appellation: params + Contrib: FL03 +*/ +//! # Parameters +//! +//! ## Overview +//! +pub use self::{kinds::*, param::*, utils::*}; + +pub(crate) mod kinds; +pub(crate) mod param; + +use ndarray::prelude::{Array, Dimension, Ix2}; +use num::Float; + +pub trait Parameter +where + D: Dimension, + T: Float, +{ + /// Returns an owned reference of the param + fn param(&self) -> &Array; + /// Returns a mutable reference of the param + fn param_mut(&mut self) -> &mut Array; + /// Sets the param + fn set_param(&mut self, param: Array); +} + +pub trait Biased +where + D: Dimension, + T: Float, +{ + /// Returns an owned reference to the bias of the layer. + fn bias(&self) -> &Array; + /// Returns a mutable reference to the bias of the layer. + fn bias_mut(&mut self) -> &mut Array; + /// Sets the bias of the layer. + fn set_bias(&mut self, bias: Array); +} + +pub trait Weighted +where + D: Dimension, + T: Float, +{ + /// Returns an owned reference to the weights of the layer. + fn weights(&self) -> &Array; + /// Returns a mutable reference to the weights of the layer. + fn weights_mut(&mut self) -> &mut Array; + /// Sets the weights of the layer. + fn set_weights(&mut self, weights: Array); +} + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests {} diff --git a/core/src/params/param.rs b/core/src/params/param.rs new file mode 100644 index 00000000..c42f04e6 --- /dev/null +++ b/core/src/params/param.rs @@ -0,0 +1,84 @@ +/* + Appellation: param + Contrib: FL03 +*/ +use super::ParamKind; +use ndarray::prelude::{Array, Dimension, Ix2}; +use num::Float; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct Param +where + T: Float, + D: Dimension, +{ + kind: ParamKind, + name: String, + params: Array, +} + +impl Param +where + T: Float, + D: Dimension, +{ + pub fn new(kind: ParamKind, name: impl ToString, params: Array) -> Self { + Self { + kind, + name: name.to_string(), + params, + } + } + + pub fn kind(&self) -> &ParamKind { + &self.kind + } + + pub fn kind_mut(&mut self) -> &mut ParamKind { + &mut self.kind + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn name_mut(&mut self) -> &mut String { + &mut self.name + } + + pub fn params(&self) -> &Array { + &self.params + } + + pub fn params_mut(&mut self) -> &mut Array { + &mut self.params + } + + pub fn set_kind(&mut self, kind: ParamKind) { + self.kind = kind; + } + + pub fn set_name(&mut self, name: String) { + self.name = name; + } + + pub fn set_params(&mut self, params: Array) { + self.params = params; + } + + pub fn with_kind(mut self, kind: ParamKind) -> Self { + self.kind = kind; + self + } + + pub fn with_name(mut self, name: impl ToString) -> Self { + self.name = name.to_string(); + self + } + + pub fn with_params(mut self, params: Array) -> Self { + self.params = params; + self + } +} diff --git a/core/src/specs.rs b/core/src/specs.rs index 7d01a3db..e8b73a6f 100644 --- a/core/src/specs.rs +++ b/core/src/specs.rs @@ -2,13 +2,13 @@ Appellation: specs Contrib: FL03 */ -use ndarray::prelude::{Array, Axis, Dimension, Ix2}; +use ndarray::prelude::{Array, Axis, Dimension, Ix2, NdFloat}; use ndarray::IntoDimension; // use ndarray::linalg::Dot; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Uniform}; use ndarray_rand::RandomExt; -use num::{Float, One, Zero}; +use num::{Float, FromPrimitive, One, Signed, Zero}; use std::ops; pub trait Arithmetic: @@ -70,6 +70,10 @@ pub trait BinaryNum: One + Zero {} impl BinaryNum for T where T: One + Zero {} +pub trait FloatExt: FromPrimitive + NdFloat + Signed + SampleUniform {} + +impl FloatExt for T where T: FromPrimitive + NdFloat + Signed + SampleUniform {} + pub trait Pair { fn pair(&self) -> (A, B); } diff --git a/ml/neural/src/arch/mod.rs b/ml/neural/src/arch/mod.rs index 0da86531..8d528df1 100644 --- a/ml/neural/src/arch/mod.rs +++ b/ml/neural/src/arch/mod.rs @@ -31,7 +31,8 @@ pub(crate) mod utils {} mod tests { use super::*; use crate::core::prelude::linarr; - use crate::prelude::{Forward, Layer, LayerShape, Sigmoid, Stack}; + use crate::func::activate::{ReLU, Sigmoid, Softmax}; + use crate::prelude::{Forward, Layer, LayerShape, Stack}; use ndarray::prelude::Ix2; #[test] @@ -73,10 +74,13 @@ mod tests { let _y = linarr::((samples, outputs)).unwrap(); // layers - let input = Layer::::from(LayerShape::new(inputs, outputs)).init(false); - let output = Layer::::from(LayerShape::new(outputs, outputs)).init(false); + let in_features = LayerShape::new(inputs, outputs); + let out_features = LayerShape::new(outputs, outputs); + let input = Layer::::from(in_features).init(false); + let hidden = Layer::::from(out_features).init(false); + let output = Layer::::from(out_features).init(false); - let network = ShallowNetwork::new(input, output); + let network = ShallowNetwork::new(input, hidden, output); assert!(network.validate_dims()); let pred = network.forward(&x); diff --git a/ml/neural/src/arch/shallow.rs b/ml/neural/src/arch/shallow.rs index 352bdb2d..1a2dc023 100644 --- a/ml/neural/src/arch/shallow.rs +++ b/ml/neural/src/arch/shallow.rs @@ -2,30 +2,41 @@ Appellation: shallow Contrib: FL03 */ -use crate::func::activate::{Activate, Linear}; -use crate::prelude::{Features, Forward, Layer, Parameterized}; +use crate::func::activate::{Activate, Linear, ReLU, Softmax}; +use crate::prelude::{Features, Forward, Layer, LayerShape, Parameterized}; use ndarray::prelude::{Array2, NdFloat}; use num::Float; -pub struct ShallowNetwork +pub struct ShallowNetwork where T: Float, I: Activate, + H: Activate, O: Activate, { pub input: Layer, + pub hidden: Layer, pub output: Layer, } -impl ShallowNetwork +impl ShallowNetwork where T: Float, I: Activate, + H: Activate, O: Activate, { - pub fn new(input: Layer, output: Layer) -> Self { - Self { input, output } + pub fn new(input: Layer, hidden: Layer, output: Layer) -> Self { + Self { + input, + hidden, + output, + } + } + + pub fn hidden(&self) -> &Layer { + &self.hidden } pub fn input(&self) -> &Layer { @@ -37,19 +48,41 @@ where } pub fn validate_dims(&self) -> bool { - self.input.features().outputs() == self.output.features().inputs() + self.input.features().outputs() == self.hidden.features().inputs() + && self.hidden.features().outputs() == self.output.features().inputs() + } +} + +impl ShallowNetwork +where + T: Float, + I: Activate + Default, + H: Activate + Default, + O: Activate + Default, +{ + pub fn create(inputs: usize, outputs: usize) -> Self { + let s1 = LayerShape::new(inputs, outputs); + let s2 = LayerShape::new(outputs, outputs); + + let input = Layer::::from(s1); + let hidden = Layer::::new(s2); + let output = Layer::::new(s2); + + Self::new(input, hidden, output) } } -impl Forward> for ShallowNetwork +impl Forward> for ShallowNetwork where T: NdFloat, I: Activate, + H: Activate, O: Activate, { type Output = Array2; fn forward(&self, args: &Array2) -> Self::Output { - self.output.forward(&self.input.forward(args)) + self.output + .forward(&self.hidden.forward(&self.input.forward(args))) } } diff --git a/ml/neural/src/errors/error.rs b/ml/neural/src/errors/error.rs index ea2128a5..e155ea8c 100644 --- a/ml/neural/src/errors/error.rs +++ b/ml/neural/src/errors/error.rs @@ -25,26 +25,89 @@ use strum::{Display, EnumIs, EnumIter, EnumVariantNames}; #[non_exhaustive] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] -pub enum Errors { - Async, - Codec, - Connection, - Custom(String), +pub enum MlError { Data, Dimension, #[default] - Error, - Execution, - IO, - Null, - Parse, - Process, - Runtime, - Syntax, - Unknown, + Error(String), + Network(NetworkError), } -pub enum NetworkError {} +impl std::error::Error for MlError {} + +impl From> for MlError { + fn from(err: Box) -> Self { + Self::Error(err.to_string()) + } +} + +impl From for MlError { + fn from(err: String) -> Self { + Self::Error(err) + } +} + +impl From<&str> for MlError { + fn from(err: &str) -> Self { + Self::Error(err.to_string()) + } +} + +impl From for MlError { + fn from(err: NetworkError) -> Self { + Self::Network(err) + } +} + +#[derive( + Clone, + Debug, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, + SmartDefault, +)] +#[non_exhaustive] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum ComputeError { + #[default] + Compute(String), + ShapeError(String), +} + +#[derive( + Clone, + Debug, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, + SmartDefault, +)] +#[non_exhaustive] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum NetworkError { + Layer, + #[default] + Network(String), +} pub enum ActivationError {} diff --git a/ml/neural/src/func/activate/activator.rs b/ml/neural/src/func/activate/activator.rs index ebb7a5af..d4af82e7 100644 --- a/ml/neural/src/func/activate/activator.rs +++ b/ml/neural/src/func/activate/activator.rs @@ -2,26 +2,161 @@ Appellation: activator Contrib: FL03 */ -use super::Activate; +use super::{Activate, Gradient}; +use ndarray::prelude::{Array, Dimension, Ix2}; +use serde::{Deserialize, Serialize}; -pub trait Activations { - fn linear() -> Activator - where - Self: Sized; +pub struct Activator +where + D: Dimension, +{ + method: Box>, } -pub struct A(Box>); +impl Activator +where + D: Dimension, +{ + pub fn new(method: Box>) -> Self { + Self { method } + } -pub struct Activator { - method: Box>, -} + pub fn from_method(method: impl Activate + 'static) -> Self { + Self::new(Box::new(method)) + } -impl Activator { - pub fn new(method: Box>) -> Self { - Self { method } + pub fn boxed(&self) -> &Box> { + &self.method } - pub fn method(&self) -> &dyn Activate { + pub fn method(&self) -> &dyn Activate { self.method.as_ref() } } + +impl Activator +where + D: Dimension, + T: Clone, +{ + pub fn linear() -> Self { + Self::new(Box::new(super::Linear::new())) + } +} + +impl Activate for Activator +where + D: Dimension, + dyn Activate: Activate, +{ + fn activate(&self, args: &Array) -> Array { + self.method.activate(args) + } +} + +impl Gradient for Activator +where + D: Dimension, + dyn Activate: Gradient, +{ + fn gradient(&self, args: &Array) -> Array { + self.method.gradient(args) + } +} + +impl Clone for Activator +where + D: Dimension, + T: Clone, + dyn Activate: Clone, +{ + fn clone(&self) -> Self { + Self::new(self.method.clone()) + } +} + +impl Default for Activator +where + D: Dimension, + dyn Activate: Default, +{ + fn default() -> Self { + Self::new(Box::new(>::default())) + } +} + +// impl Copy for A where D: Dimension, T: Copy, dyn Activate: Copy {} + +impl PartialEq for Activator +where + D: Dimension, + dyn Activate: PartialEq, +{ + fn eq(&self, other: &Self) -> bool { + self.method.eq(&other.method) + } +} + +impl Eq for Activator +where + D: Dimension, + dyn Activate: Eq, +{ +} + +impl std::fmt::Debug for Activator +where + D: Dimension, + dyn Activate: std::fmt::Debug, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.method.fmt(f) + } +} + +impl std::fmt::Display for Activator +where + D: Dimension, + dyn Activate: std::fmt::Display, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.method.fmt(f) + } +} + +impl Serialize for Activator +where + D: Dimension, + dyn Activate: Serialize, +{ + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + self.method.serialize(serializer) + } +} + +impl<'de, T, D> Deserialize<'de> for Activator +where + D: Dimension, + dyn Activate: Deserialize<'de>, +{ + fn deserialize(deserializer: D2) -> Result + where + D2: serde::Deserializer<'de>, + { + Ok(Self::new(Box::new(>::deserialize( + deserializer, + )?))) + } +} + +impl From>> for Activator +where + D: Dimension, +{ + fn from(method: Box>) -> Self { + Self::new(method) + } +} diff --git a/ml/neural/src/func/activate/mod.rs b/ml/neural/src/func/activate/mod.rs index 0c7b1d93..9bec62b9 100644 --- a/ml/neural/src/func/activate/mod.rs +++ b/ml/neural/src/func/activate/mod.rs @@ -13,7 +13,7 @@ pub(crate) mod linear; pub(crate) mod nl; // use crate::core::prelude::ShapeResult; -use ndarray::prelude::{Array, ArrayD, Dimension, Ix2, IxDyn}; +use ndarray::prelude::{Array, Dimension, Ix2, IxDyn}; pub type ActivationFn = Box) -> Array>; @@ -32,25 +32,27 @@ where } } -pub trait A +pub trait Activate where - T: Clone, + D: Dimension, { - // fn activate(&self, args: &Array) -> ShapeResult> { - // let shape = args.shape(); - // let res = self.activate_dyn(&args.into_dyn()); - // let res: Array = res.to_shape(shape)?.to_owned(); - // Ok(res) - // } - - fn activate_dyn(&self, args: &ArrayD) -> ArrayD; + fn activate(&self, args: &Array) -> Array; } -pub trait Activate +pub trait ActivateExt: Activate + Clone + 'static where D: Dimension, { - fn activate(&self, args: &Array) -> Array; + fn boxed(&self) -> ActivateDyn { + Box::new(self.clone()) + } +} + +impl ActivateExt for F +where + D: Dimension, + F: Activate + Clone + 'static, +{ } impl Activate for F @@ -72,7 +74,7 @@ where } } -pub trait Objective: Activate +pub trait Gradient where D: Dimension, { @@ -88,16 +90,7 @@ where // } // } -impl Activate for Box> -where - D: Dimension, -{ - fn activate(&self, args: &Array) -> Array { - self.as_ref().activate(args) - } -} - -impl Objective for Box> +impl Gradient for Box> where D: Dimension, { diff --git a/ml/neural/src/func/activate/nl/relu.rs b/ml/neural/src/func/activate/nl/relu.rs index ed09e259..42b6fdc3 100644 --- a/ml/neural/src/func/activate/nl/relu.rs +++ b/ml/neural/src/func/activate/nl/relu.rs @@ -2,7 +2,7 @@ Appellation: relu Contrib: FL03 */ -use crate::func::activate::Objective; +use crate::func::activate::Gradient; use ndarray::prelude::{Array, Dimension}; use num::{One, Zero}; use serde::{Deserialize, Serialize}; @@ -50,7 +50,7 @@ impl ReLU { // } // } -impl Objective for ReLU +impl Gradient for ReLU where D: Dimension, T: Clone + One + PartialOrd + Zero, diff --git a/ml/neural/src/func/activate/nl/sigmoid.rs b/ml/neural/src/func/activate/nl/sigmoid.rs index e294a76f..142547fa 100644 --- a/ml/neural/src/func/activate/nl/sigmoid.rs +++ b/ml/neural/src/func/activate/nl/sigmoid.rs @@ -2,7 +2,7 @@ Appellation: sigmoid Contrib: FL03 */ -use crate::func::activate::Objective; +use crate::func::activate::Gradient; use ndarray::prelude::{Array, Dimension}; use num::Float; use serde::{Deserialize, Serialize}; @@ -42,7 +42,7 @@ impl Sigmoid { // } // } -impl Objective for Sigmoid +impl Gradient for Sigmoid where D: Dimension, T: Float, diff --git a/ml/neural/src/func/activate/nl/softmax.rs b/ml/neural/src/func/activate/nl/softmax.rs index 6acd484b..519ab654 100644 --- a/ml/neural/src/func/activate/nl/softmax.rs +++ b/ml/neural/src/func/activate/nl/softmax.rs @@ -2,7 +2,7 @@ Appellation: softmax Contrib: FL03 */ -use crate::func::activate::Objective; +use crate::func::activate::Gradient; use ndarray::prelude::{Array, Axis, Dimension, NdFloat}; use ndarray::RemoveAxis; use num::Float; @@ -66,7 +66,7 @@ impl Softmax { // } // } -impl Objective for Softmax +impl Gradient for Softmax where D: Dimension + RemoveAxis, T: NdFloat, diff --git a/ml/neural/src/func/activate/nl/tanh.rs b/ml/neural/src/func/activate/nl/tanh.rs index b0865502..2757fc2e 100644 --- a/ml/neural/src/func/activate/nl/tanh.rs +++ b/ml/neural/src/func/activate/nl/tanh.rs @@ -2,7 +2,7 @@ Appellation: tanh Contrib: FL03 */ -use crate::func::activate::Objective; +use crate::func::activate::Gradient; use ndarray::prelude::{Array, Dimension}; use num::Float; use serde::{Deserialize, Serialize}; @@ -42,7 +42,7 @@ impl Tanh { // } // } -impl Objective for Tanh +impl Gradient for Tanh where D: Dimension, T: Float, diff --git a/ml/neural/src/func/loss/mod.rs b/ml/neural/src/func/loss/mod.rs index b6b535cc..4a39aaca 100644 --- a/ml/neural/src/func/loss/mod.rs +++ b/ml/neural/src/func/loss/mod.rs @@ -25,14 +25,11 @@ pub trait Loss { fn loss(&self, pred: &T, target: &T) -> Self::Output; } -pub trait Losses -where - T: Float, -{ - fn loss(&self, pred: &Array, target: &Array) -> Array; -} +pub trait LossPartial { + type Output; -// pub type LinearWeightGradient = fn() + fn partial(&self, pred: &T, target: &T) -> Self::Output; +} pub struct MSE; @@ -97,3 +94,32 @@ pub(crate) mod utils { (target.clone() - pred.clone()).mapv(|x| x.powi(2)).mean() } } + +#[cfg(test)] +mod tests { + use super::regress::*; + use super::*; + use crate::core::GenerateRandom; + use computare::prelude::RoundTo; + use ndarray::prelude::{Array, Ix2}; + use ndarray_stats::DeviationExt; + + #[test] + fn test_mse() { + let (m, n) = (3, 3); + let shape = (m, n); + + let ns = m * n; + + let targets: Array = Array::linspace(0.0, ns as f64, ns) + .into_shape(shape) + .unwrap(); + let pred = Array::::uniform_between(3.0, shape); + + let loss = MeanSquaredError::new().loss(&pred, &targets).round_to(4); + + let exp = targets.mean_sq_err(&pred).unwrap().round_to(4); + + assert_eq!(&loss, &exp); + } +} diff --git a/ml/neural/src/func/loss/regress.rs b/ml/neural/src/func/loss/regress.rs index 61deac0c..491a3f0b 100644 --- a/ml/neural/src/func/loss/regress.rs +++ b/ml/neural/src/func/loss/regress.rs @@ -3,8 +3,10 @@ Contrib: FL03 */ use super::Loss; -use ndarray::prelude::{Array, Dimension, NdFloat}; -use num::Float; +use ndarray::linalg::Dot; +use ndarray::prelude::{Array, Array1, Array2, Dimension, NdFloat}; +use num::{Float, FromPrimitive}; +use serde::{Deserialize, Serialize}; use std::ops; pub enum RegressiveLoss { @@ -14,7 +16,11 @@ pub enum RegressiveLoss { Other(String), } -pub struct HuberLoss { +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct HuberLoss +where + T: Float, +{ delta: T, } @@ -59,6 +65,9 @@ where } } +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] pub struct MeanAbsoluteError; impl Loss> for MeanAbsoluteError @@ -78,8 +87,52 @@ where } } +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] pub struct MeanSquaredError; +impl MeanSquaredError { + pub fn new() -> Self { + Self + } + + pub fn partial_slope( + data: &Array2, + target: &Array1, + bias: &Array1, + weights: &Array2, + ) -> T + where + T: FromPrimitive + NdFloat, + { + let pred = data.dot(&weights.t().to_owned()) + bias.clone(); + let error = target - &pred; + let w = data.t().dot(&error) * (-T::from(2).unwrap()); + w.mean().unwrap() + } + pub fn partial( + data: &Array, + bias: &Array1, + slope: &Array, + target: &Array1, + ) -> (T, T) + where + D: Dimension, + T: FromPrimitive + NdFloat, + Array: Dot>, + as Dot>>::Output: ops::Add, Output = Array> + + ops::Sub, Output = Array> + + ops::Mul>, + Array1: ops::Sub, Output = Array>, + { + let predicted = data.dot(&slope.t().to_owned()) + bias.clone(); + let w = data.dot(&(target.clone() - predicted.clone())) * (-T::from(2).unwrap()); + let b = (target.clone() - predicted) * (-T::from(2).unwrap()); + (w.mean().unwrap(), b.mean().unwrap()) + } +} + impl Loss> for MeanSquaredError where D: Dimension, diff --git a/ml/neural/src/func/mod.rs b/ml/neural/src/func/mod.rs index 7df9245c..9776773b 100644 --- a/ml/neural/src/func/mod.rs +++ b/ml/neural/src/func/mod.rs @@ -22,12 +22,21 @@ pub mod prop; pub(crate) mod block; pub(crate) mod rms; +use ndarray::prelude::{Array, Dimension, Ix2}; + pub trait Lin { type Output; fn linear(&self, args: &T) -> Self::Output; } +pub trait Objective +where + D: Dimension, +{ + fn objective(&self, args: &Array) -> Array; +} + pub(crate) mod utils { use ndarray::linalg::Dot; use ndarray::prelude::{Array, Dimension}; diff --git a/ml/neural/src/func/prop/mod.rs b/ml/neural/src/func/prop/mod.rs index f1b081a6..8b62f037 100644 --- a/ml/neural/src/func/prop/mod.rs +++ b/ml/neural/src/func/prop/mod.rs @@ -18,7 +18,7 @@ pub type ForwardDyn = Box, Output = Ar pub trait Backward: Forward { type Optim; - fn backward(&mut self, data: &T, opt: &Self::Optim); + fn backward(&mut self, args: &T, grad: &T); } pub trait Forward { diff --git a/ml/neural/src/layers/cmp/features.rs b/ml/neural/src/layers/cmp/features.rs index daf249a9..614c1274 100644 --- a/ml/neural/src/layers/cmp/features.rs +++ b/ml/neural/src/layers/cmp/features.rs @@ -2,102 +2,11 @@ Appellation: features Contrib: FL03 */ +use super::Features; use ndarray::prelude::{Dimension, Ix2}; use ndarray::IntoDimension; use serde::{Deserialize, Serialize}; -pub trait Features { - fn inputs(&self) -> usize; - - fn network(&self) -> usize { - self.inputs() * self.outputs() - } - - fn outputs(&self) -> usize; - - fn in_by_out(&self) -> (usize, usize) { - (self.inputs(), self.outputs()) - } - - fn out_by_in(&self) -> (usize, usize) { - (self.outputs(), self.inputs()) - } - - fn input_scale(&self) -> T { - (T::one() / T::from(self.inputs()).unwrap()).sqrt() - } -} - -impl Features for usize { - fn inputs(&self) -> usize { - *self - } - - fn outputs(&self) -> usize { - 1 - } -} - -impl Features for (usize, usize) { - fn inputs(&self) -> usize { - self.1 - } - - fn outputs(&self) -> usize { - self.0 - } -} - -impl Features for [usize; 2] { - fn inputs(&self) -> usize { - self[1] - } - - fn outputs(&self) -> usize { - self[0] - } -} - -pub trait FeaturesExt: Features + IntoDimension -where - D: Dimension, -{ - fn new(inputs: usize, outputs: usize) -> Self; - - fn single(inputs: usize) -> Self - where - Self: Sized, - { - Self::new(inputs, 1) - } -} - -// impl FeaturesExt for T -// where -// T: Features + IntoDimension, -// { -// fn new(inputs: usize, outputs: usize) -> Self { -// Self::from_dimension(ndarray::Ix2(outputs, inputs)) -// } -// } - -pub trait FromFeatures { - fn from_features(features: Sh) -> Self; -} - -pub trait IntoFeatures { - fn into_features(self) -> LayerShape; -} - -impl IntoFeatures for S -where - S: Into, -{ - fn into_features(self) -> LayerShape { - self.into() - } -} - #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] diff --git a/ml/neural/src/layers/cmp/mod.rs b/ml/neural/src/layers/cmp/mod.rs index 5bb0b5dd..47beedbb 100644 --- a/ml/neural/src/layers/cmp/mod.rs +++ b/ml/neural/src/layers/cmp/mod.rs @@ -8,6 +8,99 @@ pub use self::{features::*, kinds::*, utils::*}; pub(crate) mod features; pub(crate) mod kinds; +use ndarray::prelude::{Dimension, Ix2}; +use ndarray::IntoDimension; +use num::Float; + +pub trait Features { + fn inputs(&self) -> usize; + + fn network(&self) -> usize { + self.inputs() * self.outputs() + } + + fn outputs(&self) -> usize; + + fn in_by_out(&self) -> (usize, usize) { + (self.inputs(), self.outputs()) + } + + fn out_by_in(&self) -> (usize, usize) { + (self.outputs(), self.inputs()) + } + + fn input_scale(&self) -> T { + (T::one() / T::from(self.inputs()).unwrap()).sqrt() + } +} + +impl Features for usize { + fn inputs(&self) -> usize { + *self + } + + fn outputs(&self) -> usize { + 1 + } +} + +impl Features for (usize, usize) { + fn inputs(&self) -> usize { + self.1 + } + + fn outputs(&self) -> usize { + self.0 + } +} + +impl Features for [usize; 2] { + fn inputs(&self) -> usize { + self[1] + } + + fn outputs(&self) -> usize { + self[0] + } +} + +pub trait FeaturesExt: Features + IntoDimension +where + D: Dimension, +{ + fn new(inputs: usize, outputs: usize) -> Self; + + fn single(inputs: usize) -> Self + where + Self: Sized, + { + Self::new(inputs, 1) + } +} + +impl FeaturesExt for (usize, usize) { + fn new(inputs: usize, outputs: usize) -> Self { + (outputs, inputs) + } +} + +pub trait FromFeatures { + fn from_features(features: Sh) -> Self; +} + +pub trait IntoFeatures { + fn into_features(self) -> LayerShape; +} + +impl IntoFeatures for S +where + S: Into, +{ + fn into_features(self) -> LayerShape { + self.into() + } +} + pub(crate) mod utils {} #[cfg(test)] diff --git a/ml/neural/src/layers/exp/config.rs b/ml/neural/src/layers/exp/config.rs index 6db6a7ab..01b4c348 100644 --- a/ml/neural/src/layers/exp/config.rs +++ b/ml/neural/src/layers/exp/config.rs @@ -7,7 +7,81 @@ use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct LayerConfig { - pub features: LayerShape, + biased: bool, + features: LayerShape, kind: LayerKind, name: String, } + +impl LayerConfig { + pub fn new(biased: bool, features: LayerShape, kind: LayerKind, name: impl ToString) -> Self { + Self { + biased, + features, + kind, + name: name.to_string(), + } + } + + pub fn is_biased(&self) -> bool { + self.biased + } + + pub fn features(&self) -> LayerShape { + self.features + } + + pub fn kind(&self) -> LayerKind { + self.kind + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn set_biased(&mut self, biased: bool) { + self.biased = biased; + } + + pub fn set_features(&mut self, features: LayerShape) { + self.features = features; + } + + pub fn set_kind(&mut self, kind: LayerKind) { + self.kind = kind; + } + + pub fn set_name(&mut self, name: String) { + self.name = name; + } + + pub fn biased(mut self) -> Self { + self.biased = true; + self + } + + pub fn unbiased(mut self) -> Self { + self.biased = false; + self + } + + pub fn with_bias(mut self, biased: bool) -> Self { + self.biased = biased; + self + } + + pub fn with_features(mut self, features: LayerShape) -> Self { + self.features = features; + self + } + + pub fn with_kind(mut self, kind: LayerKind) -> Self { + self.kind = kind; + self + } + + pub fn with_name(mut self, name: String) -> Self { + self.name = name; + self + } +} diff --git a/ml/neural/src/layers/exp/layer.rs b/ml/neural/src/layers/exp/layer.rs index 0a9a1a13..6fe7a07a 100644 --- a/ml/neural/src/layers/exp/layer.rs +++ b/ml/neural/src/layers/exp/layer.rs @@ -2,7 +2,7 @@ Appellation: model Contrib: FL03 */ -use crate::func::activate::{Activate, Linear}; +use crate::func::activate::{Activate, Activator, Linear}; use crate::layers::{LayerParams, LayerShape}; use crate::prelude::{Features, Forward, Neuron, Node, Parameterized, Params}; use ndarray::prelude::{Array2, Ix1, NdFloat}; @@ -10,39 +10,31 @@ use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct Layer +pub struct Layer where - A: Activate, T: Float, { - activator: A, + activator: Activator, pub features: LayerShape, name: String, params: LayerParams, } -impl Layer +impl Layer where - A: Default + Activate, T: Float, { - pub fn new(features: LayerShape) -> Self { + pub fn new(activator: impl Activate + 'static, features: impl Into) -> Self { + let features = features.into(); Self { - activator: A::default(), + activator: Activator::new(Box::new(activator)), features, name: String::new(), params: LayerParams::new(features), } } -} -impl Layer -where - A: Activate, - T: Float, -{ - pub fn activator(&self) -> &A { + pub fn activator(&self) -> &Activator { &self.activator } @@ -54,11 +46,8 @@ where self.name = name.to_string(); } - pub fn set_node(&mut self, idx: usize, neuron: &Neuron) - where - A: Activate, - { - self.params.set_node(idx, neuron.node().clone()); + pub fn set_node(&mut self, idx: usize, node: &Node) { + self.params.set_node(idx, node.clone()); } pub fn validate_layer(&self, other: &Self, next: bool) -> bool { @@ -74,24 +63,8 @@ where } } -impl Layer +impl Layer where - A: Activate + Clone + 'static, - T: Float, -{ - pub fn as_dyn(&self) -> Layer>> { - Layer { - activator: Box::new(self.activator.clone()), - features: self.features.clone(), - name: self.name.clone(), - params: self.params.clone(), - } - } -} - -impl Layer -where - A: Activate, T: Float + 'static, { pub fn apply_gradient(&mut self, gamma: T, gradient: F) @@ -107,9 +80,8 @@ where } } -impl Layer +impl Layer where - A: Activate, T: NdFloat, { pub fn linear(&self, args: &Array2) -> Array2 { @@ -117,9 +89,8 @@ where } } -impl Layer +impl Layer where - A: Activate, T: Float + SampleUniform, { pub fn init(mut self, biased: bool) -> Self { @@ -128,9 +99,8 @@ where } } -impl Forward> for Layer +impl Forward> for Layer where - A: Activate, T: NdFloat, { type Output = Array2; @@ -140,9 +110,8 @@ where } } -impl Parameterized for Layer +impl Parameterized for Layer where - A: Activate, T: Float, { type Features = LayerShape; @@ -175,19 +144,17 @@ where // } // } -impl From for Layer +impl From for Layer where - A: Activate + Default, - T: Float, + T: Float + 'static, { fn from(features: LayerShape) -> Self { - Self::new(features) + Self::new(Activator::linear(), features) } } -impl IntoIterator for Layer +impl IntoIterator for Layer where - A: Activate + Default, T: Float, { type Item = Node; @@ -198,15 +165,14 @@ where } } -impl FromIterator> for Layer +impl FromIterator> for Layer where - A: Activate + Default, T: Float, { fn from_iter>>(nodes: I) -> Self { let params = LayerParams::from_iter(nodes); Self { - activator: A::default(), + activator: Activator::linear(), features: *params.features(), name: String::new(), params, diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 44b48dff..7ee61309 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -2,12 +2,13 @@ Appellation: model Contrib: FL03 */ -use super::{LayerKind, LayerParams, LayerPosition, LayerShape}; -use crate::func::activate::{Activate, Linear}; +use super::{LayerParams, LayerShape}; +use crate::func::activate::{Activate, Gradient, Linear}; use crate::prelude::{Features, Forward, Neuron, Node, Parameterized, Params}; use ndarray::prelude::{Array2, Ix1, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; -use num::Float; +use ndarray_stats::DeviationExt; +use num::{Float, Signed}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] @@ -20,7 +21,6 @@ where pub features: LayerShape, name: String, params: LayerParams, - position: LayerPosition, } impl Layer @@ -28,27 +28,14 @@ where A: Default + Activate, T: Float, { - pub fn new(features: LayerShape, position: LayerPosition) -> Self { + pub fn new(features: LayerShape) -> Self { Self { activator: A::default(), features, name: String::new(), params: LayerParams::new(features), - position, } } - - pub fn input(features: LayerShape) -> Self { - Self::new(features, LayerPosition::input()) - } - - pub fn hidden(features: LayerShape, index: usize) -> Self { - Self::new(features, LayerPosition::hidden(index)) - } - - pub fn output(features: LayerShape, index: usize) -> Self { - Self::new(features, LayerPosition::output(index)) - } } impl Layer @@ -60,22 +47,10 @@ where &self.activator } - pub fn index(&self) -> usize { - self.position().index() - } - - pub fn kind(&self) -> &LayerKind { - self.position().kind() - } - pub fn name(&self) -> &str { &self.name } - pub fn position(&self) -> &LayerPosition { - &self.position - } - pub fn set_name(&mut self, name: impl ToString) { self.name = name.to_string(); } @@ -87,16 +62,6 @@ where self.params.set_node(idx, neuron.node().clone()); } - pub fn update_position(&mut self, idx: usize, output: bool) { - self.position = if idx == 0 { - LayerPosition::input() - } else if output { - LayerPosition::output(idx) - } else { - LayerPosition::hidden(idx) - }; - } - pub fn validate_layer(&self, other: &Self, next: bool) -> bool { if next { return self.features().inputs() == other.features().outputs(); @@ -121,7 +86,6 @@ where features: self.features.clone(), name: self.name.clone(), params: self.params.clone(), - position: self.position.clone(), } } } @@ -154,6 +118,30 @@ where } } +impl Layer +where + A: Activate + Gradient, + T: NdFloat + Signed, +{ + pub fn grad(&mut self, gamma: T, args: &Array2, targets: &Array2) -> T { + let ns = T::from(args.shape()[0]).unwrap(); + let pred = self.forward(args); + + let scale = T::from(2).unwrap() * ns; + + let errors = &pred - targets; + let dz = errors * self.activator.gradient(&pred); + let dw = args.t().dot(&dz) / scale; + + self.params_mut().weights_mut().scaled_add(-gamma, &dw.t()); + + let loss = targets + .mean_sq_err(&pred) + .expect("Failed to calculate loss"); + T::from(loss).unwrap() + } +} + impl Layer where A: Activate, @@ -216,15 +204,15 @@ where } } -impl PartialOrd for Layer -where - A: Activate + PartialEq, - T: Float, -{ - fn partial_cmp(&self, other: &Self) -> Option { - self.position.partial_cmp(&other.position) - } -} +// impl PartialOrd for Layer +// where +// A: Activate + PartialEq, +// T: Float, +// { +// fn partial_cmp(&self, other: &Self) -> Option { +// self.position.partial_cmp(&other.position) +// } +// } // impl From for Layer // where @@ -243,7 +231,7 @@ where T: Float, { fn from(features: LayerShape) -> Self { - Self::new(features, LayerPosition::input()) + Self::new(features) } } @@ -272,7 +260,6 @@ where features: *params.features(), name: String::new(), params, - position: LayerPosition::input(), } } } diff --git a/ml/neural/src/models/mod.rs b/ml/neural/src/models/mod.rs index 6d5b0416..ad23ace9 100644 --- a/ml/neural/src/models/mod.rs +++ b/ml/neural/src/models/mod.rs @@ -4,16 +4,24 @@ */ //! # Model //! -pub use self::{config::*, model::*, stack::*, utils::*}; +pub use self::{config::*, model::*, params::*, utils::*}; pub(crate) mod config; pub(crate) mod model; -pub(crate) mod stack; +pub(crate) mod params; use crate::prelude::Forward; use ndarray::prelude::Array2; use num::Float; +pub trait Compiler { + type Opt; + + fn compile(&mut self); + + fn optimizer(&self) -> &Self::Opt; +} + pub trait Module: Forward, Output = Array2> where T: Float, diff --git a/ml/neural/src/models/stack.rs b/ml/neural/src/models/params.rs similarity index 100% rename from ml/neural/src/models/stack.rs rename to ml/neural/src/models/params.rs diff --git a/ml/neural/src/nn/ffn/mlp.rs b/ml/neural/src/nn/ffn/mlp.rs index 4a6d0351..ca6f8d07 100644 --- a/ml/neural/src/nn/ffn/mlp.rs +++ b/ml/neural/src/nn/ffn/mlp.rs @@ -66,9 +66,9 @@ where let input_shape = LayerShape::new(inputs, outputs); let shape = LayerShape::new(outputs, outputs); - let input = Layer::input(input_shape); + let input = Layer::from(input_shape); let stack = Stack::square(hidden, outputs, outputs); - let output = Layer::output(shape, hidden + 1); + let output = Layer::from(shape); Self::new(input, stack, output) } @@ -77,9 +77,9 @@ where hidden: impl IntoIterator, outputs: (usize, usize), ) -> Self { - let input = Layer::input(LayerShape::from(inputs)); + let input = Layer::from(LayerShape::from(inputs)); let stack = Stack::new().build_layers(hidden); - let output = Layer::output(outputs.into(), stack.len() + 1); + let output = Layer::from(LayerShape::from(outputs)); Self::new(input, stack, output) } diff --git a/ml/neural/src/nn/ffn/mod.rs b/ml/neural/src/nn/ffn/mod.rs index 7d4d8ae0..f0a6d506 100644 --- a/ml/neural/src/nn/ffn/mod.rs +++ b/ml/neural/src/nn/ffn/mod.rs @@ -9,28 +9,6 @@ pub use self::{mlp::*, model::*, utils::*}; pub(crate) mod mlp; pub(crate) mod model; -use ndarray::prelude::{Array, Array2, Dimension, Ix2}; -use num::Float; - -pub trait Optimizer -where - T: Float, -{ - fn step(&mut self, grad: &Array2) -> Array2; -} - -pub trait FeedForward -where - D: Dimension, - T: Float, -{ - type Opt; - - fn backward(&mut self, args: &Array2, targets: &Array, opt: &Self::Opt) -> Array2; - - fn forward(&self, args: &Array2) -> Array; -} - pub(crate) mod utils {} #[cfg(tets)] diff --git a/ml/neural/src/nn/ffn/model.rs b/ml/neural/src/nn/ffn/model.rs index feaf351e..6daec964 100644 --- a/ml/neural/src/nn/ffn/model.rs +++ b/ml/neural/src/nn/ffn/model.rs @@ -2,14 +2,53 @@ Appellation: model Contrib: FL03 */ -use crate::prelude::Activate; -use ndarray::prelude::Array2; + +use crate::func::activate::Activator; +use crate::prelude::{Features, Forward, Layer, Parameterized}; + +use ndarray::prelude::{Array2, NdFloat}; use num::Float; -pub trait Model +pub struct FFN +where + T: Float, +{ + layers: Vec>>, +} + +impl FFN where - A: Activate, T: Float, { - fn forward(&self, args: &Array2) -> Array2; + pub fn new() -> Self { + Self { layers: Vec::new() } + } + + pub fn add_layer(&mut self, layer: Layer>) { + self.layers.push(layer); + } + + pub fn validate_dims(&self) -> bool { + let depth = self.layers.len(); + let mut dim = true; + for (i, layer) in self.layers[..(depth - 1)].into_iter().enumerate() { + dim = dim && layer.features().outputs() == self.layers[i + 1].features().inputs(); + } + dim + } +} + +impl Forward> for FFN +where + T: NdFloat, +{ + type Output = Array2; + + fn forward(&self, args: &Array2) -> Self::Output { + let mut out = args.clone(); + for layer in self.layers.iter() { + out = layer.forward(&out); + } + out + } } diff --git a/ml/neural/src/nn/kinds.rs b/ml/neural/src/nn/kinds.rs new file mode 100644 index 00000000..600f0c00 --- /dev/null +++ b/ml/neural/src/nn/kinds.rs @@ -0,0 +1,157 @@ +/* + Appellation: kinds + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum NetworkKind { + #[serde(alias = "convolution", alias = "conv", alias = "cnn")] + Convolution = 0, + #[default] + #[serde(alias = "feed_forward", alias = "ffn")] + FeedForward = 1, + #[serde(alias = "graph", alias = "gnn")] + Graph = 2, + #[serde(alias = "recurrent", alias = "rnn")] + Recurrent = 3, +} + +impl NetworkKind { + pub fn cnn() -> Self { + Self::Convolution + } + + pub fn ffn() -> Self { + Self::FeedForward + } + + pub fn gnn() -> Self { + Self::Graph + } + + pub fn rnn() -> Self { + Self::Recurrent + } +} + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum Learning { + Reinforcement = 0, + #[default] + Supervised = 1, + Unsupervised = 2, +} + +impl Learning { + pub fn reinforcement() -> Self { + Self::Reinforcement + } + + pub fn supervised() -> Self { + Self::Supervised + } + + pub fn unsupervised() -> Self { + Self::Unsupervised + } +} + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum NetworkType { + #[serde(alias = "autoencoder", alias = "ae")] + Autoencoder = 0, + #[default] + #[serde(alias = "classifier", alias = "clf")] + Classifier = 1, + #[serde(alias = "regressor", alias = "reg")] + Regressor = 2, +} + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum NetworkStyle { + #[default] + Deep, + Shallow, +} diff --git a/ml/neural/src/nn/mod.rs b/ml/neural/src/nn/mod.rs index c730a11d..64df5a1d 100644 --- a/ml/neural/src/nn/mod.rs +++ b/ml/neural/src/nn/mod.rs @@ -3,85 +3,13 @@ Contrib: FL03 */ //! # Neural Network -pub use self::{position::*, sequential::*, utils::*}; +pub use self::{kinds::*, position::*, sequential::*, utils::*}; +pub(crate) mod kinds; pub(crate) mod position; pub(crate) mod sequential; pub mod ffn; pub mod gnn; -use crate::core::BoxResult; -use crate::layers::Layer; -use crate::Trainable; -use ndarray::prelude::{Array, Axis, Dimension, Ix2}; -use num::Float; - -pub trait NeuralNet: Trainable -where - T: Float, -{ - fn depth(&self) -> usize { - self.layers().len() - } - - fn layers(&self) -> &[Layer]; - - fn input_layer(&self) -> &Layer { - &self.layers()[0] - } - - fn output_layer(&self) -> &Layer { - &self.layers()[self.depth() - 1] - } -} - -pub trait DeepNeuralNet: NeuralNet -where - T: Float, -{ - fn hidden_layers(&self) -> &[Layer]; -} - -pub trait Compile {} - -pub trait Train -where - T: Float, -{ - fn train(&mut self, input: &Array, target: &Array) -> BoxResult; - - fn train_batch( - &mut self, - batch_size: usize, - input: &Array, - target: &Array, - ) -> BoxResult - where - T: std::iter::Sum, - { - let res = input - .axis_chunks_iter(Axis(0), batch_size) - .zip(target.axis_chunks_iter(Axis(0), batch_size)) - .map(|(x, y)| self.train(&x.to_owned(), &y.to_owned()).expect("")) - .sum(); - Ok(res) - } -} - -pub trait Predict -where - D: Dimension, - T: Float, -{ - type Output; - - fn predict(&self, input: &Array) -> BoxResult; - - fn predict_batch(&self, input: &[Array]) -> BoxResult> { - let res = input.iter().map(|x| self.predict(x).expect("")).collect(); - Ok(res) - } -} - pub(crate) mod utils {} diff --git a/ml/neural/src/nn/position.rs b/ml/neural/src/nn/position.rs index 77227851..83d9a3bd 100644 --- a/ml/neural/src/nn/position.rs +++ b/ml/neural/src/nn/position.rs @@ -3,4 +3,8 @@ Contrib: FL03 */ -pub struct Position {} +pub struct Position { + index: usize, +} + +pub struct Scope {} diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index d9643c4e..c654ccc3 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -2,45 +2,56 @@ Appellation: specs Contrib: FL03 */ -use crate::layers::LayerDyn; -use ndarray::prelude::{Array, Array2, Dimension, Ix2}; +use crate::core::BoxResult; +use crate::func::loss::Loss; +use ndarray::prelude::{Array, Axis, Dimension, Ix2}; use num::Float; -pub trait FeedForward +pub trait Compile where D: Dimension, T: Float, { - fn forward(&self, args: &Array2) -> Array; -} + type Opt; -pub trait Initializer -where - D: Dimension, - T: Float, -{ - fn init_weight(&self) -> Array; + fn compile(&mut self, loss: impl Loss>, optimizer: Self::Opt) -> BoxResult<()>; } -pub trait LinearStep +pub trait Predict where D: Dimension, T: Float, { type Output; - fn linear(&self, args: &Array) -> Self::Output; -} + fn predict(&self, input: &Array) -> BoxResult; -pub trait Trainable { - fn train(&mut self, args: &Array2, targets: &Array2) -> Array2; + fn predict_batch(&self, input: &[Array]) -> BoxResult> { + let res = input.iter().map(|x| self.predict(x).expect("")).collect(); + Ok(res) + } } -pub trait NetworkModel: IntoIterator> +pub trait Train where T: Float, { - fn forward(&self, args: &Array2) -> Array2; + fn train(&mut self, input: &Array, target: &Array) -> BoxResult; - fn backward(&mut self, args: &Array2, targets: &Array2) -> Array2; + fn train_batch( + &mut self, + batch_size: usize, + input: &Array, + target: &Array, + ) -> BoxResult + where + T: std::iter::Sum, + { + let res = input + .axis_chunks_iter(Axis(0), batch_size) + .zip(target.axis_chunks_iter(Axis(0), batch_size)) + .map(|(x, y)| self.train(&x.to_owned(), &y.to_owned()).expect("")) + .sum(); + Ok(res) + } } diff --git a/ml/optim/examples/sgd.rs b/ml/optim/examples/sgd.rs index ed9c0aaf..791ef83b 100644 --- a/ml/optim/examples/sgd.rs +++ b/ml/optim/examples/sgd.rs @@ -13,7 +13,7 @@ fn main() -> anyhow::Result<()> { // Generate some example data let (x, y) = sample_data::(inputs, outputs, samples)?; - let mut model = Layer::::input(features); + let mut model = Layer::::from(features); let cost = sgd(&x, &y, &mut model, epochs, gamma, batch_size).unwrap(); println!("Losses {:?}", cost); diff --git a/ml/optim/src/grad/descent.rs b/ml/optim/src/grad/descent.rs index e213582b..a9ff2cdd 100644 --- a/ml/optim/src/grad/descent.rs +++ b/ml/optim/src/grad/descent.rs @@ -2,8 +2,8 @@ Appellation: grad Contrib: FL03 */ -use crate::neural::prelude::{Forward, Layer}; -use ndarray::prelude::{Array1, Array2, NdFloat}; +use crate::neural::prelude::{Forward, Gradient, Layer}; +use ndarray::prelude::{Array2, Ix2, NdFloat}; use ndarray_stats::DeviationExt; use num::{Float, Signed}; @@ -67,7 +67,7 @@ where &mut self, data: &Array2, targets: &Array2, - grad: impl Fn(&Array2) -> Array2, + grad: impl Gradient, ) -> anyhow::Result { let lr = self.gamma(); let ns = T::from(data.shape()[0]).unwrap(); @@ -76,7 +76,7 @@ where let scale = T::from(2).unwrap() * ns; let errors = &pred - targets; - let dz = errors * grad(&pred); + let dz = errors * grad.gradient(&pred); let dw = data.t().dot(&dz) / scale; self.model_mut() @@ -90,7 +90,7 @@ where #[cfg(test)] mod tests { use super::*; - use crate::neural::prelude::{LayerShape, Objective, Sigmoid}; + use crate::neural::prelude::{LayerShape, Sigmoid}; use ndarray::prelude::{Array, Array2}; fn sample_data(inputs: usize, outputs: usize, samples: usize) -> (Array2, Array2) { @@ -118,13 +118,13 @@ mod tests { let mut grad = GradientDescent::new(gamma, model); let l1 = { - let tmp = grad.gradient(&x, &y, |xs| Sigmoid::default().gradient(xs)); + let tmp = grad.gradient(&x, &y, Sigmoid); assert!(tmp.is_ok()); tmp.unwrap() }; let l2 = { - let tmp = grad.gradient(&x, &y, |xs| Sigmoid::default().gradient(xs)); + let tmp = grad.gradient(&x, &y, Sigmoid); assert!(tmp.is_ok()); tmp.unwrap() }; diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs index db7e825a..47d9e021 100644 --- a/ml/optim/src/grad/gradient.rs +++ b/ml/optim/src/grad/gradient.rs @@ -2,23 +2,26 @@ Appellation: grad Contrib: FL03 */ +use crate::neural::func::activate::Sigmoid; use crate::neural::models::ModelParams; -use crate::neural::prelude::{Forward, LayerParams}; +use crate::neural::prelude::{Forward, Gradient, LayerParams}; use ndarray::prelude::{Array2, NdFloat}; use ndarray_stats::DeviationExt; use num::{Float, Signed}; -pub struct Grad +pub struct Grad where + O: Gradient, T: Float, { gamma: T, params: Vec>, - objective: fn(&Array2) -> Array2, + objective: O, } -impl Grad +impl Grad where + O: Gradient, T: Float, { pub fn gamma(&self) -> T { @@ -29,8 +32,8 @@ where &mut self.gamma } - pub fn objective(&self) -> fn(&Array2) -> Array2 { - self.objective + pub fn objective(&self) -> &O { + &self.objective } pub fn model(&self) -> &[LayerParams] { @@ -42,25 +45,27 @@ where } } -impl Grad +impl Grad where + O: Gradient, T: NdFloat + Signed, { pub fn step(&mut self, data: &Array2, targets: &Array2) -> anyhow::Result { - let grad = self.objective(); - let layers = self.model().len(); - let lr = self.gamma(); let ns = T::from(data.shape()[0]).unwrap(); let mut cost = T::zero(); let params = self.params.clone(); for (i, layer) in self.params[1..].iter_mut().enumerate() { + // compute the prediction of the model let pred = params[i - 1].forward(data); + // compute the error of the prediction let errors = &pred - targets; - let dz = errors * grad(&pred); + // compute the gradient of the objective function w.r.t. the error's + let dz = errors * self.objective.gradient(&pred); + // compute the gradient of the objective function w.r.t. the model's weights let dw = data.t().dot(&dz) / ns; - layer.update_with_gradient(lr, &dw.t().to_owned()); + layer.update_with_gradient(self.gamma, &dw.t().to_owned()); let loss = targets.mean_sq_err(&pred)?; cost += T::from(loss).unwrap(); } diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index 8d23b3c0..30f513a8 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -26,6 +26,7 @@ pub struct DescentParams { } pub(crate) mod utils { + use crate::neural::func::activate::Gradient; use crate::neural::prelude::{Forward, Parameterized, Params}; use ndarray::linalg::Dot; use ndarray::prelude::{Array, Array1, Array2, Dimension, NdFloat}; @@ -37,7 +38,7 @@ pub(crate) mod utils { model: &mut A, data: &Array2, targets: &Array, - grad: impl Fn(&Array) -> Array, + grad: impl Gradient, ) -> f64 where A: Forward, Output = Array> + Parameterized, @@ -53,9 +54,10 @@ pub(crate) mod utils { let errors = &pred - targets; // compute the gradient of the objective function w.r.t. the model's weights - let dz = &errors * grad(&pred); + let dz = &errors * grad.gradient(&pred); // compute the gradient of the objective function w.r.t. the model's weights let dw = data.t().to_owned().dot(&dz) / ns; + // let dw = - model.params().bias() * dz + data.t().to_owned().dot(&dz) / ns; // compute the gradient of the objective function w.r.t. the model's bias // let db = dz.sum_axis(Axis(0)) / ns; // // Apply the gradients to the model's learnable parameters @@ -104,11 +106,16 @@ mod tests { use super::*; use crate::core::prelude::linarr; - use crate::neural::func::activate::{Linear, Objective, Sigmoid}; + use crate::neural::func::activate::{Linear, Sigmoid}; use crate::neural::prelude::{Features, Layer, LayerShape, Parameterized, Weighted}; - use ndarray::prelude::{Array1, Array2}; + use ndarray::prelude::{Array, Array1, Dimension}; + use num::Float; - fn test_grad(args: &Array2) -> Array2 { + fn test_grad(args: &Array) -> Array + where + D: Dimension, + T: Float, + { args.clone() } @@ -143,11 +150,11 @@ mod tests { let x = linarr((samples, features.inputs())).unwrap(); let y = linarr((samples, features.outputs())).unwrap(); - let mut model = Layer::::input(features).init(true); + let mut model = Layer::::from(features).init(true); let mut losses = Array1::zeros(epochs); for e in 0..epochs { - let cost = gradient(gamma, &mut model, &x, &y, |w| Sigmoid::new().gradient(w)); + let cost = gradient(gamma, &mut model, &x, &y, Sigmoid); losses[e] = cost; } assert_eq!(losses.len(), epochs); diff --git a/ml/optim/src/grad/sgd.rs b/ml/optim/src/grad/sgd.rs index d3e5e379..5777c4f4 100644 --- a/ml/optim/src/grad/sgd.rs +++ b/ml/optim/src/grad/sgd.rs @@ -231,7 +231,7 @@ mod tests { .into_shape(samples) .unwrap(); - let mut model = Layer::::hidden(features, 5).init(true); + let mut model = Layer::::new(features).init(true); // let mut sgd = StochasticGradientDescent::new(batch_size, epochs, gamma, model); // sgd.sgd(&x, &y); diff --git a/ml/optim/src/lib.rs b/ml/optim/src/lib.rs index 1bef12b9..129805f0 100644 --- a/ml/optim/src/lib.rs +++ b/ml/optim/src/lib.rs @@ -3,8 +3,9 @@ Contrib: FL03 */ //! # Concision Optim -pub use self::{primitives::*, specs::*, utils::*}; +pub use self::{optimizer::*, primitives::*, specs::*, utils::*}; +pub(crate) mod optimizer; pub(crate) mod primitives; pub(crate) mod specs; pub(crate) mod utils; @@ -15,14 +16,15 @@ pub(crate) use concision_neural as neural; pub mod cost; pub mod grad; pub mod norm; -pub mod optimize; +pub mod params; pub mod prelude { pub use crate::cost::*; pub use crate::grad::*; pub use crate::norm::*; - pub use crate::optimize::*; + pub use crate::params::*; + pub use crate::optimizer::*; pub use crate::primitives::*; pub use crate::specs::*; pub use crate::utils::*; diff --git a/ml/optim/src/optimize/mod.rs b/ml/optim/src/optimize/mod.rs deleted file mode 100644 index 01bcc45b..00000000 --- a/ml/optim/src/optimize/mod.rs +++ /dev/null @@ -1,44 +0,0 @@ -/* - Appellation: optimize - Contrib: FL03 -*/ -//! # optimize -//! -pub use self::{optimizer::*, utils::*}; - -pub(crate) mod optimizer; - -use crate::neural::prelude::Forward; -use ndarray::prelude::{Array, Array2, Dimension, Ix2}; -use num::Float; - -pub trait Optimize { - type Model: Forward, Output = Array2>; - - fn name(&self) -> &str; - - // fn optimize(&mut self, model: &mut Self::Model, args: &Array2, targets: &Array2) -> T { - // let gradients = model.backward(args, targets); - // let loss = model.loss(args, targets); - // self.update(model, &gradients); - // loss - // } -} - -pub trait Gradient -where - D: Dimension, - T: Float, -{ - fn update(&mut self, gamma: T, params: &mut Array, gradients: &Array) - where - T: 'static, - { - params.scaled_add(-gamma, gradients); - } -} - -pub(crate) mod utils {} - -#[cfg(test)] -mod tests {} diff --git a/ml/optim/src/optimize/optimizer.rs b/ml/optim/src/optimize/optimizer.rs deleted file mode 100644 index 1f34cfeb..00000000 --- a/ml/optim/src/optimize/optimizer.rs +++ /dev/null @@ -1,67 +0,0 @@ -/* - Appellation: optimizer - Contrib: FL03 -*/ -use crate::neural::prelude::Params; -use ndarray::prelude::Array2; -use num::Float; - -pub trait Optimizer -where - T: Float, -{ - fn name(&self) -> &str; - - fn step( - &mut self, - data: &Array2, - targets: &Array2, - ) -> impl Fn(&mut Box>) -> T; -} - -pub struct OptimizerStep -where - T: Float, -{ - data: Array2, - params: Vec>, - targets: Array2, -} - -impl OptimizerStep -where - T: Float, -{ - pub fn new(data: Array2, targets: Array2) -> Self { - Self { - data, - params: Vec::new(), - targets, - } - } - - pub fn zeros(inputs: usize, outputs: usize, samples: usize) -> Self { - Self { - data: Array2::zeros((samples, inputs)), - params: Vec::new(), - targets: Array2::zeros((samples, outputs)), - } - } - - pub fn params(&self) -> &[Box] { - &self.params - } - - pub fn params_mut(&mut self) -> &mut [Box] { - &mut self.params - } - - pub fn set_params(&mut self, params: Vec>) { - self.params = params; - } - - pub fn with_params(mut self, params: Vec>) -> Self { - self.params = params; - self - } -} diff --git a/ml/optim/src/optimizer.rs b/ml/optim/src/optimizer.rs new file mode 100644 index 00000000..538f027d --- /dev/null +++ b/ml/optim/src/optimizer.rs @@ -0,0 +1,33 @@ +/* + Appellation: optimizer + Contrib: FL03 +*/ +use crate::neural::prelude::Params; +use ndarray::prelude::Array2; +use num::Float; + +pub trait Optimizer +where + T: Float, +{ + type Config; + + fn config(&self) -> &Self::Config; + + fn name(&self) -> &str; + + fn step(&mut self, data: &Array2, targets: &Array2) -> T; + + fn step_with( + &mut self, + data: &Array2, + targets: &Array2, + params: &mut Box>, + ) -> T; +} + +pub trait OptimizerExt: Optimizer +where + T: Float, +{ +} diff --git a/ml/optim/src/params/mod.rs b/ml/optim/src/params/mod.rs new file mode 100644 index 00000000..62d370bd --- /dev/null +++ b/ml/optim/src/params/mod.rs @@ -0,0 +1,50 @@ +/* + Appellation: params + Contrib: FL03 +*/ +//! # Parameters +//! +//! ## Overview +//! +pub use self::utils::*; + +use num::Float; + +pub trait Minimize { + fn minimize(&self, scale: T) -> Self; +} + +pub trait Dampener +where + T: Float, +{ + fn tau(&self) -> T; // Momentum Damper +} + +pub trait Decay +where + T: Float, +{ + fn lambda(&self) -> T; // Decay Rate +} + +pub trait LearningRate +where + T: Float, +{ + fn gamma(&self) -> T; +} + +pub trait Momentum +where + T: Float, +{ + fn mu(&self) -> T; // Momentum Rate + + fn nestrov(&self) -> bool; +} + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests {} diff --git a/ml/optim/src/specs.rs b/ml/optim/src/specs.rs index 8b6d7bd7..74e10d97 100644 --- a/ml/optim/src/specs.rs +++ b/ml/optim/src/specs.rs @@ -2,7 +2,8 @@ Appellation: specs Contrib: FL03 */ -use ndarray::prelude::{Array, Dimension, Ix2}; +use crate::neural::prelude::Forward; +use ndarray::prelude::{Array, Array2, Dimension, Ix2}; use num::Float; pub trait ApplyGradient @@ -13,50 +14,15 @@ where fn apply_gradient(&mut self, gamma: T, gradients: &Array); } -pub trait Gradient -where - T: Float, -{ - fn partial(&self, x: T) -> T; - - fn gradient(&self, args: &Array) -> Array - where - D: Dimension, - { - args.mapv(|xs| self.partial(xs)) - } -} - -pub trait Minimize { - fn minimize(&self, scale: T) -> Self; -} +pub trait Optimize { + type Model: Forward, Output = Array2>; -pub trait Dampener -where - T: Float, -{ - fn tau(&self) -> T; // Momentum Damper -} - -pub trait Decay -where - T: Float, -{ - fn lambda(&self) -> T; // Decay Rate -} - -pub trait LearningRate -where - T: Float, -{ - fn gamma(&self) -> T; -} - -pub trait Momentum -where - T: Float, -{ - fn mu(&self) -> T; // Momentum Rate + fn name(&self) -> &str; - fn nestrov(&self) -> bool; + // fn optimize(&mut self, model: &mut Self::Model, args: &Array2, targets: &Array2) -> T { + // let gradients = model.backward(args, targets); + // let loss = model.loss(args, targets); + // self.update(model, &gradients); + // loss + // } } diff --git a/ml/transformers/src/attention/multi/attention.rs b/ml/transformers/src/attention/multi/attention.rs index 0d2dfa18..cbc94529 100644 --- a/ml/transformers/src/attention/multi/attention.rs +++ b/ml/transformers/src/attention/multi/attention.rs @@ -46,7 +46,7 @@ where let weights = Weight::uniform((model, model)); Self { features, - linear: Layer::input((model, model).into()), + linear: Layer::new((model, model).into()), weights, } } diff --git a/ml/transformers/src/ffn/network.rs b/ml/transformers/src/ffn/network.rs index 05096ff0..1b2cfe75 100644 --- a/ml/transformers/src/ffn/network.rs +++ b/ml/transformers/src/ffn/network.rs @@ -25,8 +25,8 @@ where { pub fn new(model: usize, network: usize) -> Self { Self { - input: Layer::input((model, network).into()), - output: Layer::output((network, model).into(), 1), + input: Layer::new((model, network).into()), + output: Layer::new((network, model).into()), params: FFNParams::new(model, network), } } From b1f126f8fa4c6848367e86fdf0aa5b9c4632307b Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 2 Dec 2023 13:23:43 -0600 Subject: [PATCH 075/118] update Signed-off-by: FL03 --- concision/examples/gradients.rs | 43 ++++++++++++++++++-- core/src/lib.rs | 8 ++++ ml/neural/src/models/model.rs | 28 +++++++++++++ ml/neural/src/models/params.rs | 14 +++++++ ml/optim/src/grad/gradient.rs | 24 +++++++++-- ml/transformers/src/transform/mod.rs | 2 - ml/transformers/src/transform/transformer.rs | 11 ++++- 7 files changed, 119 insertions(+), 11 deletions(-) diff --git a/concision/examples/gradients.rs b/concision/examples/gradients.rs index f1edec27..e54e54a6 100644 --- a/concision/examples/gradients.rs +++ b/concision/examples/gradients.rs @@ -1,8 +1,9 @@ use concision::neural::prelude::{Layer, Sigmoid}; -use concision::optim::grad::gradient; +use concision::neural::models::ModelParams; +use concision::optim::grad::{gradient, Grad}; use concision::prelude::{linarr, Features, Forward, LayerShape}; -use ndarray::prelude::Array1; +use ndarray::prelude::{Array1, Ix2}; fn main() -> anyhow::Result<()> { let (samples, inputs) = (20, 8); @@ -12,7 +13,9 @@ fn main() -> anyhow::Result<()> { let (epochs, gamma) = (1000, 0.005); - sample_gradient(epochs, features, gamma, samples)?; + // sample_gradient(epochs, features, gamma, samples)?; + + sample_model(epochs, features, gamma, samples)?; Ok(()) } @@ -44,3 +47,37 @@ pub fn sample_gradient( println!("Trained:\n\n{:?}", model.forward(&x)); Ok(()) } + +pub fn sample_model( + epochs: usize, + features: LayerShape, + gamma: f64, + samples: usize, +) -> anyhow::Result<()> { + // Generate some example data + let x = linarr::((samples, features.inputs()))?; + let y = linarr::((samples, features.outputs()))?; + + let mut shapes = vec![features,]; + shapes.extend((0..3).map(|_| LayerShape::new(features.outputs(), features.outputs()))); + + let mut model = ModelParams::::from_iter(shapes); + let mut opt = Grad::new(gamma, model.clone(), Sigmoid); + + // println!( + // "Targets (dim):\t{:?}\nPredictions:\n\n{:?}\n", + // &y.shape(), + // model.forward(&x) + // ); + + let mut losses = Array1::zeros(epochs); + for e in 0..epochs { + let cost = opt.step(&x, &y,)?; + // let cost = model.grad(gamma, &x, &y); + losses[e] = cost; + } + model = opt.model().clone(); + // println!("Losses:\n\n{:?}\n", &losses); + // println!("Trained:\n\n{:?}", model.forward(&x)); + Ok(()) +} diff --git a/core/src/lib.rs b/core/src/lib.rs index 5da3975f..acf905e4 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -17,7 +17,15 @@ pub mod states; pub mod step; pub mod vars; +pub trait Transform { + type Output; + + fn transform(&self, args: &T) -> Self::Output; +} + pub mod prelude { + pub use super::Transform; + pub use crate::epochs::*; pub use crate::errors::*; pub use crate::masks::*; diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs index 8e81ff9b..a9f5b4ec 100644 --- a/ml/neural/src/models/model.rs +++ b/ml/neural/src/models/model.rs @@ -44,6 +44,34 @@ where } } +// impl FromIterator for Model +// where +// T: Float, +// { +// fn from_iter(iter: I) -> Self +// where +// I: IntoIterator, +// { +// let params = ModelParam::from_iter(iter); +// Self { + +// } +// } +// } + +// impl FromIterator> for Model +// where +// T: Float, +// { +// fn from_iter(iter: I) -> Self +// where +// I: IntoIterator>, +// { +// Self { +// children: iter.into_iter().collect(), +// } +// } +// } impl IntoIterator for Model where T: Float, diff --git a/ml/neural/src/models/params.rs b/ml/neural/src/models/params.rs index 822cf98b..dd0360f9 100644 --- a/ml/neural/src/models/params.rs +++ b/ml/neural/src/models/params.rs @@ -121,6 +121,20 @@ where } } +impl FromIterator for ModelParams +where + T: Float, +{ + fn from_iter(iter: I) -> Self + where + I: IntoIterator, + { + Self { + children: iter.into_iter().map(LayerParams::new).collect(), + } + } +} + impl FromIterator> for ModelParams where T: Float, diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs index 47d9e021..053810c0 100644 --- a/ml/optim/src/grad/gradient.rs +++ b/ml/optim/src/grad/gradient.rs @@ -4,7 +4,7 @@ */ use crate::neural::func::activate::Sigmoid; use crate::neural::models::ModelParams; -use crate::neural::prelude::{Forward, Gradient, LayerParams}; +use crate::neural::prelude::{Forward, Gradient, LayerParams,}; use ndarray::prelude::{Array2, NdFloat}; use ndarray_stats::DeviationExt; use num::{Float, Signed}; @@ -15,7 +15,7 @@ where T: Float, { gamma: T, - params: Vec>, + params: ModelParams, objective: O, } @@ -24,6 +24,14 @@ where O: Gradient, T: Float, { + pub fn new(gamma: T, params: ModelParams, objective: O) -> Self { + Self { + gamma, + params, + objective, + } + } + pub fn gamma(&self) -> T { self.gamma } @@ -36,11 +44,11 @@ where &self.objective } - pub fn model(&self) -> &[LayerParams] { + pub fn model(&self) -> &ModelParams { &self.params } - pub fn model_mut(&mut self) -> &mut [LayerParams] { + pub fn model_mut(&mut self) -> &mut ModelParams { &mut self.params } } @@ -76,10 +84,18 @@ where #[cfg(test)] mod tests { + use super::*; + use crate::neural::models::ModelParams; #[test] fn test_gradient() { let (samples, inputs) = (20, 5); + let outputs = 4; + let _shape = (samples, inputs); + + let mut model = ModelParams::::new().build_layers([(inputs, outputs,), (outputs, outputs), (outputs, outputs)]).init_layers(false); + + } } diff --git a/ml/transformers/src/transform/mod.rs b/ml/transformers/src/transform/mod.rs index 9c65329c..7510cc93 100644 --- a/ml/transformers/src/transform/mod.rs +++ b/ml/transformers/src/transform/mod.rs @@ -8,8 +8,6 @@ pub use self::{params::*, transformer::*, utils::*}; pub(crate) mod params; pub(crate) mod transformer; -pub trait Transform {} - pub(crate) mod utils {} #[cfg(test)] diff --git a/ml/transformers/src/transform/transformer.rs b/ml/transformers/src/transform/transformer.rs index 41ce0d4e..da445d41 100644 --- a/ml/transformers/src/transform/transformer.rs +++ b/ml/transformers/src/transform/transformer.rs @@ -2,9 +2,16 @@ Appellation: transformer Contrib: FL03 */ -use super::Transform; +use crate::core::Transform; +use ndarray::prelude::{Array2, NdFloat}; #[derive(Clone, Debug, Default)] pub struct Transformer; -impl Transform for Transformer {} +impl Transform> for Transformer where T: NdFloat { + type Output = Array2; + + fn transform(&self, args: &Array2) -> Self::Output { + args.clone() + } +} From 0a869e5a89feca8ed2e36b29231a8c21e782c890 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 2 Dec 2023 13:29:49 -0600 Subject: [PATCH 076/118] update Signed-off-by: FL03 --- concision/examples/gradients.rs | 6 +++--- ml/optim/src/grad/gradient.rs | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/concision/examples/gradients.rs b/concision/examples/gradients.rs index e54e54a6..ab2ae67e 100644 --- a/concision/examples/gradients.rs +++ b/concision/examples/gradients.rs @@ -13,9 +13,9 @@ fn main() -> anyhow::Result<()> { let (epochs, gamma) = (1000, 0.005); - // sample_gradient(epochs, features, gamma, samples)?; + sample_gradient(epochs, features, gamma, samples)?; - sample_model(epochs, features, gamma, samples)?; + // sample_model(epochs, features, gamma, samples)?; Ok(()) } @@ -76,7 +76,7 @@ pub fn sample_model( // let cost = model.grad(gamma, &x, &y); losses[e] = cost; } - model = opt.model().clone(); + // model = opt.model().clone(); // println!("Losses:\n\n{:?}\n", &losses); // println!("Trained:\n\n{:?}", model.forward(&x)); Ok(()) diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs index 053810c0..ba88db36 100644 --- a/ml/optim/src/grad/gradient.rs +++ b/ml/optim/src/grad/gradient.rs @@ -31,7 +31,7 @@ where objective, } } - + pub fn gamma(&self) -> T { self.gamma } @@ -60,13 +60,16 @@ where { pub fn step(&mut self, data: &Array2, targets: &Array2) -> anyhow::Result { let ns = T::from(data.shape()[0]).unwrap(); + let depth = self.model().len(); let mut cost = T::zero(); let params = self.params.clone(); - for (i, layer) in self.params[1..].iter_mut().enumerate() { + + + for (i, layer) in self.params[..(depth - 1)].iter_mut().enumerate() { // compute the prediction of the model - let pred = params[i - 1].forward(data); + let pred = params[i + 1].forward(data); // compute the error of the prediction let errors = &pred - targets; // compute the gradient of the objective function w.r.t. the error's From 23b883ff3b048199867562d18e672b446a4aec1e Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 3 Dec 2023 12:22:43 -0600 Subject: [PATCH 077/118] update Signed-off-by: FL03 --- concision/examples/gradients.rs | 21 ++- core/src/params/group.rs | 173 ++++++++++++++++++ core/src/params/kinds.rs | 50 ++++- core/src/params/mod.rs | 31 ++-- core/src/params/param.rs | 29 ++- data/src/specs.rs | 4 +- ml/neural/src/arch/shallow.rs | 4 +- ml/neural/src/errors/error.rs | 36 +++- ml/neural/src/func/activate/linear.rs | 19 +- ml/neural/src/func/loss/mod.rs | 103 +---------- ml/neural/src/func/loss/reg/huber.rs | 75 ++++++++ ml/neural/src/func/loss/reg/mae.rs | 31 ++++ ml/neural/src/func/loss/reg/mod.rs | 113 ++++++++++++ ml/neural/src/func/loss/reg/mse.rs | 79 ++++++++ ml/neural/src/func/loss/regress.rs | 151 --------------- ml/neural/src/func/prop/mod.rs | 23 +-- ml/neural/src/func/prop/propagation.rs | 16 -- ml/neural/src/layers/cmp/features.rs | 2 +- ml/neural/src/layers/exp/config.rs | 26 ++- ml/neural/src/layers/exp/layer.rs | 35 ++-- ml/neural/src/layers/layer.rs | 23 ++- ml/neural/src/layers/mod.rs | 10 +- ml/neural/src/layers/params.rs | 27 +-- ml/neural/src/models/config.rs | 4 + ml/neural/src/models/exp/mod.rs | 13 ++ ml/neural/src/models/exp/modules.rs | 28 +++ ml/neural/src/models/mod.rs | 32 +--- ml/neural/src/models/params.rs | 50 ++++- ml/neural/src/neurons/neuron.rs | 19 +- ml/neural/src/neurons/node.rs | 20 +- ml/neural/src/nn/gnn/mod.rs | 3 +- ml/neural/src/nn/gnn/model.rs | 19 ++ ml/neural/src/nn/gnn/tasks.rs | 18 ++ ml/neural/src/primitives.rs | 6 +- ml/neural/src/specs.rs | 14 +- ml/optim/src/grad/gradient.rs | 16 +- ml/optim/src/grad/mod.rs | 3 +- ml/optim/src/grad/modes.rs | 34 ++++ ml/optim/src/grad/sgd.rs | 19 +- ml/optim/src/optimizer.rs | 5 + ml/optim/src/specs.rs | 17 +- .../src/attention/multi/attention.rs | 2 +- ml/transformers/src/ffn/network.rs | 4 +- ml/transformers/src/transform/transformer.rs | 13 +- 44 files changed, 941 insertions(+), 479 deletions(-) create mode 100644 core/src/params/group.rs create mode 100644 ml/neural/src/func/loss/reg/huber.rs create mode 100644 ml/neural/src/func/loss/reg/mae.rs create mode 100644 ml/neural/src/func/loss/reg/mod.rs create mode 100644 ml/neural/src/func/loss/reg/mse.rs delete mode 100644 ml/neural/src/func/loss/regress.rs delete mode 100644 ml/neural/src/func/prop/propagation.rs create mode 100644 ml/neural/src/models/exp/mod.rs create mode 100644 ml/neural/src/models/exp/modules.rs create mode 100644 ml/neural/src/nn/gnn/tasks.rs create mode 100644 ml/optim/src/grad/modes.rs diff --git a/concision/examples/gradients.rs b/concision/examples/gradients.rs index ab2ae67e..6e85f34c 100644 --- a/concision/examples/gradients.rs +++ b/concision/examples/gradients.rs @@ -1,6 +1,6 @@ -use concision::neural::prelude::{Layer, Sigmoid}; use concision::neural::models::ModelParams; -use concision::optim::grad::{gradient, Grad}; +use concision::neural::prelude::{Layer, Sigmoid}; +use concision::optim::grad::*; use concision::prelude::{linarr, Features, Forward, LayerShape}; use ndarray::prelude::{Array1, Ix2}; @@ -27,10 +27,11 @@ pub fn sample_gradient( samples: usize, ) -> anyhow::Result<()> { // Generate some example data - let x = linarr((samples, features.inputs()))?; - let y = linarr((samples, features.outputs()))?; + let x = linarr::((samples, features.inputs()))?; + let mut y = linarr::((samples, features.outputs()))?; + y.map_inplace(|ys| *ys = ys.powi(2)); - let mut model = Layer::::from(features).init(false); + let mut model = Layer::::from(features).init(false); println!( "Targets (dim):\t{:?}\nPredictions:\n\n{:?}\n", &y.shape(), @@ -39,8 +40,8 @@ pub fn sample_gradient( let mut losses = Array1::zeros(epochs); for e in 0..epochs { - let cost = gradient(gamma, &mut model, &x, &y, Sigmoid); - // let cost = model.grad(gamma, &x, &y); + // let cost = gradient(gamma, &mut model, &x, &y, Sigmoid); + let cost = model.grad(gamma, &x, &y); losses[e] = cost; } println!("Losses:\n\n{:?}\n", &losses); @@ -58,10 +59,10 @@ pub fn sample_model( let x = linarr::((samples, features.inputs()))?; let y = linarr::((samples, features.outputs()))?; - let mut shapes = vec![features,]; + let mut shapes = vec![features]; shapes.extend((0..3).map(|_| LayerShape::new(features.outputs(), features.outputs()))); - let mut model = ModelParams::::from_iter(shapes); + let model = ModelParams::::from_iter(shapes); let mut opt = Grad::new(gamma, model.clone(), Sigmoid); // println!( @@ -72,7 +73,7 @@ pub fn sample_model( let mut losses = Array1::zeros(epochs); for e in 0..epochs { - let cost = opt.step(&x, &y,)?; + let cost = opt.step(&x, &y)?; // let cost = model.grad(gamma, &x, &y); losses[e] = cost; } diff --git a/core/src/params/group.rs b/core/src/params/group.rs new file mode 100644 index 00000000..19f678ff --- /dev/null +++ b/core/src/params/group.rs @@ -0,0 +1,173 @@ +/* + Appellation: group + Contrib: FL03 +*/ +use super::{Biased, Weighted}; +use crate::prelude::GenerateRandom; +use ndarray::linalg::Dot; +use ndarray::prelude::{Array, Axis, Dimension, Ix2, NdFloat}; +use ndarray::{IntoDimension, RemoveAxis}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use num::Float; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ParamGroup +where + T: Float, + D: Dimension, +{ + bias: Array, + features: D, + weights: Array, +} + +impl ParamGroup +where + T: Float, + D: Dimension + RemoveAxis, +{ + pub fn new(dim: impl IntoDimension) -> Self { + let dim = dim.into_dimension(); + Self { + bias: Array::zeros(dim.remove_axis(Axis(dim.ndim() - 1))), + features: dim.clone(), + weights: Array::zeros(dim), + } + } +} + +impl ParamGroup +where + T: Float, + D: Dimension, +{ + pub fn features(&self) -> &D { + &self.features + } + + pub fn inputs(&self) -> usize { + self.weights.shape().last().unwrap().clone() + } + + pub fn outputs(&self) -> usize { + if self.features.ndim() == 1 { + return 1; + } + self.weights.shape().first().unwrap().clone() + } +} + +impl ParamGroup +where + D: Dimension, + T: NdFloat, + Self: Biased + Weighted, +{ + pub fn linear(&self, data: &Array) -> Array + where + Array: Dot, Output = Array> + + std::ops::Add, Output = Array>, + { + data.dot(&self.weights().t().to_owned()) + self.bias().clone() + } +} + +impl ParamGroup +where + D: Dimension + RemoveAxis, + T: Float + SampleUniform, +{ + pub fn init(mut self, biased: bool) -> Self { + if biased { + self = self.init_bias(); + } + self.init_weight() + } + + pub fn init_bias(mut self) -> Self { + let dk = (T::one() / T::from(self.inputs()).unwrap()).sqrt(); + self.bias = Array::uniform_between( + dk, + self.features() + .remove_axis(Axis(self.features().ndim() - 1)) + .clone(), + ); + self + } + + pub fn init_weight(mut self) -> Self { + let dk = (T::one() / T::from(self.inputs()).unwrap()).sqrt(); + self.weights = Array::uniform_between(dk, self.features().clone()); + self + } +} + +impl Biased for ParamGroup +where + D: Dimension + RemoveAxis, + T: Float, +{ + fn bias(&self) -> &Array { + &self.bias + } + + fn bias_mut(&mut self) -> &mut Array { + &mut self.bias + } + + fn set_bias(&mut self, bias: Array) { + self.bias = bias; + } +} + +impl Weighted for ParamGroup +where + D: Dimension, + T: Float, +{ + fn weights(&self) -> &Array { + &self.weights + } + + fn weights_mut(&mut self) -> &mut Array { + &mut self.weights + } + + fn set_weights(&mut self, weights: Array) { + self.weights = weights; + } +} + +impl<'a, T, D> Deserialize<'a> for ParamGroup +where + T: Deserialize<'a> + Float, + D: Deserialize<'a> + Dimension, + ::Smaller: Deserialize<'a> + Dimension, +{ + fn deserialize(deserializer: Der) -> Result + where + Der: serde::Deserializer<'a>, + { + let (bias, features, weights) = Deserialize::deserialize(deserializer)?; + Ok(Self { + bias, + features, + weights, + }) + } +} + +impl Serialize for ParamGroup +where + T: Float + Serialize, + D: Dimension + RemoveAxis + Serialize, + ::Smaller: Dimension + Serialize, +{ + fn serialize(&self, serializer: Ser) -> Result + where + Ser: serde::Serializer, + { + (self.bias(), self.features(), self.weights()).serialize(serializer) + } +} diff --git a/core/src/params/kinds.rs b/core/src/params/kinds.rs index 6e1bf00a..0920bf0d 100644 --- a/core/src/params/kinds.rs +++ b/core/src/params/kinds.rs @@ -3,14 +3,13 @@ Contrib: FL03 */ use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; +use strum::{EnumIs, EnumIter, EnumString, EnumVariantNames}; #[derive( Clone, Debug, Default, Deserialize, - Display, EnumIs, EnumIter, EnumString, @@ -32,3 +31,50 @@ pub enum ParamKind { Weight, Other(String), } + +impl ParamKind { + pub fn bias() -> Self { + Self::Bias + } + + pub fn weight() -> Self { + Self::Weight + } + + pub fn other(name: impl ToString) -> Self { + Self::Other(name.to_string()) + } +} + +impl std::fmt::Display for ParamKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let content = match self { + ParamKind::Bias => "bias", + ParamKind::Weight => "weight", + ParamKind::Other(name) => name, + }; + write!(f, "{}", content) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + #[test] + fn test_param_kind_map() { + let name = "test"; + let other = ParamKind::other(name); + + let data = [ + (ParamKind::Bias, 0), + (ParamKind::Weight, 1), + (other.clone(), 2), + (ParamKind::other("mask"), 3), + ]; + let store = HashMap::::from_iter(data); + assert_eq!(store.get(&ParamKind::Bias), Some(&0)); + assert_eq!(store.get(&other), Some(&2)); + } +} diff --git a/core/src/params/mod.rs b/core/src/params/mod.rs index 5816ee68..6ccbc699 100644 --- a/core/src/params/mod.rs +++ b/core/src/params/mod.rs @@ -6,25 +6,19 @@ //! //! ## Overview //! -pub use self::{kinds::*, param::*, utils::*}; +pub use self::{group::*, kinds::*, param::*, utils::*}; +pub(crate) mod group; pub(crate) mod kinds; pub(crate) mod param; use ndarray::prelude::{Array, Dimension, Ix2}; use num::Float; -pub trait Parameter -where - D: Dimension, - T: Float, -{ - /// Returns an owned reference of the param - fn param(&self) -> &Array; - /// Returns a mutable reference of the param - fn param_mut(&mut self) -> &mut Array; - /// Sets the param - fn set_param(&mut self, param: Array); +pub trait Param { + fn kind(&self) -> ParamKind; + + fn name(&self) -> &str; } pub trait Biased @@ -53,6 +47,19 @@ where fn set_weights(&mut self, weights: Array); } +pub trait Params +where + D: Dimension, + T: Float, +{ + /// Returns an owned reference to the parameters of the layer. + fn params(&self) -> &Array; + /// Returns a mutable reference to the parameters of the layer. + fn params_mut(&mut self) -> &mut Array; + /// Sets the parameters of the layer. + fn set_params(&mut self, params: Array); +} + pub(crate) mod utils {} #[cfg(test)] diff --git a/core/src/params/param.rs b/core/src/params/param.rs index c42f04e6..a83755a0 100644 --- a/core/src/params/param.rs +++ b/core/src/params/param.rs @@ -3,31 +3,41 @@ Contrib: FL03 */ use super::ParamKind; +use crate::prelude::GenerateRandom; use ndarray::prelude::{Array, Dimension, Ix2}; +use ndarray::IntoDimension; +use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub struct Param +pub struct Parameter where T: Float, D: Dimension, { + features: D, kind: ParamKind, name: String, params: Array, } -impl Param +impl Parameter where T: Float, D: Dimension, { - pub fn new(kind: ParamKind, name: impl ToString, params: Array) -> Self { + pub fn new( + features: impl IntoDimension, + kind: ParamKind, + name: impl ToString, + ) -> Self { + let features = features.into_dimension(); Self { + features: features.clone(), kind, name: name.to_string(), - params, + params: Array::zeros(features), } } @@ -82,3 +92,14 @@ where self } } + +impl Parameter +where + D: Dimension, + T: Float + SampleUniform, +{ + pub fn init_uniform(mut self, dk: T) -> Self { + self.params = Array::uniform_between(dk, self.clone().features); + self + } +} diff --git a/data/src/specs.rs b/data/src/specs.rs index 8b000db7..be8f7cd5 100644 --- a/data/src/specs.rs +++ b/data/src/specs.rs @@ -3,13 +3,13 @@ Contrib: FL03 */ -pub trait Records { +pub trait Records { fn features(&self) -> usize; fn samples(&self) -> usize; } -impl Records for S +impl Records for S where S: AsRef<(usize, usize)>, { diff --git a/ml/neural/src/arch/shallow.rs b/ml/neural/src/arch/shallow.rs index 1a2dc023..8a16ab5c 100644 --- a/ml/neural/src/arch/shallow.rs +++ b/ml/neural/src/arch/shallow.rs @@ -65,8 +65,8 @@ where let s2 = LayerShape::new(outputs, outputs); let input = Layer::::from(s1); - let hidden = Layer::::new(s2); - let output = Layer::::new(s2); + let hidden = Layer::::from(s2); + let output = Layer::::from(s2); Self::new(input, hidden, output) } diff --git a/ml/neural/src/errors/error.rs b/ml/neural/src/errors/error.rs index e155ea8c..e5e7ce73 100644 --- a/ml/neural/src/errors/error.rs +++ b/ml/neural/src/errors/error.rs @@ -26,11 +26,12 @@ use strum::{Display, EnumIs, EnumIter, EnumVariantNames}; #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] pub enum MlError { - Data, - Dimension, + Data(String), + Dimension(String), #[default] Error(String), Network(NetworkError), + Process(ProcessError), } impl std::error::Error for MlError {} @@ -78,9 +79,34 @@ impl From for MlError { #[non_exhaustive] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] -pub enum ComputeError { +pub enum PredictError { #[default] - Compute(String), + Other(String), +} + +#[derive( + Clone, + Debug, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, + SmartDefault, +)] +#[non_exhaustive] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum ProcessError { + Arithmetic(String), + #[default] + Process(String), ShapeError(String), } @@ -104,7 +130,7 @@ pub enum ComputeError { #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] pub enum NetworkError { - Layer, + Layer(String), #[default] Network(String), } diff --git a/ml/neural/src/func/activate/linear.rs b/ml/neural/src/func/activate/linear.rs index 909f3e5a..3836e9f6 100644 --- a/ml/neural/src/func/activate/linear.rs +++ b/ml/neural/src/func/activate/linear.rs @@ -2,6 +2,7 @@ Appellation: linear Contrib: FL03 */ +use super::Gradient; use ndarray::prelude::{Array, Dimension}; use num::One; use serde::{Deserialize, Serialize}; @@ -44,15 +45,15 @@ impl Linear { } } -// impl Activate for Linear -// where -// D: Dimension, -// T: Clone, -// { -// fn activate(&self, args: &Array) -> Array { -// args.clone() -// } -// } +impl Gradient for Linear +where + D: Dimension, + T: Clone + One, +{ + fn gradient(&self, args: &Array) -> Array { + args.mapv(|x| Self::derivative(x)) + } +} impl Fn<(&T,)> for Linear where diff --git a/ml/neural/src/func/loss/mod.rs b/ml/neural/src/func/loss/mod.rs index 4a39aaca..b0883478 100644 --- a/ml/neural/src/func/loss/mod.rs +++ b/ml/neural/src/func/loss/mod.rs @@ -12,12 +12,7 @@ pub use self::{kinds::*, utils::*}; pub(crate) mod kinds; -pub mod regress; - -use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Array1, Array2, Dimension, NdFloat}; -use num::{Float, FromPrimitive}; -use std::ops; +pub mod reg; pub trait Loss { type Output; @@ -25,101 +20,13 @@ pub trait Loss { fn loss(&self, pred: &T, target: &T) -> Self::Output; } -pub trait LossPartial { +pub trait LossWith { type Output; - fn partial(&self, pred: &T, target: &T) -> Self::Output; -} - -pub struct MSE; - -impl MSE { - pub fn partial_slope( - data: &Array2, - target: &Array1, - bias: &Array1, - weights: &Array2, - ) -> T - where - T: FromPrimitive + NdFloat, - { - let pred = data.dot(&weights.t().to_owned()) + bias.clone(); - let error = target - &pred; - let w = data.t().dot(&error) * (-T::from(2).unwrap()); - w.mean().unwrap() - } - pub fn partial( - data: &Array, - bias: &Array1, - slope: &Array, - target: &Array1, - ) -> (T, T) - where - D: Dimension, - T: FromPrimitive + NdFloat, - Array: Dot>, - as Dot>>::Output: ops::Add, Output = Array> - + ops::Sub, Output = Array> - + ops::Mul>, - Array1: ops::Sub, Output = Array>, - { - let predicted = data.dot(&slope.t().to_owned()) + bias.clone(); - let w = data.dot(&(target.clone() - predicted.clone())) * (-T::from(2).unwrap()); - let b = (target.clone() - predicted) * (-T::from(2).unwrap()); - (w.mean().unwrap(), b.mean().unwrap()) - } + fn loss(&self, other: &Self) -> Self::Output; } -pub(crate) mod utils { - use ndarray::prelude::{Array, Array1}; - use ndarray::{Dimension, ScalarOperand}; - use num::{Float, FromPrimitive}; - use std::ops; - - pub fn mae<'a, T, D>(pred: &Array, target: &Array1) -> Option - where - T: Float + FromPrimitive + ScalarOperand, - D: Dimension, - Array1: ops::Sub, Output = Array>, - { - (target.clone() - pred.clone()).mapv(|x| x.abs()).mean() - } - - pub fn mse(pred: &Array, target: &Array1) -> Option - where - T: Float + FromPrimitive + ScalarOperand, - D: Dimension, - Array1: ops::Sub, Output = Array>, - { - (target.clone() - pred.clone()).mapv(|x| x.powi(2)).mean() - } -} +pub(crate) mod utils {} #[cfg(test)] -mod tests { - use super::regress::*; - use super::*; - use crate::core::GenerateRandom; - use computare::prelude::RoundTo; - use ndarray::prelude::{Array, Ix2}; - use ndarray_stats::DeviationExt; - - #[test] - fn test_mse() { - let (m, n) = (3, 3); - let shape = (m, n); - - let ns = m * n; - - let targets: Array = Array::linspace(0.0, ns as f64, ns) - .into_shape(shape) - .unwrap(); - let pred = Array::::uniform_between(3.0, shape); - - let loss = MeanSquaredError::new().loss(&pred, &targets).round_to(4); - - let exp = targets.mean_sq_err(&pred).unwrap().round_to(4); - - assert_eq!(&loss, &exp); - } -} +mod tests {} diff --git a/ml/neural/src/func/loss/reg/huber.rs b/ml/neural/src/func/loss/reg/huber.rs new file mode 100644 index 00000000..a2abf340 --- /dev/null +++ b/ml/neural/src/func/loss/reg/huber.rs @@ -0,0 +1,75 @@ +/* + Appellation: regress + Contrib: FL03 +*/ +use crate::func::loss::Loss; +use ndarray::prelude::{Array, Dimension, NdFloat}; +use num::Float; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct HuberLoss +where + T: Float, +{ + delta: T, +} + +impl HuberLoss +where + T: Float, +{ + pub fn new(delta: T) -> Self { + Self { delta } + } + + pub fn delta(&self) -> T { + self.delta + } + + pub fn delta_mut(&mut self) -> &mut T { + &mut self.delta + } + + pub fn set_delta(&mut self, delta: T) { + self.delta = delta; + } + + pub fn with_delta(mut self, delta: T) -> Self { + self.delta = delta; + self + } +} + +impl Loss> for HuberLoss +where + D: Dimension, + T: NdFloat, +{ + type Output = T; + + fn loss(&self, pred: &Array, target: &Array) -> Self::Output { + let half = T::from(0.5).unwrap(); + let mut loss = T::zero(); + for (x, y) in pred.iter().cloned().zip(target.iter().cloned()) { + let diff = x - y; + if diff.abs() <= self.delta() { + // If the difference is sufficiently small, use the squared error. + loss += half * diff.powi(2); + } else { + // Otherwise, use a variant of the absolute error. + loss += self.delta * (diff.abs() - half * self.delta()); + } + } + loss / T::from(pred.len()).unwrap() + } +} + +impl From for HuberLoss +where + T: Float, +{ + fn from(delta: T) -> Self { + Self::new(delta) + } +} diff --git a/ml/neural/src/func/loss/reg/mae.rs b/ml/neural/src/func/loss/reg/mae.rs new file mode 100644 index 00000000..4d34c182 --- /dev/null +++ b/ml/neural/src/func/loss/reg/mae.rs @@ -0,0 +1,31 @@ +/* + Appellation: mae + Contrib: FL03 +*/ +use crate::func::loss::Loss; +use ndarray::prelude::{Array, Dimension, NdFloat}; +use num::FromPrimitive; +use serde::{Deserialize, Serialize}; + +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] +pub struct MeanAbsoluteError; + +impl MeanAbsoluteError { + pub fn new() -> Self { + Self + } +} + +impl Loss> for MeanAbsoluteError +where + D: Dimension, + T: FromPrimitive + NdFloat, +{ + type Output = T; + + fn loss(&self, pred: &Array, target: &Array) -> Self::Output { + (pred - target).mapv(T::abs).mean().unwrap() + } +} diff --git a/ml/neural/src/func/loss/reg/mod.rs b/ml/neural/src/func/loss/reg/mod.rs new file mode 100644 index 00000000..5049a0d0 --- /dev/null +++ b/ml/neural/src/func/loss/reg/mod.rs @@ -0,0 +1,113 @@ +/* + Appellation: reg + Contrib: FL03 +*/ +pub use self::{huber::*, mae::*, mse::*, utils::*}; + +pub(crate) mod huber; +pub(crate) mod mae; +pub(crate) mod mse; + +use crate::func::loss::Loss; +use ndarray::prelude::{Array, Dimension, NdFloat}; +use num::FromPrimitive; + +pub enum RegressiveLoss { + Huber(HuberLoss), + MeanAbsoluteError, + MeanSquaredError, + Other(String), +} + +pub trait RegressiveLosses { + fn huber(&self, delta: T, other: &Self) -> T; + fn mae(&self, other: &Self) -> T; + fn mse(&self, other: &Self) -> T; +} + +impl RegressiveLosses for Array +where + D: Dimension, + T: FromPrimitive + NdFloat, +{ + fn huber(&self, delta: T, other: &Self) -> T { + HuberLoss::new(delta).loss(self, other) + } + + fn mae(&self, other: &Self) -> T { + MeanAbsoluteError::new().loss(self, other) + } + + fn mse(&self, other: &Self) -> T { + MeanSquaredError::new().loss(self, other) + } +} + +pub(crate) mod utils { + use ndarray::prelude::{Array, Dimension, NdFloat}; + use num::FromPrimitive; + + pub fn mae(pred: &Array, target: &Array) -> Option + where + D: Dimension, + T: FromPrimitive + NdFloat, + { + (pred - target).mapv(T::abs).mean() + } + + pub fn mse(pred: &Array, target: &Array) -> Option + where + D: Dimension, + T: FromPrimitive + NdFloat, + { + (pred - target).mapv(|x| x.powi(2)).mean() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::GenerateRandom; + use crate::func::loss::Loss; + use computare::prelude::RoundTo; + use ndarray::prelude::{Array, Ix2}; + use ndarray_stats::DeviationExt; + + #[test] + fn test_mae() { + let (m, n) = (3, 3); + let shape = (m, n); + + let ns = m * n; + + let targets: Array = Array::linspace(0.0, ns as f64, ns) + .into_shape(shape) + .unwrap(); + let pred = Array::::uniform_between(3.0, shape); + + let loss = MeanAbsoluteError::new().loss(&pred, &targets).round_to(4); + + let exp = targets.mean_abs_err(&pred).unwrap().round_to(4); + + assert_eq!(&loss, &exp); + } + + #[test] + fn test_mse() { + let (m, n) = (3, 3); + let shape = (m, n); + + let ns = m * n; + + let targets: Array = Array::linspace(0.0, ns as f64, ns) + .into_shape(shape) + .unwrap(); + let pred = Array::::uniform_between(3.0, shape); + + let loss = MeanSquaredError::new().loss(&pred, &targets).round_to(4); + + let exp = targets.mean_sq_err(&pred).unwrap().round_to(4); + + assert_eq!(&loss, &exp); + } +} diff --git a/ml/neural/src/func/loss/reg/mse.rs b/ml/neural/src/func/loss/reg/mse.rs new file mode 100644 index 00000000..95007d8d --- /dev/null +++ b/ml/neural/src/func/loss/reg/mse.rs @@ -0,0 +1,79 @@ +/* + Appellation: mse + Contrib: FL03 +*/ +use crate::func::loss::Loss; +use ndarray::linalg::Dot; +use ndarray::prelude::{Array, Array1, Array2, Dimension, NdFloat}; +use num::FromPrimitive; +use serde::{Deserialize, Serialize}; +use std::ops; + +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] +pub struct MeanSquaredError; + +impl MeanSquaredError { + pub fn new() -> Self { + Self + } + + pub fn mse(&self, pred: &Array, target: &Array) -> T + where + D: Dimension, + T: FromPrimitive + NdFloat, + { + (pred - target).mapv(|x| x.powi(2)).mean().unwrap() + } + + pub fn wg( + data: &Array2, + target: &Array, + bias: &Array, + weights: &Array, + ) -> T + where + D: Dimension, + T: FromPrimitive + NdFloat, + Array2: Dot, Output = Array>, + { + let pred = data.dot(&weights.t().to_owned()) + bias.clone(); + let error = target - &pred; + let dw = data.t().to_owned().dot(&error) * (-T::from(2).unwrap()); + dw.mean().unwrap() + } + + pub fn partial( + data: &Array, + bias: &Array1, + slope: &Array, + target: &Array1, + ) -> (T, T) + where + D: Dimension, + T: FromPrimitive + NdFloat, + Array: Dot>, + as Dot>>::Output: ops::Add, Output = Array> + + ops::Sub, Output = Array> + + ops::Mul>, + Array1: ops::Sub, Output = Array>, + { + let predicted = data.dot(&slope.t().to_owned()) + bias.clone(); + let w = data.dot(&(target.clone() - predicted.clone())) * (-T::from(2).unwrap()); + let b = (target.clone() - predicted) * (-T::from(2).unwrap()); + (w.mean().unwrap(), b.mean().unwrap()) + } +} + +impl Loss> for MeanSquaredError +where + D: Dimension, + T: FromPrimitive + NdFloat, +{ + type Output = T; + + fn loss(&self, pred: &Array, target: &Array) -> Self::Output { + (pred - target).mapv(|x| x.powi(2)).mean().unwrap() + } +} diff --git a/ml/neural/src/func/loss/regress.rs b/ml/neural/src/func/loss/regress.rs deleted file mode 100644 index 491a3f0b..00000000 --- a/ml/neural/src/func/loss/regress.rs +++ /dev/null @@ -1,151 +0,0 @@ -/* - Appellation: regress - Contrib: FL03 -*/ -use super::Loss; -use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Array1, Array2, Dimension, NdFloat}; -use num::{Float, FromPrimitive}; -use serde::{Deserialize, Serialize}; -use std::ops; - -pub enum RegressiveLoss { - Huber(HuberLoss), - MeanAbsoluteError, - MeanSquaredError, - Other(String), -} - -#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] -pub struct HuberLoss -where - T: Float, -{ - delta: T, -} - -impl HuberLoss -where - T: Float, -{ - pub fn new(delta: T) -> Self { - Self { delta } - } - - pub fn delta(&self) -> T { - self.delta - } - - pub fn set_delta(&mut self, delta: T) { - self.delta = delta; - } -} - -impl Loss> for HuberLoss -where - D: Dimension, - T: NdFloat, -{ - type Output = T; - - fn loss(&self, pred: &Array, target: &Array) -> Self::Output { - let half = T::from(0.5).unwrap(); - let mut loss = T::zero(); - for (x, y) in pred.iter().cloned().zip(target.iter().cloned()) { - let diff = x - y; - if diff.abs() <= self.delta() { - // If the difference is sufficiently small, use the squared error. - loss += half * diff.powi(2); - } else { - // Otherwise, use a variant of the absolute error. - loss += self.delta * (diff.abs() - half * self.delta); - } - } - loss / T::from(pred.len()).unwrap() - } -} - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct MeanAbsoluteError; - -impl Loss> for MeanAbsoluteError -where - D: Dimension, - T: Float + ops::AddAssign + ops::DivAssign, -{ - type Output = T; - - fn loss(&self, pred: &Array, target: &Array) -> Self::Output { - let mut res = T::zero(); - for (p, t) in pred.iter().cloned().zip(target.iter().cloned()) { - res += (p - t).abs(); - } - res /= T::from(pred.len()).unwrap(); - res - } -} - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct MeanSquaredError; - -impl MeanSquaredError { - pub fn new() -> Self { - Self - } - - pub fn partial_slope( - data: &Array2, - target: &Array1, - bias: &Array1, - weights: &Array2, - ) -> T - where - T: FromPrimitive + NdFloat, - { - let pred = data.dot(&weights.t().to_owned()) + bias.clone(); - let error = target - &pred; - let w = data.t().dot(&error) * (-T::from(2).unwrap()); - w.mean().unwrap() - } - pub fn partial( - data: &Array, - bias: &Array1, - slope: &Array, - target: &Array1, - ) -> (T, T) - where - D: Dimension, - T: FromPrimitive + NdFloat, - Array: Dot>, - as Dot>>::Output: ops::Add, Output = Array> - + ops::Sub, Output = Array> - + ops::Mul>, - Array1: ops::Sub, Output = Array>, - { - let predicted = data.dot(&slope.t().to_owned()) + bias.clone(); - let w = data.dot(&(target.clone() - predicted.clone())) * (-T::from(2).unwrap()); - let b = (target.clone() - predicted) * (-T::from(2).unwrap()); - (w.mean().unwrap(), b.mean().unwrap()) - } -} - -impl Loss> for MeanSquaredError -where - D: Dimension, - T: NdFloat, -{ - type Output = T; - - fn loss(&self, pred: &Array, target: &Array) -> Self::Output { - let res = pred - .iter() - .cloned() - .zip(target.iter().cloned()) - .fold(T::zero(), |i, (p, t)| i + (p - t).powi(2)); - res / T::from(pred.len()).unwrap() - } -} diff --git a/ml/neural/src/func/prop/mod.rs b/ml/neural/src/func/prop/mod.rs index 8b62f037..1896d22a 100644 --- a/ml/neural/src/func/prop/mod.rs +++ b/ml/neural/src/func/prop/mod.rs @@ -5,26 +5,11 @@ //! # Propagation //! //! This module describes the propagation of data through a neural network. -pub use self::{modes::*, propagation::*, utils::*}; +pub use self::{modes::*, utils::*}; pub(crate) mod modes; -pub(crate) mod propagation; - -// pub mod forward; -use ndarray::prelude::{Array, Ix2}; - -pub type ForwardDyn = Box, Output = Array>>; - -pub trait Backward: Forward { - type Optim; - - fn backward(&mut self, args: &T, grad: &T); -} - -pub trait Forward { - type Output; - - fn forward(&self, args: &T) -> Self::Output; -} pub(crate) mod utils {} + +#[cfg(test)] +mod tests {} diff --git a/ml/neural/src/func/prop/propagation.rs b/ml/neural/src/func/prop/propagation.rs deleted file mode 100644 index 589b2cd1..00000000 --- a/ml/neural/src/func/prop/propagation.rs +++ /dev/null @@ -1,16 +0,0 @@ -/* - Appellation: propagation - Contrib: FL03 -*/ -use super::PropagationMode; - -pub struct Propagator { - pub epochs: usize, - pub mode: PropagationMode, -} - -impl Propagator { - pub fn new(epochs: usize, mode: PropagationMode) -> Self { - Self { epochs, mode } - } -} diff --git a/ml/neural/src/layers/cmp/features.rs b/ml/neural/src/layers/cmp/features.rs index 614c1274..742dac1c 100644 --- a/ml/neural/src/layers/cmp/features.rs +++ b/ml/neural/src/layers/cmp/features.rs @@ -61,7 +61,7 @@ impl IntoDimension for LayerShape { impl From for Ix2 { fn from(features: LayerShape) -> Self { - ndarray::Ix2(features.outputs, features.inputs) + features.into_dimension() } } diff --git a/ml/neural/src/layers/exp/config.rs b/ml/neural/src/layers/exp/config.rs index 01b4c348..00ed5705 100644 --- a/ml/neural/src/layers/exp/config.rs +++ b/ml/neural/src/layers/exp/config.rs @@ -5,7 +5,7 @@ use crate::layers::{LayerKind, LayerShape}; use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] pub struct LayerConfig { biased: bool, features: LayerShape, @@ -27,8 +27,12 @@ impl LayerConfig { self.biased } - pub fn features(&self) -> LayerShape { - self.features + pub fn features(&self) -> &LayerShape { + &self.features + } + + pub fn features_mut(&mut self) -> &mut LayerShape { + &mut self.features } pub fn kind(&self) -> LayerKind { @@ -65,11 +69,6 @@ impl LayerConfig { self } - pub fn with_bias(mut self, biased: bool) -> Self { - self.biased = biased; - self - } - pub fn with_features(mut self, features: LayerShape) -> Self { self.features = features; self @@ -85,3 +84,14 @@ impl LayerConfig { self } } + +impl From for LayerConfig { + fn from(features: LayerShape) -> Self { + Self { + biased: false, + features, + kind: LayerKind::default(), + name: String::new(), + } + } +} diff --git a/ml/neural/src/layers/exp/layer.rs b/ml/neural/src/layers/exp/layer.rs index 6fe7a07a..496eb78d 100644 --- a/ml/neural/src/layers/exp/layer.rs +++ b/ml/neural/src/layers/exp/layer.rs @@ -2,6 +2,7 @@ Appellation: model Contrib: FL03 */ +use super::LayerConfig; use crate::func::activate::{Activate, Activator, Linear}; use crate::layers::{LayerParams, LayerShape}; use crate::prelude::{Features, Forward, Neuron, Node, Parameterized, Params}; @@ -15,8 +16,7 @@ where T: Float, { activator: Activator, - pub features: LayerShape, - name: String, + config: LayerConfig, params: LayerParams, } @@ -24,13 +24,12 @@ impl Layer where T: Float, { - pub fn new(activator: impl Activate + 'static, features: impl Into) -> Self { - let features = features.into(); + pub fn new(activator: impl Activate + 'static, config: LayerConfig) -> Self { + let params = LayerParams::new(*config.features()); Self { activator: Activator::new(Box::new(activator)), - features, - name: String::new(), - params: LayerParams::new(features), + config, + params, } } @@ -38,12 +37,12 @@ where &self.activator } - pub fn name(&self) -> &str { - &self.name + pub fn config(&self) -> &LayerConfig { + &self.config } - pub fn set_name(&mut self, name: impl ToString) { - self.name = name.to_string(); + pub fn config_mut(&mut self) -> &mut LayerConfig { + &mut self.config } pub fn set_node(&mut self, idx: usize, node: &Node) { @@ -56,11 +55,6 @@ where } self.features().outputs() == other.features().inputs() } - - pub fn with_name(mut self, name: impl ToString) -> Self { - self.name = name.to_string(); - self - } } impl Layer @@ -118,11 +112,11 @@ where type Params = LayerParams; fn features(&self) -> &LayerShape { - &self.features + self.config().features() } fn features_mut(&mut self) -> &mut LayerShape { - &mut self.features + self.config_mut().features_mut() } fn params(&self) -> &LayerParams { @@ -149,7 +143,7 @@ where T: Float + 'static, { fn from(features: LayerShape) -> Self { - Self::new(Activator::linear(), features) + Self::new(Activator::linear(), features.into()) } } @@ -173,8 +167,7 @@ where let params = LayerParams::from_iter(nodes); Self { activator: Activator::linear(), - features: *params.features(), - name: String::new(), + config: LayerConfig::from(*params.features()), params, } } diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 7ee61309..9af0be36 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -18,7 +18,7 @@ where T: Float, { activator: A, - pub features: LayerShape, + features: LayerShape, name: String, params: LayerParams, } @@ -28,7 +28,8 @@ where A: Default + Activate, T: Float, { - pub fn new(features: LayerShape) -> Self { + pub fn from_features(inputs: usize, outputs: usize) -> Self { + let features = LayerShape::new(inputs, outputs); Self { activator: A::default(), features, @@ -43,6 +44,15 @@ where A: Activate, T: Float, { + pub fn new(activator: A, features: LayerShape, name: impl ToString) -> Self { + Self { + activator, + features, + name: name.to_string(), + params: LayerParams::new(features), + } + } + pub fn activator(&self) -> &A { &self.activator } @@ -114,7 +124,7 @@ where T: NdFloat, { pub fn linear(&self, args: &Array2) -> Array2 { - args.dot(&self.params.weights().t()) + self.params.bias() + self.params().forward(args) } } @@ -231,7 +241,12 @@ where T: Float, { fn from(features: LayerShape) -> Self { - Self::new(features) + Self { + activator: A::default(), + features, + name: String::new(), + params: LayerParams::new(features), + } } } diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index fe06f625..611db39f 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -13,19 +13,23 @@ pub(crate) mod stack; pub mod exp; use crate::prelude::{Activate, ActivateDyn, Forward, Node}; -use ndarray::prelude::{Array2, Ix2}; +use ndarray::prelude::{Array2, Ix2, NdFloat}; // use ndarray::IntoDimension; use num::Float; pub type LayerDyn = Layer>; -pub trait L: IntoIterator> +pub trait L: Forward> where A: Activate, T: Float, { + fn activator(&self) -> &A; + fn features(&self) -> LayerShape; + fn name(&self) -> &str; + fn params(&self) -> &LayerParams; fn is_biased(&self) -> bool; @@ -102,7 +106,7 @@ mod tests { let layer = Layer::::from(features).init(true); for node in layer.into_iter() { - assert_eq!(node.features(), inputs); + assert_eq!(node.features(), &inputs); assert_eq!(node.bias().dim(), ()); } } diff --git a/ml/neural/src/layers/params.rs b/ml/neural/src/layers/params.rs index 9e7f1c47..fe7f3449 100644 --- a/ml/neural/src/layers/params.rs +++ b/ml/neural/src/layers/params.rs @@ -5,10 +5,12 @@ use super::LayerShape; use crate::core::prelude::GenerateRandom; use crate::prelude::{Biased, Features, Forward, Node, Weighted}; -use ndarray::prelude::{Array1, Array2, Axis, Ix2, NdFloat}; +use ndarray::linalg::Dot; +use ndarray::prelude::{Array, Array1, Array2, Axis, Dimension, Ix2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; +use std::ops; #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct LayerParams { @@ -148,25 +150,16 @@ where } } -impl Forward> for LayerParams +impl Forward> for LayerParams where + D: Dimension, T: NdFloat, + Array: Dot, Output = Array> + ops::Add, Output = Array>, { - type Output = Array1; + type Output = Array; - fn forward(&self, input: &Array1) -> Self::Output { - input.dot(self.weights()) + self.bias() - } -} - -impl Forward> for LayerParams -where - T: NdFloat, -{ - type Output = Array2; - - fn forward(&self, input: &Array2) -> Self::Output { - input.dot(self.weights()) + self.bias() + fn forward(&self, input: &Array) -> Self::Output { + input.dot(&self.weights().t().to_owned()) + self.bias().clone() } } @@ -195,7 +188,7 @@ where let nodes = nodes.into_iter().collect::>(); let mut iter = nodes.iter(); let node = iter.next().unwrap(); - let shape = LayerShape::new(node.features(), nodes.len()); + let shape = LayerShape::new(*node.features(), nodes.len()); let mut params = LayerParams::new(shape); params.set_node(0, node.clone()); for (i, node) in iter.into_iter().enumerate() { diff --git a/ml/neural/src/models/config.rs b/ml/neural/src/models/config.rs index 64d24648..cf28ad27 100644 --- a/ml/neural/src/models/config.rs +++ b/ml/neural/src/models/config.rs @@ -17,4 +17,8 @@ impl ModelConfig { pub fn layers(&self) -> usize { self.layers } + + pub fn n_hidden(&self) -> usize { + self.layers - 2 + } } diff --git a/ml/neural/src/models/exp/mod.rs b/ml/neural/src/models/exp/mod.rs new file mode 100644 index 00000000..348e9548 --- /dev/null +++ b/ml/neural/src/models/exp/mod.rs @@ -0,0 +1,13 @@ +/* + Appellation: exp + Contrib: FL03 +*/ +//! # Experimental Models +pub use self::{modules::*, utils::*}; + +pub(crate) mod modules; + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests {} diff --git a/ml/neural/src/models/exp/modules.rs b/ml/neural/src/models/exp/modules.rs new file mode 100644 index 00000000..7a7ebd82 --- /dev/null +++ b/ml/neural/src/models/exp/modules.rs @@ -0,0 +1,28 @@ +/* + Appellation: modules + Contrib: FL03 +*/ +//! # Model +//! +use crate::prelude::Forward; +use ndarray::prelude::Array2; +use num::Float; + +pub trait Module: Forward, Output = Array2> +where + T: Float, +{ + type Config; + + fn add_module(&mut self, module: impl Module); + + fn compile(&mut self); + /// Returns a collection of all proceeding [Module]s in the network + fn children(&self) -> &Vec>; + + fn children_mut(&mut self) -> &mut Vec>; + /// Returns a collection of all [Module]s in the network + fn modules(&self) -> Vec<&impl Module>; + + fn name(&self) -> &str; +} diff --git a/ml/neural/src/models/mod.rs b/ml/neural/src/models/mod.rs index ad23ace9..504e6242 100644 --- a/ml/neural/src/models/mod.rs +++ b/ml/neural/src/models/mod.rs @@ -10,33 +10,9 @@ pub(crate) mod config; pub(crate) mod model; pub(crate) mod params; -use crate::prelude::Forward; -use ndarray::prelude::Array2; -use num::Float; - -pub trait Compiler { - type Opt; - - fn compile(&mut self); - - fn optimizer(&self) -> &Self::Opt; -} - -pub trait Module: Forward, Output = Array2> -where - T: Float, -{ - fn add_module(&mut self, module: impl Module); - - fn compile(&mut self); - /// Returns a collection of all proceeding [Module]s in the network - fn children(&self) -> &Vec>; - - fn children_mut(&mut self) -> &mut Vec>; - /// Returns a collection of all [Module]s in the network - fn modules(&self) -> Vec<&impl Module>; - - fn name(&self) -> &str; -} +pub mod exp; pub(crate) mod utils {} + +#[cfg(test)] +mod tests {} diff --git a/ml/neural/src/models/params.rs b/ml/neural/src/models/params.rs index dd0360f9..6d210d45 100644 --- a/ml/neural/src/models/params.rs +++ b/ml/neural/src/models/params.rs @@ -2,14 +2,48 @@ Appellation: stack Contrib: FL03 */ -use crate::prelude::{Features, LayerParams, LayerShape}; +use crate::prelude::{Features, LayerParams, LayerPosition, LayerShape}; use ndarray::prelude::{Dimension, Ix2}; use ndarray::IntoDimension; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; + use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use std::ops; +pub struct ModelMap +where + T: Float, +{ + store: HashMap>, +} + +impl ModelMap +where + T: Float, +{ + pub fn with_shapes(shapes: impl IntoIterator) -> Self + where + Sh: IntoDimension, + { + let tmp = Vec::from_iter(shapes.into_iter().map(IntoDimension::into_dimension)); + let mut store = HashMap::new(); + for (i, (inputs, outputs)) in tmp.iter().map(|s| s.into_pattern()).enumerate() { + let features = LayerShape::new(inputs, outputs); + let position = if i == 0 { + LayerPosition::input() + } else if i == tmp.len() - 1 { + LayerPosition::output(i) + } else { + LayerPosition::hidden(i) + }; + store.insert(position, LayerParams::new(features)); + } + Self { store } + } +} + #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] pub struct ModelParams where @@ -38,12 +72,18 @@ where where Sh: IntoDimension, { + let tmp = Vec::from_iter(shapes.into_iter().map(IntoDimension::into_dimension)); let mut children = Vec::new(); - for (inputs, outputs) in shapes - .into_iter() - .map(|s| s.into_dimension().into_pattern()) - { + for (i, (inputs, outputs)) in tmp.iter().map(|s| s.into_pattern()).enumerate() { let features = LayerShape::new(inputs, outputs); + let position = if i == 0 { + LayerPosition::input() + } else if i == tmp.len() - 1 { + LayerPosition::output(i) + } else { + LayerPosition::hidden(i) + }; + children.push(LayerParams::new(features)); } Self { children } diff --git a/ml/neural/src/neurons/neuron.rs b/ml/neural/src/neurons/neuron.rs index 11ed789d..b72723a5 100644 --- a/ml/neural/src/neurons/neuron.rs +++ b/ml/neural/src/neurons/neuron.rs @@ -17,7 +17,6 @@ where T: Float, { activation: A, - features: usize, node: Node, } @@ -26,14 +25,6 @@ where A: Activate, T: Float, { - pub fn activation(&self) -> &A { - &self.activation - } - - pub fn features(&self) -> usize { - self.features - } - pub fn node(&self) -> &Node { &self.node } @@ -54,7 +45,6 @@ where pub fn with_rho>(self, rho: B) -> Neuron { Neuron { activation: rho, - features: self.features, node: self.node, } } @@ -78,7 +68,6 @@ where pub fn new(features: usize) -> Self { Self { activation: A::default(), - features, node: Node::new(features), } } @@ -171,11 +160,11 @@ where type Params = Node; fn features(&self) -> &Self::Features { - &self.features + self.node.features() } fn features_mut(&mut self) -> &mut Self::Features { - &mut self.features + self.node.features_mut() } fn params(&self) -> &Self::Params { @@ -217,7 +206,6 @@ where fn from((weights, bias): (Array1, Array0)) -> Self { Self { activation: A::default(), - features: weights.len(), node: Node::from((weights, bias)), } } @@ -231,7 +219,6 @@ where fn from((weights, bias): (Array1, T)) -> Self { Self { activation: A::default(), - features: weights.len(), node: Node::from((weights, bias)), } } @@ -245,7 +232,6 @@ where fn from((weights, bias, activation): (Array1, Array0, A)) -> Self { Self { activation, - features: weights.len(), node: Node::from((weights, bias)), } } @@ -259,7 +245,6 @@ where fn from((weights, bias, activation): (Array1, T, A)) -> Self { Self { activation, - features: weights.len(), node: Node::from((weights, bias)), } } diff --git a/ml/neural/src/neurons/node.rs b/ml/neural/src/neurons/node.rs index 860d6761..3c001828 100644 --- a/ml/neural/src/neurons/node.rs +++ b/ml/neural/src/neurons/node.rs @@ -15,7 +15,7 @@ where T: Float, { bias: Array0, - pub features: usize, + features: usize, weights: Array1, } @@ -31,8 +31,8 @@ where } } - pub fn features(&self) -> usize { - self.features + pub fn features(&self) -> &usize { + &self.features } pub fn features_mut(&mut self) -> &mut usize { @@ -44,7 +44,7 @@ where } pub fn with_bias(mut self, bias: Array0) -> Self { - self.bias = bias.into(); + self.bias = bias; self } @@ -87,6 +87,7 @@ where impl Node where T: FromPrimitive + NdFloat, + Self: Weighted, { pub fn apply_gradient(&mut self, gamma: T, gradient: G) where @@ -102,7 +103,12 @@ where { activator(&self.linear(data)) } - +} +impl Node +where + T: FromPrimitive + NdFloat, + Self: Biased + Weighted, +{ pub fn linear(&self, data: &Array2) -> Array1 { data.dot(&self.weights().t()) + self.bias() } @@ -110,7 +116,7 @@ where impl Forward> for Node where - Self: Biased, + Self: Biased + Weighted, T: FromPrimitive + NdFloat, { type Output = T; @@ -122,7 +128,7 @@ where impl Forward> for Node where - Self: Biased, + Self: Biased + Weighted, T: FromPrimitive + NdFloat, { type Output = Array1; diff --git a/ml/neural/src/nn/gnn/mod.rs b/ml/neural/src/nn/gnn/mod.rs index 644ca496..2b8f4900 100644 --- a/ml/neural/src/nn/gnn/mod.rs +++ b/ml/neural/src/nn/gnn/mod.rs @@ -4,9 +4,10 @@ */ //! # Graph Neural Network //! -pub use self::{model::*, utils::*}; +pub use self::{model::*, tasks::*, utils::*}; pub(crate) mod model; +pub(crate) mod tasks; use num::Float; diff --git a/ml/neural/src/nn/gnn/model.rs b/ml/neural/src/nn/gnn/model.rs index e2a0d764..147a580e 100644 --- a/ml/neural/src/nn/gnn/model.rs +++ b/ml/neural/src/nn/gnn/model.rs @@ -2,3 +2,22 @@ Appellation: model Contrib: FL03 */ +use crate::prelude::Node; +use ndarray::prelude::Array; +use num::Float; +use petgraph::prelude::{Directed, Graph}; + +pub enum Edges { + Biased { bias: f64, weights: Vec }, + + Unbiased { weights: Vec }, + + Empty, +} + +pub struct GraphModel +where + T: Float, +{ + graph: Graph, K>, +} diff --git a/ml/neural/src/nn/gnn/tasks.rs b/ml/neural/src/nn/gnn/tasks.rs new file mode 100644 index 00000000..128d00c9 --- /dev/null +++ b/ml/neural/src/nn/gnn/tasks.rs @@ -0,0 +1,18 @@ +/* + Appellation: model + Contrib: FL03 +*/ +use num::Float; + +pub trait GraphTask +where + T: Float, +{ + type G; + + fn depth(&self) -> usize { + self.layers().len() + } + + fn layers(&self) -> &[Self::G]; +} diff --git a/ml/neural/src/primitives.rs b/ml/neural/src/primitives.rs index 23346bb7..120654da 100644 --- a/ml/neural/src/primitives.rs +++ b/ml/neural/src/primitives.rs @@ -11,9 +11,13 @@ pub(crate) mod constants { pub(crate) mod statics {} pub(crate) mod types { + use crate::prelude::Forward; + use ndarray::prelude::{Array, Ix2}; pub type BoxedFunction = Box T>; - + /// + pub type ForwardDyn = Box, Output = Array>>; + /// pub type LayerBias = ndarray::Array1; pub type LayerWeight = ndarray::Array2; diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index c654ccc3..d8c3ac2f 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -4,9 +4,19 @@ */ use crate::core::BoxResult; use crate::func::loss::Loss; -use ndarray::prelude::{Array, Axis, Dimension, Ix2}; +use ndarray::prelude::{Array, Array1, Axis, Dimension, Ix2}; use num::Float; +pub trait Backward: Forward { + fn backward(&mut self, args: &T, grad: &T); +} + +pub trait Forward { + type Output; + + fn forward(&self, args: &T) -> Self::Output; +} + pub trait Compile where D: Dimension, @@ -26,7 +36,7 @@ where fn predict(&self, input: &Array) -> BoxResult; - fn predict_batch(&self, input: &[Array]) -> BoxResult> { + fn predict_batch(&self, input: &[Array]) -> BoxResult> { let res = input.iter().map(|x| self.predict(x).expect("")).collect(); Ok(res) } diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs index ba88db36..3e8ad074 100644 --- a/ml/optim/src/grad/gradient.rs +++ b/ml/optim/src/grad/gradient.rs @@ -4,7 +4,7 @@ */ use crate::neural::func::activate::Sigmoid; use crate::neural::models::ModelParams; -use crate::neural::prelude::{Forward, Gradient, LayerParams,}; +use crate::neural::prelude::{Forward, Gradient, LayerParams}; use ndarray::prelude::{Array2, NdFloat}; use ndarray_stats::DeviationExt; use num::{Float, Signed}; @@ -65,8 +65,6 @@ where let mut cost = T::zero(); let params = self.params.clone(); - - for (i, layer) in self.params[..(depth - 1)].iter_mut().enumerate() { // compute the prediction of the model let pred = params[i + 1].forward(data); @@ -88,7 +86,11 @@ where #[cfg(test)] mod tests { use super::*; + use crate::core::prelude::linarr; use crate::neural::models::ModelParams; + use crate::neural::prelude::{Features, LayerShape}; + + use ndarray::prelude::Ix2; #[test] fn test_gradient() { @@ -97,8 +99,14 @@ mod tests { let _shape = (samples, inputs); - let mut model = ModelParams::::new().build_layers([(inputs, outputs,), (outputs, outputs), (outputs, outputs)]).init_layers(false); + let features = LayerShape::new(inputs, outputs); + + let x = linarr::((samples, features.inputs())).unwrap(); + let y = linarr::((samples, features.outputs())).unwrap(); + let mut shapes = vec![features]; + shapes.extend((0..3).map(|_| LayerShape::new(features.outputs(), features.outputs()))); + let mut model = ModelParams::::from_iter(shapes); } } diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index 30f513a8..52e6a146 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -3,10 +3,11 @@ Contrib: FL03 */ //! # Gradient Descent -pub use self::{descent::*, gradient::*, utils::*}; +pub use self::{descent::*, gradient::*, modes::*, utils::*}; pub(crate) mod descent; pub(crate) mod gradient; +pub(crate) mod modes; pub mod sgd; diff --git a/ml/optim/src/grad/modes.rs b/ml/optim/src/grad/modes.rs new file mode 100644 index 00000000..0aa05f5a --- /dev/null +++ b/ml/optim/src/grad/modes.rs @@ -0,0 +1,34 @@ +/* + Appellation: kinds + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum Mode { + Batch, + #[default] + Descent, + Stochastic, +} diff --git a/ml/optim/src/grad/sgd.rs b/ml/optim/src/grad/sgd.rs index 5777c4f4..12473391 100644 --- a/ml/optim/src/grad/sgd.rs +++ b/ml/optim/src/grad/sgd.rs @@ -213,25 +213,24 @@ where #[cfg(test)] mod tests { use super::*; - use crate::core::prelude::GenerateRandom; + use crate::core::prelude::linarr; use crate::neural::prelude::{LayerShape, Sigmoid}; - use ndarray::prelude::{Array, Array1}; #[test] fn test_sgd() { - let (samples, inputs, outputs) = (20, 5, 4); - let shape = (samples, inputs); + let (samples, inputs) = (20, 5); + let outputs = 4; let features = LayerShape::new(inputs, outputs); - let (batch_size, epochs, gamma) = (10, 1, 0.01); + let (_bs, _epochs, _gamma) = (10, 1, 0.01); // Generate some example data - let x = Array::linspace(1., 100., 100).into_shape(shape).unwrap(); - let y = Array::linspace(1., 100., samples) - .into_shape(samples) - .unwrap(); + let x = linarr::((samples, inputs)).unwrap(); + let _y = linarr::((samples, outputs)).unwrap(); - let mut model = Layer::::new(features).init(true); + let model = Layer::::from(features).init(true); + + let _pred = model.forward(&x); // let mut sgd = StochasticGradientDescent::new(batch_size, epochs, gamma, model); // sgd.sgd(&x, &y); diff --git a/ml/optim/src/optimizer.rs b/ml/optim/src/optimizer.rs index 538f027d..abc293d9 100644 --- a/ml/optim/src/optimizer.rs +++ b/ml/optim/src/optimizer.rs @@ -31,3 +31,8 @@ where T: Float, { } + +pub struct Opt { + epochs: usize, + gamma: T, +} diff --git a/ml/optim/src/specs.rs b/ml/optim/src/specs.rs index 74e10d97..c35bd373 100644 --- a/ml/optim/src/specs.rs +++ b/ml/optim/src/specs.rs @@ -14,15 +14,10 @@ where fn apply_gradient(&mut self, gamma: T, gradients: &Array); } -pub trait Optimize { - type Model: Forward, Output = Array2>; - - fn name(&self) -> &str; - - // fn optimize(&mut self, model: &mut Self::Model, args: &Array2, targets: &Array2) -> T { - // let gradients = model.backward(args, targets); - // let loss = model.loss(args, targets); - // self.update(model, &gradients); - // loss - // } +pub trait Autograd +where + D: Dimension, + T: Float, +{ + fn autograd(&mut self, loss: &Array) -> Array; } diff --git a/ml/transformers/src/attention/multi/attention.rs b/ml/transformers/src/attention/multi/attention.rs index cbc94529..27ef8aaf 100644 --- a/ml/transformers/src/attention/multi/attention.rs +++ b/ml/transformers/src/attention/multi/attention.rs @@ -46,7 +46,7 @@ where let weights = Weight::uniform((model, model)); Self { features, - linear: Layer::new((model, model).into()), + linear: Layer::from_features(model, model), weights, } } diff --git a/ml/transformers/src/ffn/network.rs b/ml/transformers/src/ffn/network.rs index 1b2cfe75..3a99a657 100644 --- a/ml/transformers/src/ffn/network.rs +++ b/ml/transformers/src/ffn/network.rs @@ -25,8 +25,8 @@ where { pub fn new(model: usize, network: usize) -> Self { Self { - input: Layer::new((model, network).into()), - output: Layer::new((network, model).into()), + input: Layer::from_features(model, network), + output: Layer::from_features(network, model), params: FFNParams::new(model, network), } } diff --git a/ml/transformers/src/transform/transformer.rs b/ml/transformers/src/transform/transformer.rs index da445d41..b89a40bf 100644 --- a/ml/transformers/src/transform/transformer.rs +++ b/ml/transformers/src/transform/transformer.rs @@ -8,10 +8,13 @@ use ndarray::prelude::{Array2, NdFloat}; #[derive(Clone, Debug, Default)] pub struct Transformer; -impl Transform> for Transformer where T: NdFloat { - type Output = Array2; +impl Transform> for Transformer +where + T: NdFloat, +{ + type Output = Array2; - fn transform(&self, args: &Array2) -> Self::Output { - args.clone() - } + fn transform(&self, args: &Array2) -> Self::Output { + args.clone() + } } From 4b8912ab031ae9477615b309e80ffa449db0a40c Mon Sep 17 00:00:00 2001 From: FL03 Date: Mon, 4 Dec 2023 15:49:29 -0600 Subject: [PATCH 078/118] update Signed-off-by: FL03 --- concision/examples/gradients.rs | 19 +- ml/neural/src/layers/exp/layer.rs | 2 +- ml/neural/src/layers/layer.rs | 18 +- ml/neural/src/models/model.rs | 41 ++++- ml/neural/src/models/params.rs | 20 +-- ml/neural/src/neurons/mod.rs | 5 +- ml/neural/src/neurons/neuron.rs | 261 ---------------------------- ml/neural/src/neurons/node.rs | 1 + ml/neural/src/neurons/perceptron.rs | 248 ++++++++++++++++++++++++-- ml/optim/src/grad/gradient.rs | 1 + 10 files changed, 318 insertions(+), 298 deletions(-) delete mode 100644 ml/neural/src/neurons/neuron.rs diff --git a/concision/examples/gradients.rs b/concision/examples/gradients.rs index 6e85f34c..8ee604e3 100644 --- a/concision/examples/gradients.rs +++ b/concision/examples/gradients.rs @@ -1,4 +1,4 @@ -use concision::neural::models::ModelParams; +use concision::neural::models::{Model, ModelConfig, ModelParams}; use concision::neural::prelude::{Layer, Sigmoid}; use concision::optim::grad::*; use concision::prelude::{linarr, Features, Forward, LayerShape}; @@ -13,9 +13,9 @@ fn main() -> anyhow::Result<()> { let (epochs, gamma) = (1000, 0.005); - sample_gradient(epochs, features, gamma, samples)?; + // sample_gradient(epochs, features, gamma, samples)?; - // sample_model(epochs, features, gamma, samples)?; + sample_model(epochs, features, gamma, samples)?; Ok(()) } @@ -62,8 +62,10 @@ pub fn sample_model( let mut shapes = vec![features]; shapes.extend((0..3).map(|_| LayerShape::new(features.outputs(), features.outputs()))); - let model = ModelParams::::from_iter(shapes); - let mut opt = Grad::new(gamma, model.clone(), Sigmoid); + let config = ModelConfig::new(4); + let params = ModelParams::::from_iter(shapes); + let mut model = Model::::new(config).with_params(params); + // let mut opt = Grad::new(gamma, model.clone(), Sigmoid); // println!( // "Targets (dim):\t{:?}\nPredictions:\n\n{:?}\n", @@ -71,11 +73,12 @@ pub fn sample_model( // model.forward(&x) // ); - let mut losses = Array1::zeros(epochs); + // let mut losses = Array1::zeros(epochs); for e in 0..epochs { - let cost = opt.step(&x, &y)?; + let _cost = model.gradient(&x, &y, gamma, Sigmoid)?; + // let cost = opt.step(&x, &y)?; // let cost = model.grad(gamma, &x, &y); - losses[e] = cost; + // losses[e] = cost; } // model = opt.model().clone(); // println!("Losses:\n\n{:?}\n", &losses); diff --git a/ml/neural/src/layers/exp/layer.rs b/ml/neural/src/layers/exp/layer.rs index 496eb78d..09d04941 100644 --- a/ml/neural/src/layers/exp/layer.rs +++ b/ml/neural/src/layers/exp/layer.rs @@ -5,7 +5,7 @@ use super::LayerConfig; use crate::func::activate::{Activate, Activator, Linear}; use crate::layers::{LayerParams, LayerShape}; -use crate::prelude::{Features, Forward, Neuron, Node, Parameterized, Params}; +use crate::prelude::{Features, Forward, Node, Parameterized, Params, Perceptron}; use ndarray::prelude::{Array2, Ix1, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 9af0be36..a4466e16 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -4,7 +4,7 @@ */ use super::{LayerParams, LayerShape}; use crate::func::activate::{Activate, Gradient, Linear}; -use crate::prelude::{Features, Forward, Neuron, Node, Parameterized, Params}; +use crate::prelude::{Features, Forward, Node, Parameterized, Params, Perceptron}; use ndarray::prelude::{Array2, Ix1, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_stats::DeviationExt; @@ -65,7 +65,7 @@ where self.name = name.to_string(); } - pub fn set_node(&mut self, idx: usize, neuron: &Neuron) + pub fn set_node(&mut self, idx: usize, neuron: &Perceptron) where A: Activate, { @@ -177,6 +177,20 @@ where } } +// impl Forward> for Layer +// where +// A: Activate, +// D: Dimension, +// T: NdFloat, +// Array: Dot, Output = Array>, +// { +// type Output = Array2; + +// fn forward(&self, args: &Array2) -> Self::Output { +// self.activator.activate(&self.linear(args)) +// } +// } + impl Forward> for Layer where A: Activate, diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs index a9f5b4ec..ca4bee66 100644 --- a/ml/neural/src/models/model.rs +++ b/ml/neural/src/models/model.rs @@ -3,8 +3,8 @@ Contrib: FL03 */ use super::{ModelConfig, ModelParams}; -use crate::prelude::{Forward, LayerParams}; -use ndarray::prelude::{Array2, NdFloat}; +use crate::prelude::{Forward, Gradient, LayerParams, Weighted}; +use ndarray::prelude::{Array1, Array2, NdFloat}; use num::Float; #[derive(Clone, Debug)] @@ -42,6 +42,43 @@ where pub fn params_mut(&mut self) -> &mut ModelParams { &mut self.params } + + pub fn with_params(mut self, params: ModelParams) -> Self { + self.params = params; + self + } +} + +impl Model +where + T: NdFloat, +{ + pub fn gradient(&mut self, data: &Array2, targets: &Array2, gamma: T, grad: impl Gradient) -> anyhow::Result<()> { + let mut grads = Vec::new(); + // let mut loss = Array1::zeros(self.params.len()); + let mut store = vec![data.clone()]; + + for layer in self.clone().into_iter() { + let pred = layer.forward(&store.last().unwrap()); + store.push(pred); + } + + let error = targets - store.last().unwrap(); + let dz = &error * grad.gradient(&error); + grads.push(dz.clone()); + + for i in (1..self.params.len()).rev() { + let wt = self.params[i].weights().t(); + let dz = &dz.dot(&wt) * grad.gradient(&store[i]); + grads[i - 1] = dz.clone(); + } + + for i in 0..self.params.len() { + let gradient = &store[i].t().dot(&grads[i]); + self.params[i].weights_mut().scaled_add(-gamma, &gradient.t()); + } + Ok(()) + } } // impl FromIterator for Model diff --git a/ml/neural/src/models/params.rs b/ml/neural/src/models/params.rs index 6d210d45..2bf2b6b1 100644 --- a/ml/neural/src/models/params.rs +++ b/ml/neural/src/models/params.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use crate::prelude::{Features, LayerParams, LayerPosition, LayerShape}; -use ndarray::prelude::{Dimension, Ix2}; +use ndarray::prelude::{Dimension, Ix2, NdFloat}; use ndarray::IntoDimension; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; @@ -74,16 +74,8 @@ where { let tmp = Vec::from_iter(shapes.into_iter().map(IntoDimension::into_dimension)); let mut children = Vec::new(); - for (i, (inputs, outputs)) in tmp.iter().map(|s| s.into_pattern()).enumerate() { + for (inputs, outputs) in tmp.iter().map(|s| s.into_pattern()) { let features = LayerShape::new(inputs, outputs); - let position = if i == 0 { - LayerPosition::input() - } else if i == tmp.len() - 1 { - LayerPosition::output(i) - } else { - LayerPosition::hidden(i) - }; - children.push(LayerParams::new(features)); } Self { children } @@ -143,6 +135,14 @@ where } } +impl ModelParams +where + T: NdFloat, +{ + +} + + impl AsRef<[LayerParams]> for ModelParams where T: Float, diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index 65d51dbf..29d9cda9 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -3,9 +3,8 @@ Contrib: FL03 */ //! # neurons -pub use self::{neuron::*, node::*, perceptron::*, synapse::*, utils::*}; +pub use self::{node::*, perceptron::*, synapse::*, utils::*}; -pub(crate) mod neuron; pub(crate) mod node; pub(crate) mod perceptron; pub(crate) mod synapse; @@ -61,7 +60,7 @@ mod tests { let data = array![[10.0, 10.0, 6.0, 1.0, 8.0]]; let weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; - let neuron = Neuron::::new(5).with_weights(weights.clone()); + let neuron = Perceptron::::new(5).with_weights(weights.clone()); let linear = data.dot(&weights) + bias; let exp = softmax(&linear); diff --git a/ml/neural/src/neurons/neuron.rs b/ml/neural/src/neurons/neuron.rs deleted file mode 100644 index b72723a5..00000000 --- a/ml/neural/src/neurons/neuron.rs +++ /dev/null @@ -1,261 +0,0 @@ -/* - Appellation: neuron - Contrib: FL03 -*/ -use super::Node; -use crate::func::activate::{Activate, Linear}; -use crate::prelude::{Forward, Parameterized, ParameterizedExt, Weighted}; -use ndarray::prelude::{Array0, Array1, Array2, Ix1, NdFloat}; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use num::Float; - -/// Artificial Neuron -#[derive(Clone, Debug, PartialEq)] -pub struct Neuron -where - A: Activate, - T: Float, -{ - activation: A, - node: Node, -} - -impl Neuron -where - A: Activate, - T: Float, -{ - pub fn node(&self) -> &Node { - &self.node - } - - pub fn node_mut(&mut self) -> &mut Node { - &mut self.node - } - - pub fn rho(&self) -> &A { - &self.activation - } - - pub fn with_bias(mut self, bias: Array0) -> Self { - self.node = self.node.with_bias(bias); - self - } - - pub fn with_rho>(self, rho: B) -> Neuron { - Neuron { - activation: rho, - node: self.node, - } - } - - pub fn with_node(mut self, node: Node) -> Self { - self.node = node; - self - } - - pub fn with_weights(mut self, weights: Array1) -> Self { - self.node = self.node.with_weights(weights); - self - } -} - -impl Neuron -where - T: NdFloat, - A: Activate + Default, -{ - pub fn new(features: usize) -> Self { - Self { - activation: A::default(), - node: Node::new(features), - } - } -} - -impl Neuron -where - T: NdFloat, - A: Activate, -{ - pub fn apply_gradient(&mut self, gamma: T, gradient: G) - where - G: Fn(&Array1) -> Array1, - { - let grad = gradient(self.node().weights()); - self.update_with_gradient(gamma, &grad); - } - - pub fn update_with_gradient(&mut self, gamma: T, grad: &Array1) { - self.weights_mut().scaled_add(-gamma, grad); - } -} - -impl Neuron -where - T: Float + SampleUniform, - A: Activate, -{ - pub fn init(mut self, biased: bool) -> Self { - if biased { - self = self.init_bias(); - } - self.init_weight() - } - - pub fn init_bias(mut self) -> Self { - self.node = self.node.init_bias(); - self - } - - pub fn init_weight(mut self) -> Self { - self.node = self.node.init_weight(); - self - } -} - -// impl Biased for Neuron -// where -// T: Float, -// A: Activate, -// { -// fn bias(&self) -> &Array0 { -// self.node.bias() -// } - -// fn bias_mut(&mut self) -> &mut Array0 { -// self.node.bias_mut() -// } - -// fn set_bias(&mut self, bias: Array0) { -// self.node.set_bias(bias); -// } -// } - -// impl Weighted for Neuron -// where -// T: Float, -// A: Activate, -// { -// fn weights(&self) -> &Array1 { -// self.node.weights() -// } - -// fn weights_mut(&mut self) -> &mut Array1 { -// self.node.weights_mut() -// } - -// fn set_weights(&mut self, weights: Array1) { -// self.node.set_weights(weights); -// } -// } - -impl Parameterized for Neuron -where - A: Activate, - T: Float, -{ - type Features = usize; - - type Params = Node; - - fn features(&self) -> &Self::Features { - self.node.features() - } - - fn features_mut(&mut self) -> &mut Self::Features { - self.node.features_mut() - } - - fn params(&self) -> &Self::Params { - &self.node - } - - fn params_mut(&mut self) -> &mut Self::Params { - &mut self.node - } -} - -// impl Forward> for Neuron { -// type Output = f64; - -// fn forward(&self, args: &Array1) -> Self::Output { -// self.rho().activate(args.dot(&self.weights().t().to_owned()) + self.bias) -// } - -// } - -impl Forward> for Neuron -where - T: NdFloat, - A: Activate, -{ - type Output = Array1; - - fn forward(&self, args: &Array2) -> Self::Output { - let linstep = args.dot(&self.weights().t()) + self.bias(); - self.rho().activate(&linstep) - } -} - -impl From<(Array1, Array0)> for Neuron -where - T: Float, - A: Activate + Default, -{ - fn from((weights, bias): (Array1, Array0)) -> Self { - Self { - activation: A::default(), - node: Node::from((weights, bias)), - } - } -} - -impl From<(Array1, T)> for Neuron -where - T: NdFloat, - A: Activate + Default, -{ - fn from((weights, bias): (Array1, T)) -> Self { - Self { - activation: A::default(), - node: Node::from((weights, bias)), - } - } -} - -impl From<(Array1, Array0, A)> for Neuron -where - T: Float, - A: Activate, -{ - fn from((weights, bias, activation): (Array1, Array0, A)) -> Self { - Self { - activation, - node: Node::from((weights, bias)), - } - } -} - -impl From<(Array1, T, A)> for Neuron -where - T: NdFloat, - A: Activate, -{ - fn from((weights, bias, activation): (Array1, T, A)) -> Self { - Self { - activation, - node: Node::from((weights, bias)), - } - } -} - -impl From> for (Array1, Array0) -where - T: Float, - A: Activate, -{ - fn from(neuron: Neuron) -> Self { - neuron.node().clone().into() - } -} diff --git a/ml/neural/src/neurons/node.rs b/ml/neural/src/neurons/node.rs index 3c001828..fb9fdb76 100644 --- a/ml/neural/src/neurons/node.rs +++ b/ml/neural/src/neurons/node.rs @@ -31,6 +31,7 @@ where } } + pub fn features(&self) -> &usize { &self.features } diff --git a/ml/neural/src/neurons/perceptron.rs b/ml/neural/src/neurons/perceptron.rs index 9a71a5c9..da2c5bab 100644 --- a/ml/neural/src/neurons/perceptron.rs +++ b/ml/neural/src/neurons/perceptron.rs @@ -1,35 +1,261 @@ /* - Appellation: perceptron + Appellation: neuron Contrib: FL03 */ use super::Node; -use crate::prelude::Forward; -use ndarray::prelude::{Array1, Array2, NdFloat}; -use num::{Float, FromPrimitive}; +use crate::func::activate::{Activate, Linear}; +use crate::prelude::{Forward, Parameterized, ParameterizedExt, Weighted}; +use ndarray::prelude::{Array0, Array1, Array2, Ix1, NdFloat}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use num::Float; -pub struct Perceptron +/// Artificial Neuron +#[derive(Clone, Debug, PartialEq)] +pub struct Perceptron where + A: Activate, T: Float, { + activation: A, node: Node, } -impl Perceptron +impl Perceptron where + A: Activate, T: Float, { - pub fn new(node: Node) -> Self { - Self { node } + pub fn node(&self) -> &Node { + &self.node + } + + pub fn node_mut(&mut self) -> &mut Node { + &mut self.node + } + + pub fn rho(&self) -> &A { + &self.activation + } + + pub fn with_bias(mut self, bias: Array0) -> Self { + self.node = self.node.with_bias(bias); + self + } + + pub fn with_rho>(self, rho: B) -> Perceptron { + Perceptron { + activation: rho, + node: self.node, + } + } + + pub fn with_node(mut self, node: Node) -> Self { + self.node = node; + self + } + + pub fn with_weights(mut self, weights: Array1) -> Self { + self.node = self.node.with_weights(weights); + self + } +} + +impl Perceptron +where + T: NdFloat, + A: Activate + Default, +{ + pub fn new(features: usize) -> Self { + Self { + activation: A::default(), + node: Node::new(features), + } + } +} + +impl Perceptron +where + T: NdFloat, + A: Activate, +{ + pub fn apply_gradient(&mut self, gamma: T, gradient: G) + where + G: Fn(&Array1) -> Array1, + { + let grad = gradient(self.node().weights()); + self.update_with_gradient(gamma, &grad); + } + + pub fn update_with_gradient(&mut self, gamma: T, grad: &Array1) { + self.node.weights_mut().scaled_add(-gamma, grad); } } -impl Forward> for Perceptron +impl Perceptron where - T: FromPrimitive + NdFloat, + T: Float + SampleUniform, + A: Activate, +{ + pub fn init(mut self, biased: bool) -> Self { + if biased { + self = self.init_bias(); + } + self.init_weight() + } + + pub fn init_bias(mut self) -> Self { + self.node = self.node.init_bias(); + self + } + + pub fn init_weight(mut self) -> Self { + self.node = self.node.init_weight(); + self + } +} + +// impl Biased for Neuron +// where +// T: Float, +// A: Activate, +// { +// fn bias(&self) -> &Array0 { +// self.node.bias() +// } + +// fn bias_mut(&mut self) -> &mut Array0 { +// self.node.bias_mut() +// } + +// fn set_bias(&mut self, bias: Array0) { +// self.node.set_bias(bias); +// } +// } + +impl Weighted for Perceptron +where + T: Float, + A: Activate, +{ + fn weights(&self) -> &Array1 { + self.node.weights() + } + + fn weights_mut(&mut self) -> &mut Array1 { + self.node.weights_mut() + } + + fn set_weights(&mut self, weights: Array1) { + self.node.set_weights(weights); + } +} + +impl Parameterized for Perceptron +where + A: Activate, + T: Float, +{ + type Features = usize; + + type Params = Node; + + fn features(&self) -> &Self::Features { + self.node.features() + } + + fn features_mut(&mut self) -> &mut Self::Features { + self.node.features_mut() + } + + fn params(&self) -> &Self::Params { + &self.node + } + + fn params_mut(&mut self) -> &mut Self::Params { + &mut self.node + } +} + +// impl Forward> for Neuron { +// type Output = f64; + +// fn forward(&self, args: &Array1) -> Self::Output { +// self.rho().activate(args.dot(&self.weights().t().to_owned()) + self.bias) +// } + +// } + +impl Forward> for Perceptron +where + T: NdFloat, + A: Activate, { type Output = Array1; fn forward(&self, args: &Array2) -> Self::Output { - self.node.forward(args) + let linstep = args.dot(&self.node().weights().t()) + self.bias(); + self.rho().activate(&linstep) + } +} + +impl From<(Array1, Array0)> for Perceptron +where + T: Float, + A: Activate + Default, +{ + fn from((weights, bias): (Array1, Array0)) -> Self { + Self { + activation: A::default(), + node: Node::from((weights, bias)), + } + } +} + +impl From<(Array1, T)> for Perceptron +where + T: NdFloat, + A: Activate + Default, +{ + fn from((weights, bias): (Array1, T)) -> Self { + Self { + activation: A::default(), + node: Node::from((weights, bias)), + } + } +} + +impl From<(Array1, Array0, A)> for Perceptron +where + T: Float, + A: Activate, +{ + fn from((weights, bias, activation): (Array1, Array0, A)) -> Self { + Self { + activation, + node: Node::from((weights, bias)), + } + } +} + +impl From<(Array1, T, A)> for Perceptron +where + T: NdFloat, + A: Activate, +{ + fn from((weights, bias, activation): (Array1, T, A)) -> Self { + Self { + activation, + node: Node::from((weights, bias)), + } + } +} + +impl From> for (Array1, Array0) +where + T: Float, + A: Activate, +{ + fn from(neuron: Perceptron) -> Self { + neuron.node().clone().into() } } diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs index 3e8ad074..94ce579e 100644 --- a/ml/optim/src/grad/gradient.rs +++ b/ml/optim/src/grad/gradient.rs @@ -17,6 +17,7 @@ where gamma: T, params: ModelParams, objective: O, + } impl Grad From 02d8c5fe4baa2c6af882d448a1d08b5f16c6d11c Mon Sep 17 00:00:00 2001 From: FL03 Date: Mon, 4 Dec 2023 22:44:48 -0600 Subject: [PATCH 079/118] Update model Signed-off-by: FL03 --- concision/examples/gradients.rs | 28 +++++++++---------- ml/neural/src/models/model.rs | 48 ++++++++++++++++++++++++--------- ml/neural/src/models/params.rs | 24 ++++++++++------- ml/neural/src/neurons/node.rs | 1 - ml/optim/src/grad/descent.rs | 9 +++++-- ml/optim/src/grad/gradient.rs | 1 - 6 files changed, 72 insertions(+), 39 deletions(-) diff --git a/concision/examples/gradients.rs b/concision/examples/gradients.rs index 8ee604e3..40a1b387 100644 --- a/concision/examples/gradients.rs +++ b/concision/examples/gradients.rs @@ -1,6 +1,6 @@ use concision::neural::models::{Model, ModelConfig, ModelParams}; use concision::neural::prelude::{Layer, Sigmoid}; -use concision::optim::grad::*; +// use concision::optim::grad::*; use concision::prelude::{linarr, Features, Forward, LayerShape}; use ndarray::prelude::{Array1, Ix2}; @@ -11,7 +11,7 @@ fn main() -> anyhow::Result<()> { let features = LayerShape::new(inputs, outputs); - let (epochs, gamma) = (1000, 0.005); + let (epochs, gamma) = (100000, 0.0005); // sample_gradient(epochs, features, gamma, samples)?; @@ -55,6 +55,8 @@ pub fn sample_model( gamma: f64, samples: usize, ) -> anyhow::Result<()> { + let mut losses = Array1::zeros(epochs); + // Generate some example data let x = linarr::((samples, features.inputs()))?; let y = linarr::((samples, features.outputs()))?; @@ -64,24 +66,22 @@ pub fn sample_model( let config = ModelConfig::new(4); let params = ModelParams::::from_iter(shapes); - let mut model = Model::::new(config).with_params(params); + let mut model = Model::::new(config).with_params(params).init(false); // let mut opt = Grad::new(gamma, model.clone(), Sigmoid); - // println!( - // "Targets (dim):\t{:?}\nPredictions:\n\n{:?}\n", - // &y.shape(), - // model.forward(&x) - // ); + println!( + "Targets (dim):\t{:?}\nPredictions:\n\n{:?}\n", + &y.shape(), + model.forward(&x) + ); - // let mut losses = Array1::zeros(epochs); for e in 0..epochs { - let _cost = model.gradient(&x, &y, gamma, Sigmoid)?; + let cost = model.gradient(&x, &y, gamma, Sigmoid)?; // let cost = opt.step(&x, &y)?; // let cost = model.grad(gamma, &x, &y); - // losses[e] = cost; + losses[e] = cost; } - // model = opt.model().clone(); - // println!("Losses:\n\n{:?}\n", &losses); - // println!("Trained:\n\n{:?}", model.forward(&x)); + println!("Losses:\n\n{:?}\n", &losses); + println!("Trained:\n\n{:?}", model.forward(&x)); Ok(()) } diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs index ca4bee66..ec14dc77 100644 --- a/ml/neural/src/models/model.rs +++ b/ml/neural/src/models/model.rs @@ -4,8 +4,10 @@ */ use super::{ModelConfig, ModelParams}; use crate::prelude::{Forward, Gradient, LayerParams, Weighted}; -use ndarray::prelude::{Array1, Array2, NdFloat}; -use num::Float; +use ndarray::prelude::{Array2, NdFloat}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_stats::DeviationExt; +use num::{Float, Signed}; #[derive(Clone, Debug)] pub struct Model @@ -51,33 +53,55 @@ where impl Model where - T: NdFloat, + T: NdFloat + Signed, { - pub fn gradient(&mut self, data: &Array2, targets: &Array2, gamma: T, grad: impl Gradient) -> anyhow::Result<()> { - let mut grads = Vec::new(); - // let mut loss = Array1::zeros(self.params.len()); + pub fn gradient( + &mut self, + data: &Array2, + targets: &Array2, + gamma: T, + grad: impl Gradient, + ) -> anyhow::Result { + let mut grads = Vec::with_capacity(self.params().len()); + let mut store = vec![data.clone()]; for layer in self.clone().into_iter() { let pred = layer.forward(&store.last().unwrap()); store.push(pred); } - - let error = targets - store.last().unwrap(); + + let error = store.last().unwrap() - targets; let dz = &error * grad.gradient(&error); grads.push(dz.clone()); for i in (1..self.params.len()).rev() { let wt = self.params[i].weights().t(); - let dz = &dz.dot(&wt) * grad.gradient(&store[i]); - grads[i - 1] = dz.clone(); + let delta = grads.last().unwrap().dot(&wt); + let dp = grad.gradient(&store[i]); + let gradient = delta * &dp; + grads.push(gradient); } + grads.reverse(); for i in 0..self.params.len() { let gradient = &store[i].t().dot(&grads[i]); - self.params[i].weights_mut().scaled_add(-gamma, &gradient.t()); + self.params[i] + .weights_mut() + .scaled_add(-gamma, &gradient.t()); } - Ok(()) + let loss = self.forward(data).mean_sq_err(targets)?; + Ok(loss) + } +} + +impl Model +where + T: Float + SampleUniform, +{ + pub fn init(mut self, biased: bool) -> Self { + self.params = self.params.init(biased); + self } } diff --git a/ml/neural/src/models/params.rs b/ml/neural/src/models/params.rs index 2bf2b6b1..ab7b16eb 100644 --- a/ml/neural/src/models/params.rs +++ b/ml/neural/src/models/params.rs @@ -127,21 +127,29 @@ impl ModelParams where T: Float + SampleUniform, { - pub fn init_layers(mut self, biased: bool) -> Self { + pub fn init(mut self, biased: bool) -> Self { self.children .iter_mut() .for_each(|l| *l = l.clone().init(biased)); self } -} -impl ModelParams -where - T: NdFloat, -{ + pub fn init_bias(mut self) -> Self { + self.children + .iter_mut() + .for_each(|l| *l = l.clone().init_bias()); + self + } + pub fn init_weight(mut self) -> Self { + self.children + .iter_mut() + .for_each(|l| *l = l.clone().init_weight()); + self + } } +impl ModelParams where T: NdFloat {} impl AsRef<[LayerParams]> for ModelParams where @@ -332,9 +340,7 @@ mod tests { let shapes = [(inputs, outputs), (outputs, outputs), (outputs, 1)]; - let params = ModelParams::::new() - .build_layers(shapes) - .init_layers(true); + let params = ModelParams::::new().build_layers(shapes).init(true); // validate the dimensions of the model params assert!(params.validate_shapes()); diff --git a/ml/neural/src/neurons/node.rs b/ml/neural/src/neurons/node.rs index fb9fdb76..3c001828 100644 --- a/ml/neural/src/neurons/node.rs +++ b/ml/neural/src/neurons/node.rs @@ -31,7 +31,6 @@ where } } - pub fn features(&self) -> &usize { &self.features } diff --git a/ml/optim/src/grad/descent.rs b/ml/optim/src/grad/descent.rs index a9ff2cdd..af6ce20b 100644 --- a/ml/optim/src/grad/descent.rs +++ b/ml/optim/src/grad/descent.rs @@ -12,8 +12,9 @@ pub struct GradientDescent where T: Float, { - pub gamma: T, + gamma: T, model: Layer, + store: Vec>, } impl GradientDescent @@ -21,7 +22,11 @@ where T: Float, { pub fn new(gamma: T, model: Layer) -> Self { - Self { gamma, model } + Self { + gamma, + model, + store: Vec::new(), + } } pub fn gamma(&self) -> T { diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs index 94ce579e..3e8ad074 100644 --- a/ml/optim/src/grad/gradient.rs +++ b/ml/optim/src/grad/gradient.rs @@ -17,7 +17,6 @@ where gamma: T, params: ModelParams, objective: O, - } impl Grad From c6e7a5ba474ed4bdae3baa8ea15b3d78a9592e03 Mon Sep 17 00:00:00 2001 From: FL03 Date: Thu, 7 Dec 2023 10:01:17 -0600 Subject: [PATCH 080/118] update Signed-off-by: FL03 --- ml/neural/src/layers/cmp/kinds.rs | 24 ++++++ ml/neural/src/models/exp/mod.rs | 3 +- ml/neural/src/models/exp/store.rs | 134 ++++++++++++++++++++++++++++++ ml/neural/src/models/model.rs | 26 ++++-- ml/neural/src/models/params.rs | 54 +++++------- ml/optim/src/grad/gradient.rs | 86 ++++++++++++++----- 6 files changed, 260 insertions(+), 67 deletions(-) create mode 100644 ml/neural/src/models/exp/store.rs diff --git a/ml/neural/src/layers/cmp/kinds.rs b/ml/neural/src/layers/cmp/kinds.rs index 1031669d..703c429b 100644 --- a/ml/neural/src/layers/cmp/kinds.rs +++ b/ml/neural/src/layers/cmp/kinds.rs @@ -33,6 +33,30 @@ pub enum LayerKind { Output, } +impl LayerKind { + pub fn input() -> Self { + Self::Input + } + + pub fn hidden() -> Self { + Self::Hidden + } + + pub fn output() -> Self { + Self::Output + } + + pub fn create_kind(idx: usize, layers: usize) -> Self { + if idx == 0 { + Self::Input + } else if idx == layers - 1 { + Self::Output + } else { + Self::Hidden + } + } +} + #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] diff --git a/ml/neural/src/models/exp/mod.rs b/ml/neural/src/models/exp/mod.rs index 348e9548..2d02f8ad 100644 --- a/ml/neural/src/models/exp/mod.rs +++ b/ml/neural/src/models/exp/mod.rs @@ -3,9 +3,10 @@ Contrib: FL03 */ //! # Experimental Models -pub use self::{modules::*, utils::*}; +pub use self::{modules::*, store::*, utils::*}; pub(crate) mod modules; +pub(crate) mod store; pub(crate) mod utils {} diff --git a/ml/neural/src/models/exp/store.rs b/ml/neural/src/models/exp/store.rs new file mode 100644 index 00000000..392f3311 --- /dev/null +++ b/ml/neural/src/models/exp/store.rs @@ -0,0 +1,134 @@ +/* + Appellation: stack + Contrib: FL03 +*/ +use crate::prelude::{LayerParams, LayerPosition, LayerShape}; +use ndarray::prelude::{Dimension, Ix2}; +use ndarray::IntoDimension; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use num::Float; + +// use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +// use std::ops; + +pub struct ModelMap +where + T: Float, +{ + store: HashMap>, +} + +impl ModelMap +where + T: Float, +{ + pub fn new() -> Self { + Self { + store: HashMap::new(), + } + } + + pub fn with_capacity(cap: usize) -> Self { + Self { + store: HashMap::with_capacity(cap), + } + } + + pub fn build_layers(mut self, shapes: impl IntoIterator) -> Self { + // let shapes = shapes.into_iter().map(|s| (s.inputs(), s.outputs())); + let tmp = Vec::from_iter(shapes.into_iter().map(|(i, o)| LayerShape::new(i, o))); + for (i, features) in tmp.iter().enumerate() { + let position = if i == 0 { + LayerPosition::input() + } else if i == tmp.len() - 1 { + LayerPosition::output(i) + } else { + LayerPosition::hidden(i) + }; + self.store.insert(position, LayerParams::new(*features)); + } + self + } + + pub fn with_shapes(shapes: impl IntoIterator) -> Self + where + Sh: IntoDimension, + { + let tmp = Vec::from_iter(shapes.into_iter().map(IntoDimension::into_dimension)); + let mut store = HashMap::new(); + for (i, (inputs, outputs)) in tmp.iter().map(|s| s.into_pattern()).enumerate() { + let features = LayerShape::new(inputs, outputs); + let position = if i == 0 { + LayerPosition::input() + } else if i == tmp.len() - 1 { + LayerPosition::output(i) + } else { + LayerPosition::hidden(i) + }; + store.insert(position, LayerParams::new(features)); + } + Self { store } + } +} + +impl ModelMap +where + T: Float + SampleUniform, +{ + pub fn init(mut self, biased: bool) -> Self { + self.store.iter_mut().for_each(|(_, l)| { + *l = l.clone().init(biased); + }); + self + } + + pub fn init_bias(mut self) -> Self { + self.store + .iter_mut() + .for_each(|(_, l)| *l = l.clone().init_bias()); + self + } + + pub fn init_weight(mut self) -> Self { + self.store + .iter_mut() + .for_each(|(_, l)| *l = l.clone().init_weight()); + self + } +} + +impl IntoIterator for ModelMap +where + T: Float, +{ + type Item = (LayerPosition, LayerParams); + type IntoIter = std::collections::hash_map::IntoIter>; + + fn into_iter(self) -> Self::IntoIter { + self.store.into_iter() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_model_store() { + let (inputs, outputs) = (5, 3); + + let shapes = [(inputs, outputs), (outputs, outputs), (outputs, 1)]; + + let params = ModelMap::::new().build_layers(shapes).init(true); + + // validate the dimensions of the model params + // assert!(params.validate_shapes()); + + for (pos, layer) in params.into_iter() { + let shape = shapes[pos.index()]; + let features = LayerShape::new(shape.0, shape.1); + assert_eq!(layer.features(), &features); + } + } +} diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs index ec14dc77..534c63a4 100644 --- a/ml/neural/src/models/model.rs +++ b/ml/neural/src/models/model.rs @@ -62,29 +62,39 @@ where gamma: T, grad: impl Gradient, ) -> anyhow::Result { + // the number of layers in the model + let depth = self.params().len(); + // the gradients for each layer let mut grads = Vec::with_capacity(self.params().len()); - + // a store for the predictions of each layer let mut store = vec![data.clone()]; - + // compute the predictions for each layer for layer in self.clone().into_iter() { let pred = layer.forward(&store.last().unwrap()); store.push(pred); } - + // compute the error for the last layer let error = store.last().unwrap() - targets; + // compute the error gradient for the last layer let dz = &error * grad.gradient(&error); + // push the error gradient for the last layer grads.push(dz.clone()); - for i in (1..self.params.len()).rev() { + for i in (1..depth).rev() { + // get the weights for the current layer let wt = self.params[i].weights().t(); - let delta = grads.last().unwrap().dot(&wt); + // compute the delta for the current layer w.r.t. the previous layer + let dw = grads.last().unwrap().dot(&wt); + // compute the gradient w.r.t. the current layer's predictions let dp = grad.gradient(&store[i]); - let gradient = delta * &dp; + // compute the gradient for the current layer + let gradient = dw * &dp; grads.push(gradient); } + // reverse the gradients so that they are in the correct order grads.reverse(); - - for i in 0..self.params.len() { + // update the parameters for each layer + for i in 0..depth { let gradient = &store[i].t().dot(&grads[i]); self.params[i] .weights_mut() diff --git a/ml/neural/src/models/params.rs b/ml/neural/src/models/params.rs index ab7b16eb..bf051167 100644 --- a/ml/neural/src/models/params.rs +++ b/ml/neural/src/models/params.rs @@ -2,48 +2,15 @@ Appellation: stack Contrib: FL03 */ -use crate::prelude::{Features, LayerParams, LayerPosition, LayerShape}; -use ndarray::prelude::{Dimension, Ix2, NdFloat}; +use crate::prelude::{Features, Forward, LayerParams, LayerShape}; +use ndarray::prelude::{Array2, Dimension, Ix2, NdFloat}; use ndarray::IntoDimension; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; use std::ops; -pub struct ModelMap -where - T: Float, -{ - store: HashMap>, -} - -impl ModelMap -where - T: Float, -{ - pub fn with_shapes(shapes: impl IntoIterator) -> Self - where - Sh: IntoDimension, - { - let tmp = Vec::from_iter(shapes.into_iter().map(IntoDimension::into_dimension)); - let mut store = HashMap::new(); - for (i, (inputs, outputs)) in tmp.iter().map(|s| s.into_pattern()).enumerate() { - let features = LayerShape::new(inputs, outputs); - let position = if i == 0 { - LayerPosition::input() - } else if i == tmp.len() - 1 { - LayerPosition::output(i) - } else { - LayerPosition::hidden(i) - }; - store.insert(position, LayerParams::new(features)); - } - Self { store } - } -} - #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] pub struct ModelParams where @@ -151,6 +118,23 @@ where impl ModelParams where T: NdFloat {} +impl Forward> for ModelParams +where + T: NdFloat, +{ + type Output = Array2; + + fn forward(&self, input: &Array2) -> Array2 { + let mut iter = self.children.iter(); + + let mut output = iter.next().unwrap().forward(input); + for layer in iter { + output = layer.forward(&output); + } + output + } +} + impl AsRef<[LayerParams]> for ModelParams where T: Float, diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs index 3e8ad074..01a95afb 100644 --- a/ml/optim/src/grad/gradient.rs +++ b/ml/optim/src/grad/gradient.rs @@ -4,7 +4,7 @@ */ use crate::neural::func::activate::Sigmoid; use crate::neural::models::ModelParams; -use crate::neural::prelude::{Forward, Gradient, LayerParams}; +use crate::neural::prelude::{Forward, Gradient, Weighted}; use ndarray::prelude::{Array2, NdFloat}; use ndarray_stats::DeviationExt; use num::{Float, Signed}; @@ -58,28 +58,46 @@ where O: Gradient, T: NdFloat + Signed, { - pub fn step(&mut self, data: &Array2, targets: &Array2) -> anyhow::Result { - let ns = T::from(data.shape()[0]).unwrap(); + pub fn gradient(&mut self, data: &Array2, targets: &Array2) -> anyhow::Result { + let lr = self.gamma(); + // the number of layers in the model let depth = self.model().len(); - - let mut cost = T::zero(); - let params = self.params.clone(); - - for (i, layer) in self.params[..(depth - 1)].iter_mut().enumerate() { - // compute the prediction of the model - let pred = params[i + 1].forward(data); - // compute the error of the prediction - let errors = &pred - targets; - // compute the gradient of the objective function w.r.t. the error's - let dz = errors * self.objective.gradient(&pred); - // compute the gradient of the objective function w.r.t. the model's weights - let dw = data.t().dot(&dz) / ns; - layer.update_with_gradient(self.gamma, &dw.t().to_owned()); - let loss = targets.mean_sq_err(&pred)?; - cost += T::from(loss).unwrap(); + // the gradients for each layer + let mut grads = Vec::with_capacity(self.model().len()); + // a store for the predictions of each layer + let mut store = vec![data.clone()]; + // compute the predictions for each layer + for layer in self.model().clone().into_iter() { + let pred = layer.forward(&store.last().unwrap()); + store.push(pred); } - - Ok(cost) + // compute the error for the last layer + let error = store.last().unwrap() - targets; + // compute the error gradient for the last layer + let dz = &error * self.objective.gradient(&error); + // push the error gradient for the last layer + grads.push(dz.clone()); + + for i in (1..depth).rev() { + // get the weights for the current layer + let wt = self.params[i].weights().t(); + // compute the delta for the current layer w.r.t. the previous layer + let dw = grads.last().unwrap().dot(&wt); + // compute the gradient w.r.t. the current layer's predictions + let dp = self.objective.gradient(&store[i]); + // compute the gradient for the current layer + let gradient = dw * &dp; + grads.push(gradient); + } + // reverse the gradients so that they are in the correct order + grads.reverse(); + // update the parameters for each layer + for i in 0..depth { + let gradient = &store[i].t().dot(&grads[i]); + self.params[i].weights_mut().scaled_add(-lr, &gradient.t()); + } + let loss = self.model().forward(data).mean_sq_err(targets)?; + Ok(loss) } } @@ -88,10 +106,19 @@ mod tests { use super::*; use crate::core::prelude::linarr; use crate::neural::models::ModelParams; - use crate::neural::prelude::{Features, LayerShape}; + use crate::neural::prelude::{Features, LayerShape, Sigmoid}; use ndarray::prelude::Ix2; + pub fn assert_ok(result: Result) -> T + where + E: std::fmt::Debug, + T: std::fmt::Debug, + { + assert!(result.is_ok(), "{:?}", result); + result.unwrap() + } + #[test] fn test_gradient() { let (samples, inputs) = (20, 5); @@ -107,6 +134,19 @@ mod tests { let mut shapes = vec![features]; shapes.extend((0..3).map(|_| LayerShape::new(features.outputs(), features.outputs()))); - let mut model = ModelParams::::from_iter(shapes); + let mut model = ModelParams::::from_iter(shapes).init(true); + + let mut grad = Grad::new(0.01, model.clone(), Sigmoid); + + let mut losses = Vec::new(); + + for _epoch in 0..3 { + let loss = assert_ok(grad.gradient(&x, &y)); + losses.push(loss); + } + + model = grad.model().clone(); + + assert!(losses.first().unwrap() > losses.last().unwrap()); } } From 45337cc83161e12e9756f43fdf12e4d37b8ede82 Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 8 Dec 2023 11:22:40 -0600 Subject: [PATCH 081/118] update Signed-off-by: FL03 --- ml/neural/src/func/prop/mod.rs | 3 +- ml/neural/src/func/prop/results.rs | 13 ++++ ml/neural/src/layers/exp/layer.rs | 99 ++++++++++++++++-------------- ml/neural/src/models/model.rs | 23 ++++--- ml/neural/src/neurons/node.rs | 50 ++++++++++----- ml/neural/src/params/group.rs | 66 ++++++++++++++++++-- ml/neural/src/specs.rs | 13 ++++ ml/optim/src/grad/gradient.rs | 11 ++-- ml/optim/src/grad/mod.rs | 67 +++++++------------- 9 files changed, 218 insertions(+), 127 deletions(-) create mode 100644 ml/neural/src/func/prop/results.rs diff --git a/ml/neural/src/func/prop/mod.rs b/ml/neural/src/func/prop/mod.rs index 1896d22a..4510df4a 100644 --- a/ml/neural/src/func/prop/mod.rs +++ b/ml/neural/src/func/prop/mod.rs @@ -5,9 +5,10 @@ //! # Propagation //! //! This module describes the propagation of data through a neural network. -pub use self::{modes::*, utils::*}; +pub use self::{modes::*, results::*, utils::*}; pub(crate) mod modes; +pub(crate) mod results; pub(crate) mod utils {} diff --git a/ml/neural/src/func/prop/results.rs b/ml/neural/src/func/prop/results.rs new file mode 100644 index 00000000..f71109cb --- /dev/null +++ b/ml/neural/src/func/prop/results.rs @@ -0,0 +1,13 @@ +/* + Appellation: results + Contrib: FL03 +*/ + +pub enum PropogationResult { + Forward(ForwardResult), + Backward(String), +} + +pub enum ForwardResult { + Forward(T), +} diff --git a/ml/neural/src/layers/exp/layer.rs b/ml/neural/src/layers/exp/layer.rs index 09d04941..a6f20dc5 100644 --- a/ml/neural/src/layers/exp/layer.rs +++ b/ml/neural/src/layers/exp/layer.rs @@ -5,19 +5,20 @@ use super::LayerConfig; use crate::func::activate::{Activate, Activator, Linear}; use crate::layers::{LayerParams, LayerShape}; -use crate::prelude::{Features, Forward, Node, Parameterized, Params, Perceptron}; -use ndarray::prelude::{Array2, Ix1, NdFloat}; +use crate::prelude::{Features, Forward, Node, ParamGroup, Parameterized, Params}; +use ndarray::prelude::{Array2, Dimension, Ix1, Ix2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; -pub struct Layer +pub struct Layer where + D: Dimension, T: Float, { - activator: Activator, + activator: Activator, config: LayerConfig, - params: LayerParams, + params: ParamGroup, } impl Layer @@ -25,7 +26,7 @@ where T: Float, { pub fn new(activator: impl Activate + 'static, config: LayerConfig) -> Self { - let params = LayerParams::new(*config.features()); + let params = ParamGroup::new(*config.features()); Self { activator: Activator::new(Box::new(activator)), config, @@ -45,16 +46,19 @@ where &mut self.config } - pub fn set_node(&mut self, idx: usize, node: &Node) { - self.params.set_node(idx, node.clone()); - } + // pub fn set_node(&mut self, idx: usize, node: &Node) { + // self.params + // .weights_mut() + // .slice_mut(s![idx, ..]) + // .assign(&node.weights()); + // } - pub fn validate_layer(&self, other: &Self, next: bool) -> bool { - if next { - return self.features().inputs() == other.features().outputs(); - } - self.features().outputs() == other.features().inputs() - } + // pub fn validate_layer(&self, other: &Self, next: bool) -> bool { + // if next { + // return self.features().inputs() == other.features().outputs(); + // } + // self.features().outputs() == other.features().inputs() + // } } impl Layer @@ -104,26 +108,27 @@ where } } -impl Parameterized for Layer +impl Parameterized for Layer where + D: Dimension, T: Float, { - type Features = LayerShape; - type Params = LayerParams; + type Features = D; + type Params = ParamGroup; - fn features(&self) -> &LayerShape { - self.config().features() + fn features(&self) -> &D { + self.params().features() } - fn features_mut(&mut self) -> &mut LayerShape { - self.config_mut().features_mut() + fn features_mut(&mut self) -> &mut D { + self.params_mut().features_mut() } - fn params(&self) -> &LayerParams { + fn params(&self) -> &Self::Params { &self.params } - fn params_mut(&mut self) -> &mut LayerParams { + fn params_mut(&mut self) -> &mut Self::Params { &mut self.params } } @@ -147,28 +152,28 @@ where } } -impl IntoIterator for Layer -where - T: Float, -{ - type Item = Node; - type IntoIter = std::vec::IntoIter; +// impl IntoIterator for Layer +// where +// T: Float, +// { +// type Item = Node; +// type IntoIter = std::vec::IntoIter; - fn into_iter(self) -> Self::IntoIter { - self.params.into_iter() - } -} +// fn into_iter(self) -> Self::IntoIter { +// self.params.into_iter() +// } +// } -impl FromIterator> for Layer -where - T: Float, -{ - fn from_iter>>(nodes: I) -> Self { - let params = LayerParams::from_iter(nodes); - Self { - activator: Activator::linear(), - config: LayerConfig::from(*params.features()), - params, - } - } -} +// impl FromIterator> for Layer +// where +// T: Float, +// { +// fn from_iter>>(nodes: I) -> Self { +// let params = LayerParams::from_iter(nodes); +// Self { +// activator: Activator::linear(), +// config: LayerConfig::from(*params.features()), +// params, +// } +// } +// } diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs index 534c63a4..636e719a 100644 --- a/ml/neural/src/models/model.rs +++ b/ml/neural/src/models/model.rs @@ -4,10 +4,12 @@ */ use super::{ModelConfig, ModelParams}; use crate::prelude::{Forward, Gradient, LayerParams, Weighted}; -use ndarray::prelude::{Array2, NdFloat}; +use ndarray::linalg::Dot; +use ndarray::prelude::{Array, Array1, Array2, Dimension, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_stats::DeviationExt; use num::{Float, Signed}; +use std::ops; #[derive(Clone, Debug)] pub struct Model @@ -155,19 +157,20 @@ where } } -impl Forward> for Model +impl Forward> for Model where + D: Dimension, T: NdFloat, + Array: Dot, Output = Array> + ops::Add, Output = Array>, { - type Output = Array2; + type Output = Array; - fn forward(&self, input: &Array2) -> Array2 { - let mut iter = self.clone().into_iter(); - - let mut output = iter.next().unwrap().forward(input); - for layer in iter { - output = layer.forward(&output); + fn forward(&self, input: &Array) -> Self::Output { + let mut store = vec![input.clone()]; + for layer in self.clone().into_iter() { + let pred = layer.forward(&store.last().unwrap()); + store.push(pred); } - output + store.last().unwrap().clone() } } diff --git a/ml/neural/src/neurons/node.rs b/ml/neural/src/neurons/node.rs index 3c001828..ee9d451c 100644 --- a/ml/neural/src/neurons/node.rs +++ b/ml/neural/src/neurons/node.rs @@ -5,9 +5,12 @@ use crate::core::prelude::GenerateRandom; use crate::prelude::{Biased, Forward, Weighted}; -use ndarray::prelude::{Array0, Array1, Array2, Ix1, NdFloat}; +use ndarray::linalg::Dot; +use ndarray::prelude::{Array, Array0, Array1, Array2, Dimension, Ix1, NdFloat}; +use ndarray::RemoveAxis; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::{Float, FromPrimitive}; +use std::ops; #[derive(Clone, Debug, PartialEq)] pub struct Node @@ -114,29 +117,44 @@ where } } -impl Forward> for Node +impl Forward> for Node where Self: Biased + Weighted, + D: Dimension + RemoveAxis, T: FromPrimitive + NdFloat, + Array: Dot, Output = Array>, + Array: ops::Add, Output = Array>, { - type Output = T; + type Output = Array; - fn forward(&self, data: &Array1) -> Self::Output { - data.dot(&self.weights().t()) + self.bias().first().unwrap().clone() + fn forward(&self, data: &Array) -> Self::Output { + data.dot(&self.weights().t().to_owned()) + self.bias().clone() } } -impl Forward> for Node -where - Self: Biased + Weighted, - T: FromPrimitive + NdFloat, -{ - type Output = Array1; - - fn forward(&self, data: &Array2) -> Self::Output { - data.dot(&self.weights().t()) + self.bias() - } -} +// impl Forward> for Node +// where +// Self: Biased + Weighted, +// T: FromPrimitive + NdFloat, +// { +// type Output = T; + +// fn forward(&self, data: &Array1) -> Self::Output { +// data.dot(&self.weights().t()) + self.bias().first().unwrap().clone() +// } +// } + +// impl Forward> for Node +// where +// Self: Biased + Weighted, +// T: FromPrimitive + NdFloat, +// { +// type Output = Array1; + +// fn forward(&self, data: &Array2) -> Self::Output { +// data.dot(&self.weights().t()) + self.bias().clone() +// } +// } impl Biased for Node where diff --git a/ml/neural/src/params/group.rs b/ml/neural/src/params/group.rs index d1304421..e59e0471 100644 --- a/ml/neural/src/params/group.rs +++ b/ml/neural/src/params/group.rs @@ -5,7 +5,7 @@ use super::{Biased, Weighted}; use crate::core::prelude::GenerateRandom; use crate::prelude::{Forward, Node}; -use ndarray::prelude::{Array, Axis, Dimension, Ix1, Ix2}; +use ndarray::prelude::{s, Array, Axis, Dimension, Ix1, Ix2}; use ndarray::{IntoDimension, RemoveAxis}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; @@ -46,6 +46,10 @@ where &self.features } + pub fn features_mut(&mut self) -> &mut D { + &mut self.features + } + pub fn inputs(&self) -> usize { self.weights.shape().last().unwrap().clone() } @@ -56,6 +60,15 @@ where } self.weights.shape().first().unwrap().clone() } + + // pub fn set_node(&mut self, idx: usize, node: ParamGroup) { + // let dim = self.features(); + + // self.weights + // .slice_mut(s![idx, ..]) + // .assign(&node.weights); + // self.bias.slice_mut(s![idx, ..]).assign(&node.bias); + // } } impl ParamGroup @@ -180,19 +193,64 @@ where } } -impl IntoIterator for ParamGroup +// impl IntoIterator for ParamGroup +// where +// T: Float, +// { +// type Item = Node; +// type IntoIter = std::vec::IntoIter; + +// fn into_iter(self) -> Self::IntoIter { +// self.weights() +// .axis_iter(Axis(0)) +// .zip(self.bias().axis_iter(Axis(0))) +// .map(|(w, b)| (w.to_owned(), b.to_owned()).into()) +// .collect::>() +// .into_iter() +// } +// } + +impl IntoIterator for ParamGroup where + D: Dimension + RemoveAxis, T: Float, + ::Smaller: Dimension + RemoveAxis, { - type Item = Node; + type Item = ( + Array, + Array::Smaller>, + ); type IntoIter = std::vec::IntoIter; fn into_iter(self) -> Self::IntoIter { self.weights() .axis_iter(Axis(0)) .zip(self.bias().axis_iter(Axis(0))) - .map(|(w, b)| (w.to_owned(), b.to_owned()).into()) + .map(|(w, b)| (w.to_owned(), b.to_owned())) .collect::>() .into_iter() } } + +// impl FromIterator> for ParamGroup +// where +// T: Float, +// D: Dimension + RemoveAxis, +// ::Smaller: Dimension + RemoveAxis, +// { +// fn from_iter(iter: I) -> Self +// where +// I: IntoIterator>, +// { +// let store = Vec::from_iter(iter); +// let mut features = vec![store.len()]; +// features.extend(store.first().unwrap().weights().shape()); + +// let mut group = Self::new(features.as_slice()); +// let mut iter = iter.into_iter(); +// let mut group = Self::new(weights.shape()); +// group.set_weights(weights); +// group.set_bias(bias); +// group +// } +// } diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index d8c3ac2f..31db06a6 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -17,6 +17,19 @@ pub trait Forward { fn forward(&self, args: &T) -> Self::Output; } +pub trait ForwardIter: Forward + IntoIterator +where + I: Forward, +{ +} + +impl ForwardIter for S +where + I: Forward, + S: Forward + IntoIterator, +{ +} + pub trait Compile where D: Dimension, diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs index 01a95afb..7ff36375 100644 --- a/ml/optim/src/grad/gradient.rs +++ b/ml/optim/src/grad/gradient.rs @@ -5,7 +5,7 @@ use crate::neural::func::activate::Sigmoid; use crate::neural::models::ModelParams; use crate::neural::prelude::{Forward, Gradient, Weighted}; -use ndarray::prelude::{Array2, NdFloat}; +use ndarray::prelude::{Array2, Axis, NdFloat}; use ndarray_stats::DeviationExt; use num::{Float, Signed}; @@ -63,7 +63,7 @@ where // the number of layers in the model let depth = self.model().len(); // the gradients for each layer - let mut grads = Vec::with_capacity(self.model().len()); + let mut grads = Vec::with_capacity(depth); // a store for the predictions of each layer let mut store = vec![data.clone()]; // compute the predictions for each layer @@ -93,8 +93,11 @@ where grads.reverse(); // update the parameters for each layer for i in 0..depth { - let gradient = &store[i].t().dot(&grads[i]); - self.params[i].weights_mut().scaled_add(-lr, &gradient.t()); + let grad = &grads[i]; + println!("Layer ({}) Gradient (dim): {:?}", i, grad.shape()); + let wg = &store[i].t().dot(grad); + let _bg = grad.sum_axis(Axis(0)); + self.params[i].weights_mut().scaled_add(-lr, &wg.t()); } let loss = self.model().forward(data).mean_sq_err(targets)?; Ok(loss) diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index 52e6a146..3eb6ae7f 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -28,12 +28,26 @@ pub struct DescentParams { pub(crate) mod utils { use crate::neural::func::activate::Gradient; - use crate::neural::prelude::{Forward, Parameterized, Params}; + use crate::neural::prelude::{Forward, ForwardIter, Parameterized, Params}; use ndarray::linalg::Dot; use ndarray::prelude::{Array, Array1, Array2, Dimension, NdFloat}; use ndarray_stats::DeviationExt; use num::{FromPrimitive, Signed}; + pub fn gradient_descent( + gamma: T, + model: &mut M, + objective: impl Gradient, + ) -> anyhow::Result + where + D: Dimension, + M: Forward, Output = Array> + Parameterized, + T: FromPrimitive + NdFloat, + { + let loss = 0.0; + Ok(loss) + } + pub fn gradient( gamma: T, model: &mut A, @@ -71,35 +85,6 @@ pub(crate) mod utils { .expect("Error when calculating the MSE of the model"); loss } - - pub fn gradient_descent( - params: &mut Array, - epochs: usize, - gamma: T, - partial: impl Fn(&Array) -> Array, - ) -> Array1 - where - D: Dimension, - T: FromPrimitive + NdFloat, - { - let mut losses = Array1::zeros(epochs); - for e in 0..epochs { - let grad = partial(params); - params.scaled_add(-gamma, &grad); - losses[e] = params.mean().unwrap_or_else(T::zero); - } - losses - } - - // pub fn gradient_descent_step( - // args: &Array2, - // layer: &mut Layer, - // gamma: T, - // partial: impl Fn(&Array2) -> Array2, - // ) -> T where A: Activate>, T: FromPrimitive + NdFloat { - // let grad = partial(args); - // layer.weights_mut().scaled_add(-gamma, &grad); - // } } #[cfg(test)] @@ -112,16 +97,8 @@ mod tests { use ndarray::prelude::{Array, Array1, Dimension}; use num::Float; - fn test_grad(args: &Array) -> Array - where - D: Dimension, - T: Float, - { - args.clone() - } - #[test] - fn descent() { + fn test_gradient_descent() { let (_samples, inputs, outputs) = (20, 5, 1); let (epochs, gamma) = (10, 0.001); @@ -130,12 +107,12 @@ mod tests { let mut model = Layer::::from(features).init(true); - let losses = gradient_descent( - &mut model.params_mut().weights_mut(), - epochs, - gamma, - test_grad, - ); + let mut losses = Array1::zeros(epochs); + for e in 0..epochs { + let cost = + gradient_descent(gamma, &mut model, Sigmoid).expect("Gradient Descent Error"); + losses[e] = cost; + } assert_eq!(losses.len(), epochs); } From a636298bea57030336f0dd8c69ae89133ce4e584 Mon Sep 17 00:00:00 2001 From: FL03 Date: Wed, 13 Dec 2023 15:38:44 -0600 Subject: [PATCH 082/118] update Signed-off-by: FL03 --- .docker/env.dockerfile | 11 +++ Cargo.toml | 2 +- concision/Cargo.toml | 6 ++ concision/src/lib.rs | 4 + core/src/specs.rs | 139 +++++++++++++++++----------------- core/src/utils.rs | 20 +---- core/tests/utils.rs | 1 - macros/Cargo.toml | 5 +- macros/src/lib.rs | 20 +++++ ml/ml/Cargo.toml | 6 ++ ml/ml/src/lib.rs | 5 +- ml/neural/src/nn/cnn/mod.rs | 17 +++++ ml/neural/src/nn/cnn/model.rs | 10 +++ ml/neural/src/nn/mod.rs | 2 + ml/neural/src/nn/rnn/mod.rs | 17 +++++ ml/neural/src/nn/rnn/model.rs | 10 +++ ml/neural/src/specs.rs | 26 ++++++- ml/optim/src/cost/mod.rs | 9 +-- ml/optim/src/grad/descent.rs | 24 +++--- ml/optim/src/grad/gradient.rs | 1 - ml/optim/src/grad/mod.rs | 39 ++++++---- ml/optim/src/optimizer.rs | 9 +-- ml/optim/src/specs.rs | 4 +- ml/s4/Cargo.toml | 52 +++++++++++++ ml/s4/benches/default.rs | 52 +++++++++++++ ml/s4/src/lib.rs | 23 ++++++ ml/s4/src/primitives.rs | 14 ++++ ml/s4/src/specs.rs | 4 + ml/s4/src/ssm/config.rs | 25 ++++++ ml/s4/src/ssm/mod.rs | 29 +++++++ ml/s4/src/ssm/model.rs | 27 +++++++ ml/s4/src/utils.rs | 4 + ml/s4/tests/default.rs | 8 ++ 33 files changed, 491 insertions(+), 134 deletions(-) create mode 100644 .docker/env.dockerfile create mode 100644 ml/neural/src/nn/cnn/mod.rs create mode 100644 ml/neural/src/nn/cnn/model.rs create mode 100644 ml/neural/src/nn/rnn/mod.rs create mode 100644 ml/neural/src/nn/rnn/model.rs create mode 100644 ml/s4/Cargo.toml create mode 100644 ml/s4/benches/default.rs create mode 100644 ml/s4/src/lib.rs create mode 100644 ml/s4/src/primitives.rs create mode 100644 ml/s4/src/specs.rs create mode 100644 ml/s4/src/ssm/config.rs create mode 100644 ml/s4/src/ssm/mod.rs create mode 100644 ml/s4/src/ssm/model.rs create mode 100644 ml/s4/src/utils.rs create mode 100644 ml/s4/tests/default.rs diff --git a/.docker/env.dockerfile b/.docker/env.dockerfile new file mode 100644 index 00000000..47cf4e5a --- /dev/null +++ b/.docker/env.dockerfile @@ -0,0 +1,11 @@ +FROM debian + +RUN apt-get update -y && apt-get upgrade -y + +RUN apt-get install -y \ + curl \ + git \ + gnupg \ + unzip \ + zip \ + && rm -rf /var/lib/apt/lists/* \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 5e2db1bd..3e5072a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ itertools = { features = [], version = "0.12" } lazy_static = "1" linfa = { features = [], version = "0.7" } ndarray = { features = ["serde-1"], version = "0.15" } -# ndarray-linalg = { features = [], version = "0.16" } +ndarray-linalg = { features = ["openblas-system"], version = "0.16" } ndarray-rand = { features = [], version = "0.14" } ndarray-stats = { features = [], version = "0.5.1" } num = { features = ["serde"], version = "0.4" } diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 4df1499a..5668c46d 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -52,6 +52,7 @@ ml = [ "neural", "nlp", "optim", + "s4", "transformers", ] @@ -67,6 +68,10 @@ optim = [ "concision-optim", ] +s4 = [ + "concision-s4" +] + transformers = [ "concision-transformers" ] @@ -90,6 +95,7 @@ concision-math = { features = [], optional = true, path = "../math", version = " concision-neural = { features = [], optional = true, path = "../ml/neural", version = "0.1.12" } concision-nlp = { features = [], optional = true, path = "../ml/nlp", version = "0.1.12" } concision-optim = { features = [], optional = true, path = "../ml/optim", version = "0.1.12" } +concision-s4 = { features = [], optional = true, path = "../ml/s4", version = "0.1.12" } concision-transformers = { features = [], optional = true, path = "../ml/transformers", version = "0.1.12" } [dev-dependencies] diff --git a/concision/src/lib.rs b/concision/src/lib.rs index aa93f127..b46a4885 100644 --- a/concision/src/lib.rs +++ b/concision/src/lib.rs @@ -25,6 +25,8 @@ pub use concision_neural as neural; pub use concision_nlp as nlp; #[cfg(feature = "optim")] pub use concision_optim as optim; +#[cfg(feature = "s4")] +pub use concision_s4 as s4; #[cfg(feature = "transformers")] pub use concision_transformers as transformers; @@ -47,6 +49,8 @@ pub mod prelude { pub use concision_nlp::prelude::*; #[cfg(feature = "optim")] pub use concision_optim::prelude::*; + #[cfg(feature = "s4")] + pub use concision_s4::prelude::*; #[cfg(feature = "transformers")] pub use concision_transformers::prelude::*; } diff --git a/core/src/specs.rs b/core/src/specs.rs index e8b73a6f..16f93860 100644 --- a/core/src/specs.rs +++ b/core/src/specs.rs @@ -2,43 +2,20 @@ Appellation: specs Contrib: FL03 */ -use ndarray::prelude::{Array, Axis, Dimension, Ix2, NdFloat}; +pub use self::math::*; + +use ndarray::prelude::{Array, Axis, Dimension, Ix2,}; use ndarray::IntoDimension; // use ndarray::linalg::Dot; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Uniform}; use ndarray_rand::RandomExt; -use num::{Float, FromPrimitive, One, Signed, Zero}; -use std::ops; - -pub trait Arithmetic: - ops::Add - + ops::Div - + ops::Mul - + ops::Sub -{ -} +use num::Float; -impl Arithmetic for A where - A: ops::Add - + ops::Div - + ops::Mul - + ops::Sub -{ +pub trait Discritize { + fn discritize(&self, step: T) -> Self; } -pub trait IntoAxis { - fn into_axis(self) -> Axis; -} - -impl IntoAxis for S -where - S: AsRef, -{ - fn into_axis(self) -> Axis { - Axis(*self.as_ref()) - } -} pub trait Apply { fn apply(&self, f: F) -> Self @@ -66,26 +43,8 @@ pub trait As: AsRef + AsMut {} impl As for S where S: AsRef + AsMut {} -pub trait BinaryNum: One + Zero {} - -impl BinaryNum for T where T: One + Zero {} - -pub trait FloatExt: FromPrimitive + NdFloat + Signed + SampleUniform {} -impl FloatExt for T where T: FromPrimitive + NdFloat + Signed + SampleUniform {} -pub trait Pair { - fn pair(&self) -> (A, B); -} - -impl Pair for T -where - T: Clone + Into<(A, B)>, -{ - fn pair(&self) -> (A, B) { - self.clone().into() - } -} pub trait GenerateRandom where @@ -118,30 +77,74 @@ where { } -pub trait MatrixOps: - Arithmetic, Array> + Sized -where - A: Dimension, - B: Dimension, -{ +pub trait IntoAxis { + fn into_axis(self) -> Axis; } -impl MatrixOps for Array +impl IntoAxis for S where - A: Dimension, - B: Dimension, - D: Dimension, - T: Float, - Self: Arithmetic, Array>, + S: AsRef, { + fn into_axis(self) -> Axis { + Axis(*self.as_ref()) + } } -impl MatrixOps for &Array -where - A: Dimension, - B: Dimension, - D: Dimension, - T: Float, - Self: Arithmetic, Array>, -{ -} +pub(crate) mod math { + use ndarray::prelude::{Array, Dimension, Ix2, NdFloat}; + use ndarray_rand::rand_distr::uniform::SampleUniform; + use num::{Float, FromPrimitive, One, Signed, Zero}; + use std::ops; + + pub trait BinaryNum: One + Zero {} + + impl BinaryNum for T where T: One + Zero {} + + pub trait FloatExt: FromPrimitive + NdFloat + Signed + SampleUniform {} + + impl FloatExt for T where T: FromPrimitive + NdFloat + Signed + SampleUniform {} + + pub trait Arithmetic: + ops::Add + + ops::Div + + ops::Mul + + ops::Sub + { + } + + impl Arithmetic for A where + A: ops::Add + + ops::Div + + ops::Mul + + ops::Sub + { + } + + pub trait MatrixOps: + Arithmetic, Array> + Sized + where + A: Dimension, + B: Dimension, + { + } + + impl MatrixOps for Array + where + A: Dimension, + B: Dimension, + D: Dimension, + T: Float, + Self: Arithmetic, Array>, + { + } + + impl MatrixOps for &Array + where + A: Dimension, + B: Dimension, + D: Dimension, + T: Float, + Self: Arithmetic, Array>, + { + } +} \ No newline at end of file diff --git a/core/src/utils.rs b/core/src/utils.rs index 72c48d45..06ac0b85 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -7,25 +7,7 @@ use ndarray::prelude::{Array, Axis, Dimension}; use ndarray::{concatenate, IntoDimension, RemoveAxis, ShapeError}; use num::Float; -#[macro_export] -macro_rules! linspace { - ( $x:expr ) => { - { - let dim = $x.into_dimension(); - let n = $dim.as_array_view().product(); - ndarray::Array::linspace(T::one(), T::from(n).unwrap(), n).into_shape(dim).unwrap() - } - }; - ( $( $x:expr ),* ) => { - { - let mut res = Vec::new(); - $( - res.push(linarr!($x)); - )* - res - } - }; -} + pub fn concat_iter(axis: usize, iter: impl IntoIterator>) -> Array where diff --git a/core/tests/utils.rs b/core/tests/utils.rs index f8bc311c..318fe9b3 100644 --- a/core/tests/utils.rs +++ b/core/tests/utils.rs @@ -7,7 +7,6 @@ use ndarray::prelude::{array, Array2}; #[test] fn test_linarr() { let args: Array2 = linarr((2, 3)).unwrap(); - // let b = linspace!((2, 3)); assert_eq!(&args, &array![[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); } diff --git a/macros/Cargo.toml b/macros/Cargo.toml index e063745a..07ce181c 100644 --- a/macros/Cargo.toml +++ b/macros/Cargo.toml @@ -16,12 +16,13 @@ default = [] [lib] bench = false +crate-type = ["cdylib", "rlib"] doctest = false -proc-macro = true test = false [dependencies] -proc-macro2 = "1.0.69" +ndarray.workspace = true +proc-macro2 = "1" quote = "1" syn = { features = ["full"], version = "2" } diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 7111079e..59dd418c 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -3,3 +3,23 @@ Contrib: FL03 */ //! # Concision Macros + +#[macro_export] +macro_rules! linspace { + ( $x:expr ) => { + { + let dim = $x.into_dimension(); + let n = $dim.as_array_view().product(); + ndarray::Array::linspace(T::one(), T::from(n).unwrap(), n).into_shape(dim).unwrap() + } + }; + ( $( $x:expr ),* ) => { + { + let mut res = Vec::new(); + $( + res.push(linarr!($x)); + )* + res + } + }; +} diff --git a/ml/ml/Cargo.toml b/ml/ml/Cargo.toml index 7a504bcc..29eb6a86 100644 --- a/ml/ml/Cargo.toml +++ b/ml/ml/Cargo.toml @@ -21,6 +21,7 @@ full = [ "neural", "nlp", "optim", + "s4", "transformers", ] @@ -36,6 +37,10 @@ optim = [ "concision-optim", ] +s4 = [ + "concision-s4" +] + transformers = [ "concision-transformers" ] @@ -52,6 +57,7 @@ test = true concision-neural = { features = [], optional = true, path = "../neural", version = "0.1.12" } concision-nlp = { features = [], optional = true, path = "../nlp", version = "0.1.12" } concision-optim = { features = [], optional = true, path = "../optim", version = "0.1.12" } +concision-s4 = { features = [], optional = true, path = "../s4", version = "0.1.12" } concision-transformers = { features = [], optional = true, path = "../transformers", version = "0.1.12" } [dev-dependencies] diff --git a/ml/ml/src/lib.rs b/ml/ml/src/lib.rs index 5e3d349a..d89f407e 100644 --- a/ml/ml/src/lib.rs +++ b/ml/ml/src/lib.rs @@ -13,17 +13,20 @@ pub use concision_neural as neural; pub use concision_nlp as nlp; #[cfg(feature = "optim")] pub use concision_optim as optim; +#[cfg(feature = "s4")] +pub use concision_s4 as s4; #[cfg(feature = "transformers")] pub use concision_transformers as transformers; pub mod prelude { - #[cfg(feature = "neural")] pub use concision_neural::prelude::*; #[cfg(feature = "nlp")] pub use concision_nlp::prelude::*; #[cfg(feature = "optim")] pub use concision_optim::prelude::*; + #[cfg(feature = "s4")] + pub use concision_s4::prelude::*; #[cfg(feature = "transformers")] pub use concision_transformers::prelude::*; } diff --git a/ml/neural/src/nn/cnn/mod.rs b/ml/neural/src/nn/cnn/mod.rs new file mode 100644 index 00000000..8d05f4ed --- /dev/null +++ b/ml/neural/src/nn/cnn/mod.rs @@ -0,0 +1,17 @@ +/* + Appellation: cnn + Contrib: FL03 +*/ +//! # Concurrent Neural Network (CNN) +//! +//! +pub use self::{model::*, utils::*}; + +pub(crate) mod model; + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests { + +} \ No newline at end of file diff --git a/ml/neural/src/nn/cnn/model.rs b/ml/neural/src/nn/cnn/model.rs new file mode 100644 index 00000000..d4d6a4e1 --- /dev/null +++ b/ml/neural/src/nn/cnn/model.rs @@ -0,0 +1,10 @@ +/* + Appellation: model + Contrib: FL03 +*/ +use ndarray::prelude::Array2; +use num::Float; + +pub struct CNN { + +} diff --git a/ml/neural/src/nn/mod.rs b/ml/neural/src/nn/mod.rs index 64df5a1d..b1366c57 100644 --- a/ml/neural/src/nn/mod.rs +++ b/ml/neural/src/nn/mod.rs @@ -9,7 +9,9 @@ pub(crate) mod kinds; pub(crate) mod position; pub(crate) mod sequential; +pub mod cnn; pub mod ffn; pub mod gnn; +pub mod rnn; pub(crate) mod utils {} diff --git a/ml/neural/src/nn/rnn/mod.rs b/ml/neural/src/nn/rnn/mod.rs new file mode 100644 index 00000000..84c9cf10 --- /dev/null +++ b/ml/neural/src/nn/rnn/mod.rs @@ -0,0 +1,17 @@ +/* + Appellation: Rnn + Contrib: FL03 +*/ +//! # Reccurrant Neural Network (RNN) +//! +//! +pub use self::{model::*, utils::*}; + +pub(crate) mod model; + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests { + +} \ No newline at end of file diff --git a/ml/neural/src/nn/rnn/model.rs b/ml/neural/src/nn/rnn/model.rs new file mode 100644 index 00000000..a2fc3e1b --- /dev/null +++ b/ml/neural/src/nn/rnn/model.rs @@ -0,0 +1,10 @@ +/* + Appellation: model + Contrib: FL03 +*/ +use ndarray::prelude::Array2; +use num::Float; + +pub struct RNN { + +} diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index 31db06a6..c56793b7 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -17,19 +17,25 @@ pub trait Forward { fn forward(&self, args: &T) -> Self::Output; } -pub trait ForwardIter: Forward + IntoIterator +pub trait ForwardIter: Forward + IntoIterator where - I: Forward, + I: Forward, { } -impl ForwardIter for S +impl ForwardIter for S where - I: Forward, S: Forward + IntoIterator, + I: Forward, { } +// impl ForwardIter for S +// where +// S: Forward + IntoIterator>, +// { +// } + pub trait Compile where D: Dimension, @@ -78,3 +84,15 @@ where Ok(res) } } + +pub trait Module: Forward, Output=Array> +where + D: Dimension, + T: Float, +{ + type Config; + + + + +} \ No newline at end of file diff --git a/ml/optim/src/cost/mod.rs b/ml/optim/src/cost/mod.rs index 5c2cbfb2..7dac866b 100644 --- a/ml/optim/src/cost/mod.rs +++ b/ml/optim/src/cost/mod.rs @@ -12,11 +12,10 @@ use ndarray::prelude::{Array, Array1}; use ndarray::Dimension; use num::Float; -pub trait Cost -where - T: Float, -{ - fn cost(&self, other: &Self) -> T; +pub trait Cost { + type Output; + + fn cost(&self, other: &Self) -> Self::Output; } pub trait CostArr diff --git a/ml/optim/src/grad/descent.rs b/ml/optim/src/grad/descent.rs index af6ce20b..45680743 100644 --- a/ml/optim/src/grad/descent.rs +++ b/ml/optim/src/grad/descent.rs @@ -2,29 +2,33 @@ Appellation: grad Contrib: FL03 */ -use crate::neural::prelude::{Forward, Gradient, Layer}; +use crate::neural::prelude::{Forward, Gradient, Layer, Sigmoid}; use ndarray::prelude::{Array2, Ix2, NdFloat}; use ndarray_stats::DeviationExt; use num::{Float, Signed}; #[derive(Clone)] -pub struct GradientDescent +pub struct GradientDescent where + O: Gradient, T: Float, { gamma: T, model: Layer, + objective: O, store: Vec>, } -impl GradientDescent +impl GradientDescent where + O: Gradient, T: Float, { - pub fn new(gamma: T, model: Layer) -> Self { + pub fn new(gamma: T, model: Layer, objective: O) -> Self { Self { gamma, model, + objective, store: Vec::new(), } } @@ -64,15 +68,15 @@ where } } -impl GradientDescent +impl GradientDescent where + O: Gradient, T: NdFloat + Signed, { pub fn gradient( &mut self, data: &Array2, targets: &Array2, - grad: impl Gradient, ) -> anyhow::Result { let lr = self.gamma(); let ns = T::from(data.shape()[0]).unwrap(); @@ -81,7 +85,7 @@ where let scale = T::from(2).unwrap() * ns; let errors = &pred - targets; - let dz = errors * grad.gradient(&pred); + let dz = errors * self.objective.gradient(&pred); let dw = data.t().dot(&dz) / scale; self.model_mut() @@ -120,16 +124,16 @@ mod tests { let features = LayerShape::new(inputs, outputs); let model = Layer::from(features).init(true); - let mut grad = GradientDescent::new(gamma, model); + let mut grad = GradientDescent::new(gamma, model, Sigmoid); let l1 = { - let tmp = grad.gradient(&x, &y, Sigmoid); + let tmp = grad.gradient(&x, &y,); assert!(tmp.is_ok()); tmp.unwrap() }; let l2 = { - let tmp = grad.gradient(&x, &y, Sigmoid); + let tmp = grad.gradient(&x, &y,); assert!(tmp.is_ok()); tmp.unwrap() }; diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs index 7ff36375..967ba7e0 100644 --- a/ml/optim/src/grad/gradient.rs +++ b/ml/optim/src/grad/gradient.rs @@ -94,7 +94,6 @@ where // update the parameters for each layer for i in 0..depth { let grad = &grads[i]; - println!("Layer ({}) Gradient (dim): {:?}", i, grad.shape()); let wg = &store[i].t().dot(grad); let _bg = grad.sum_axis(Axis(0)); self.params[i].weights_mut().scaled_add(-lr, &wg.t()); diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index 3eb6ae7f..19399f01 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -28,23 +28,28 @@ pub struct DescentParams { pub(crate) mod utils { use crate::neural::func::activate::Gradient; + use crate::neural::params::{Biased, Weighted}; use crate::neural::prelude::{Forward, ForwardIter, Parameterized, Params}; use ndarray::linalg::Dot; - use ndarray::prelude::{Array, Array1, Array2, Dimension, NdFloat}; + use ndarray::prelude::{Array, Array2, Dimension, NdFloat}; use ndarray_stats::DeviationExt; use num::{FromPrimitive, Signed}; - pub fn gradient_descent( + pub fn gradient_descent( gamma: T, model: &mut M, objective: impl Gradient, + data: &Array2, + targets: &Array, ) -> anyhow::Result where D: Dimension, - M: Forward, Output = Array> + Parameterized, - T: FromPrimitive + NdFloat, + M: Clone + ForwardIter, I, Output = Array>, + I: Forward, Output = Array> + Biased + Weighted, + T: FromPrimitive + NdFloat + Signed, + Array2: Dot, Output = Array>, { - let loss = 0.0; + let loss = model.forward(data).mean_sq_err(targets)?; Ok(loss) } @@ -93,24 +98,32 @@ mod tests { use super::*; use crate::core::prelude::linarr; use crate::neural::func::activate::{Linear, Sigmoid}; - use crate::neural::prelude::{Features, Layer, LayerShape, Parameterized, Weighted}; - use ndarray::prelude::{Array, Array1, Dimension}; - use num::Float; + use crate::neural::models::ModelParams; + use crate::neural::prelude::{Features, Layer, LayerShape}; + use ndarray::prelude::{Array1, Ix2}; #[test] fn test_gradient_descent() { - let (_samples, inputs, outputs) = (20, 5, 1); - let (epochs, gamma) = (10, 0.001); + let (samples, inputs) = (20, 5); + let outputs = 4; + + let _shape = (samples, inputs); let features = LayerShape::new(inputs, outputs); - let mut model = Layer::::from(features).init(true); + let x = linarr::((samples, features.inputs())).unwrap(); + let y = linarr::((samples, features.outputs())).unwrap(); + + let mut shapes = vec![features]; + shapes.extend((0..3).map(|_| LayerShape::new(features.outputs(), features.outputs()))); + + let mut model = ModelParams::::from_iter(shapes).init(true); let mut losses = Array1::zeros(epochs); for e in 0..epochs { - let cost = - gradient_descent(gamma, &mut model, Sigmoid).expect("Gradient Descent Error"); + let cost = gradient_descent(gamma, &mut model, Sigmoid, &x, &y) + .expect("Gradient Descent Error"); losses[e] = cost; } assert_eq!(losses.len(), epochs); diff --git a/ml/optim/src/optimizer.rs b/ml/optim/src/optimizer.rs index abc293d9..56c7f783 100644 --- a/ml/optim/src/optimizer.rs +++ b/ml/optim/src/optimizer.rs @@ -16,14 +16,9 @@ where fn name(&self) -> &str; - fn step(&mut self, data: &Array2, targets: &Array2) -> T; + fn load(&mut self, data: &Array2, targets: &Array2); - fn step_with( - &mut self, - data: &Array2, - targets: &Array2, - params: &mut Box>, - ) -> T; + fn step(&mut self, params: impl Params) -> T; } pub trait OptimizerExt: Optimizer diff --git a/ml/optim/src/specs.rs b/ml/optim/src/specs.rs index c35bd373..20ab0830 100644 --- a/ml/optim/src/specs.rs +++ b/ml/optim/src/specs.rs @@ -2,8 +2,7 @@ Appellation: specs Contrib: FL03 */ -use crate::neural::prelude::Forward; -use ndarray::prelude::{Array, Array2, Dimension, Ix2}; +use ndarray::prelude::{Array, Dimension, Ix2}; use num::Float; pub trait ApplyGradient @@ -21,3 +20,4 @@ where { fn autograd(&mut self, loss: &Array) -> Array; } + diff --git a/ml/s4/Cargo.toml b/ml/s4/Cargo.toml new file mode 100644 index 00000000..857fafb7 --- /dev/null +++ b/ml/s4/Cargo.toml @@ -0,0 +1,52 @@ +[package] +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "concision-s4" +readme.workspace = true +repository.workspace = true +version.workspace = true + +[features] +default = [] + + +[lib] +bench = true +crate-type = ["cdylib", "rlib"] +doctest = false +test = true + +[build-dependencies] + +[dependencies] +concision-core.workspace = true +concision-data = { path = "../../data" } +concision-neural = { path = "../neural" } + +anyhow.workspace = true +lazy_static.workspace = true +ndarray.workspace = true +ndarray-linalg.workspace = true +ndarray-rand.workspace = true +ndarray-stats.workspace = true +num.workspace = true +serde.workspace = true +serde_json.workspace = true +smart-default.workspace = true +strum.workspace = true + +[dev-dependencies] +approx = "0.5" + +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] + +[target.wasm32-unknown-unknown] + +[target.wasm32-wasi] diff --git a/ml/s4/benches/default.rs b/ml/s4/benches/default.rs new file mode 100644 index 00000000..937f2387 --- /dev/null +++ b/ml/s4/benches/default.rs @@ -0,0 +1,52 @@ +// bench.rs +#![feature(test)] + +extern crate test; + +use std::mem::replace; +use test::Bencher; + +// bench: find the `BENCH_SIZE` first terms of the fibonacci sequence +static BENCH_SIZE: usize = 20; + +// recursive fibonacci +fn fibonacci(n: usize) -> u32 { + if n < 2 { + 1 + } else { + fibonacci(n - 1) + fibonacci(n - 2) + } +} + +// iterative fibonacci +struct Fibonacci { + curr: u32, + next: u32, +} + +impl Iterator for Fibonacci { + type Item = u32; + fn next(&mut self) -> Option { + let new_next = self.curr + self.next; + let new_curr = replace(&mut self.next, new_next); + + Some(replace(&mut self.curr, new_curr)) + } +} + +fn fibonacci_sequence() -> Fibonacci { + Fibonacci { curr: 1, next: 1 } +} + +// function to benchmark must be annotated with `#[bench]` +#[bench] +fn recursive_fibonacci(b: &mut Bencher) { + // exact code to benchmark must be passed as a closure to the iter + // method of Bencher + b.iter(|| (0..BENCH_SIZE).map(fibonacci).collect::>()) +} + +#[bench] +fn iterative_fibonacci(b: &mut Bencher) { + b.iter(|| fibonacci_sequence().take(BENCH_SIZE).collect::>()) +} diff --git a/ml/s4/src/lib.rs b/ml/s4/src/lib.rs new file mode 100644 index 00000000..c8bd4853 --- /dev/null +++ b/ml/s4/src/lib.rs @@ -0,0 +1,23 @@ +/* + Appellation: concision-s4 + Contrib: FL03 +*/ +//! # Structured State Space Sequential Models (S4) +//! +//! +pub use self::{primitives::*, specs::*, utils::*}; + +pub(crate) mod primitives; +pub(crate) mod specs; +pub(crate) mod utils; + +pub mod ssm; + +pub(crate) use concision_core as core; +pub(crate) use concision_neural as neural; + +pub mod prelude { + pub use crate::primitives::*; + pub use crate::specs::*; + pub use crate::utils::*; +} diff --git a/ml/s4/src/primitives.rs b/ml/s4/src/primitives.rs new file mode 100644 index 00000000..00daa8c3 --- /dev/null +++ b/ml/s4/src/primitives.rs @@ -0,0 +1,14 @@ +/* + Appellation: primitives + Contrib: FL03 +*/ +pub use self::{constants::*, statics::*, types::*}; + +mod constants { + /// The default model size for S4 models + pub const S4_MODEL_SIZE: usize = 2048; +} + +mod statics {} + +mod types {} diff --git a/ml/s4/src/specs.rs b/ml/s4/src/specs.rs new file mode 100644 index 00000000..1d8faa71 --- /dev/null +++ b/ml/s4/src/specs.rs @@ -0,0 +1,4 @@ +/* + Appellation: specs + Contrib: FL03 +*/ diff --git a/ml/s4/src/ssm/config.rs b/ml/s4/src/ssm/config.rs new file mode 100644 index 00000000..2fbb8cd7 --- /dev/null +++ b/ml/s4/src/ssm/config.rs @@ -0,0 +1,25 @@ +/* + Appellation: config + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct SSMConfig { + pub decode: bool, + pub features: usize, +} + +impl SSMConfig { + pub fn new(decode: bool, features: usize) -> Self { + Self { decode, features } + } + + pub fn decode(&self) -> bool { + self.decode + } + + pub fn features(&self) -> usize { + self.features + } +} diff --git a/ml/s4/src/ssm/mod.rs b/ml/s4/src/ssm/mod.rs new file mode 100644 index 00000000..f5c21381 --- /dev/null +++ b/ml/s4/src/ssm/mod.rs @@ -0,0 +1,29 @@ +/* + Appellation: ssm + Contrib: FL03 +*/ +//! # State Space Models (SSM) +//! +//! +pub use self::{config::*, model::*, utils::*}; + +pub(crate) mod config; +pub(crate) mod model; + +pub trait StateSpace { + fn features(&self) -> usize; + + +} + +pub(crate) mod utils {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ssm() { + let model = SSM::create(9); + } +} \ No newline at end of file diff --git a/ml/s4/src/ssm/model.rs b/ml/s4/src/ssm/model.rs new file mode 100644 index 00000000..f42dbc92 --- /dev/null +++ b/ml/s4/src/ssm/model.rs @@ -0,0 +1,27 @@ +/* + Appellation: model + Contrib: FL03 +*/ +use super::SSMConfig; +use ndarray::prelude::Array2; +use num::Float; + + +pub struct SSM { + config: SSMConfig, + pub a: Array2, + pub b: Array2, + pub c: Array2, + pub d: Array2, +} + +impl SSM where T: Float { + pub fn create(config: SSMConfig) -> Self { + let features = config.features(); + let a = Array2::::zeros(features); + let b = Array2::::zeros((features, 1)); + let c = Array2::::zeros((1, features)); + let d = Array2::::zeros((1, 1)); + Self { config, a, b, c, d } + } +} diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs new file mode 100644 index 00000000..752dabaf --- /dev/null +++ b/ml/s4/src/utils.rs @@ -0,0 +1,4 @@ +/* + Appellation: utils + Contrib: FL03 +*/ diff --git a/ml/s4/tests/default.rs b/ml/s4/tests/default.rs new file mode 100644 index 00000000..0cac1eb5 --- /dev/null +++ b/ml/s4/tests/default.rs @@ -0,0 +1,8 @@ +#[cfg(test)] +#[test] +fn compiles() { + let f = |x: usize, y: usize| x + y; + + assert_eq!(f(10, 10), 20); + assert_ne!(f(1, 1), 3); +} From 5645cac024f562acda3194bcdafdf9ef2f5ead12 Mon Sep 17 00:00:00 2001 From: FL03 Date: Thu, 14 Dec 2023 10:30:10 -0600 Subject: [PATCH 083/118] update Signed-off-by: FL03 --- .docker/env.dockerfile | 10 +++++- Cargo.toml | 14 ++++++-- bin/cli/Cargo.toml | 16 --------- bin/cli/src/main.rs | 3 -- bin/concise/Cargo.toml | 34 ------------------- bin/concise/src/main.rs | 3 -- core/src/specs.rs | 10 ++---- core/src/utils.rs | 2 -- data/src/tensors/mod.rs | 9 +++++ data/src/tensors/tensor.rs | 33 +++++++++++++----- derive/Cargo.toml | 2 +- math/Cargo.toml | 2 -- ml/neural/src/models/mod.rs | 3 +- ml/neural/src/models/modes.rs | 64 +++++++++++++++++++++++++++++++++++ ml/neural/src/nn/cnn/mod.rs | 4 +-- ml/neural/src/nn/cnn/model.rs | 4 +-- ml/neural/src/nn/rnn/mod.rs | 4 +-- ml/neural/src/nn/rnn/model.rs | 4 +-- ml/neural/src/specs.rs | 8 ++--- ml/optim/src/grad/descent.rs | 10 ++---- ml/optim/src/specs.rs | 1 - ml/s4/Cargo.toml | 6 ++-- ml/s4/src/ssm/mod.rs | 13 ++++--- ml/s4/src/ssm/model.rs | 53 ++++++++++++++++++++++++++--- ml/s4/src/ssm/params.rs | 35 +++++++++++++++++++ ml/transformers/Cargo.toml | 2 +- 26 files changed, 229 insertions(+), 120 deletions(-) delete mode 100644 bin/cli/Cargo.toml delete mode 100644 bin/cli/src/main.rs delete mode 100644 bin/concise/Cargo.toml delete mode 100644 bin/concise/src/main.rs create mode 100644 ml/neural/src/models/modes.rs create mode 100644 ml/s4/src/ssm/params.rs diff --git a/.docker/env.dockerfile b/.docker/env.dockerfile index 47cf4e5a..85cb0408 100644 --- a/.docker/env.dockerfile +++ b/.docker/env.dockerfile @@ -3,9 +3,17 @@ FROM debian RUN apt-get update -y && apt-get upgrade -y RUN apt-get install -y \ + build-essential \ curl \ git \ gnupg \ + libprotobuf-dev \ + libssl-dev \ + pkg-config \ unzip \ zip \ - && rm -rf /var/lib/apt/lists/* \ No newline at end of file + && rm -rf /var/lib/apt/lists/* + +RUN apt-get install -y \ + liblapack-dev \ + libopenblas-dev \ diff --git a/Cargo.toml b/Cargo.toml index 3e5072a7..87e6bebc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,11 +16,14 @@ concision-core = { path = "core", version = "0.1.12" } computare = { features = ["full"], branch = "v0.1.0", git = "https://github.com/FL03/computare", version = "0.1.0" } anyhow = "1" + +faer = { features = ["ndarray"], version = "0.16" } +faer-core = "0.16" itertools = { features = [], version = "0.12" } lazy_static = "1" linfa = { features = [], version = "0.7" } ndarray = { features = ["serde-1"], version = "0.15" } -ndarray-linalg = { features = ["openblas-system"], version = "0.16" } +ndarray-linalg = { features = [], version = "0.16" } ndarray-rand = { features = [], version = "0.14" } ndarray-stats = { features = [], version = "0.5.1" } num = { features = ["serde"], version = "0.4" } @@ -37,14 +40,19 @@ default-members = [ ] members = [ - "bin/*", "concision", "core", "data", "derive", "macros", "math", - "ml/*", + # "ml/*", + "ml/ml", + "ml/neural", + "ml/nlp", + "ml/optim", + "ml/s4", + "ml/transformers", ] resolver = "2" diff --git a/bin/cli/Cargo.toml b/bin/cli/Cargo.toml deleted file mode 100644 index ea94c7a1..00000000 --- a/bin/cli/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "cli" -authors.workspace = true -categories.workspace = true -description.workspace = true -edition.workspace = true -homepage.workspace = true -keywords.workspace = true -license.workspace = true -readme.workspace = true -repository.workspace = true -version.workspace = true - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] diff --git a/bin/cli/src/main.rs b/bin/cli/src/main.rs deleted file mode 100644 index e7a11a96..00000000 --- a/bin/cli/src/main.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - println!("Hello, world!"); -} diff --git a/bin/concise/Cargo.toml b/bin/concise/Cargo.toml deleted file mode 100644 index e3d97a42..00000000 --- a/bin/concise/Cargo.toml +++ /dev/null @@ -1,34 +0,0 @@ -[package] -authors.workspace = true -categories.workspace = true -default-run = "concise" -description.workspace = true -edition.workspace = true -homepage.workspace = true -keywords.workspace = true -license.workspace = true -name = "concise" -readme.workspace = true -repository.workspace = true -version.workspace = true - -[[bin]] -name = "concise" -path = "src/main.rs" - -[build-dependencies] - -[dependencies] -anyhow.workspace = true -concision = { features = ["full"], path = "../../concision", version = "0.1.12" } -ndarray.workspace = true - -[dev-dependencies] - -[package.metadata.docs.rs] -all-features = true -rustc-args = ["--cfg", "docsrs"] - -[target.wasm32-unknown-unknown] - -[target.wasm32-wasi] \ No newline at end of file diff --git a/bin/concise/src/main.rs b/bin/concise/src/main.rs deleted file mode 100644 index e7a11a96..00000000 --- a/bin/concise/src/main.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - println!("Hello, world!"); -} diff --git a/core/src/specs.rs b/core/src/specs.rs index 16f93860..de4005a5 100644 --- a/core/src/specs.rs +++ b/core/src/specs.rs @@ -4,7 +4,7 @@ */ pub use self::math::*; -use ndarray::prelude::{Array, Axis, Dimension, Ix2,}; +use ndarray::prelude::{Array, Axis, Dimension, Ix2}; use ndarray::IntoDimension; // use ndarray::linalg::Dot; use ndarray_rand::rand_distr::uniform::SampleUniform; @@ -16,7 +16,6 @@ pub trait Discritize { fn discritize(&self, step: T) -> Self; } - pub trait Apply { fn apply(&self, f: F) -> Self where @@ -43,9 +42,6 @@ pub trait As: AsRef + AsMut {} impl As for S where S: AsRef + AsMut {} - - - pub trait GenerateRandom where D: Dimension, @@ -103,7 +99,7 @@ pub(crate) mod math { pub trait FloatExt: FromPrimitive + NdFloat + Signed + SampleUniform {} impl FloatExt for T where T: FromPrimitive + NdFloat + Signed + SampleUniform {} - + pub trait Arithmetic: ops::Add + ops::Div @@ -147,4 +143,4 @@ pub(crate) mod math { Self: Arithmetic, Array>, { } -} \ No newline at end of file +} diff --git a/core/src/utils.rs b/core/src/utils.rs index 06ac0b85..b72e6324 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -7,8 +7,6 @@ use ndarray::prelude::{Array, Axis, Dimension}; use ndarray::{concatenate, IntoDimension, RemoveAxis, ShapeError}; use num::Float; - - pub fn concat_iter(axis: usize, iter: impl IntoIterator>) -> Array where D: RemoveAxis, diff --git a/data/src/tensors/mod.rs b/data/src/tensors/mod.rs index a0ff37c2..3c2fd578 100644 --- a/data/src/tensors/mod.rs +++ b/data/src/tensors/mod.rs @@ -7,4 +7,13 @@ pub use self::{tensor::*, utils::*}; pub(crate) mod tensor; +use ndarray::prelude::{Array, Dimension, Ix2}; + +pub trait NdTensor +where + D: Dimension, +{ + fn data(&self) -> &Array; +} + pub(crate) mod utils {} diff --git a/data/src/tensors/tensor.rs b/data/src/tensors/tensor.rs index e5944170..af399da4 100644 --- a/data/src/tensors/tensor.rs +++ b/data/src/tensors/tensor.rs @@ -2,22 +2,39 @@ Appellation: tensor Contrib: FL03 */ +use ndarray::prelude::{Array, Dimension, Ix2}; +use ndarray::IntoDimension; +use num::Float; use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)] #[serde(rename_all = "lowercase")] -pub struct Tensor { - data: Vec, +pub struct Tensor +where + D: Dimension, + T: Float, +{ + data: Array, } -impl Tensor { - pub fn new() -> Self { - Self { data: Vec::new() } +impl Tensor +where + D: Dimension, + T: Float, +{ + pub fn new(shape: impl IntoDimension) -> Self { + Self { + data: Array::zeros(shape), + } } } -impl std::fmt::Display for Tensor { +impl std::fmt::Display for Tensor +where + D: Dimension, + T: Float + std::fmt::Debug, +{ fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{}", serde_json::to_string(self).unwrap()) + write!(f, "{:?}", self.data) } } diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 8320775f..f885219f 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -21,7 +21,7 @@ proc-macro = true test = false [dependencies] -proc-macro2 = "1.0.69" +proc-macro2 = "1" quote = "1" syn = { features = ["full"], version = "2" } diff --git a/math/Cargo.toml b/math/Cargo.toml index e4108012..a1080e80 100644 --- a/math/Cargo.toml +++ b/math/Cargo.toml @@ -25,8 +25,6 @@ test = true [dependencies] anyhow.workspace = true ndarray.workspace = true -ndarray-rand.workspace = true -ndarray-stats.workspace = true num.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/ml/neural/src/models/mod.rs b/ml/neural/src/models/mod.rs index 504e6242..e570cf49 100644 --- a/ml/neural/src/models/mod.rs +++ b/ml/neural/src/models/mod.rs @@ -4,10 +4,11 @@ */ //! # Model //! -pub use self::{config::*, model::*, params::*, utils::*}; +pub use self::{config::*, model::*, modes::*, params::*, utils::*}; pub(crate) mod config; pub(crate) mod model; +pub(crate) mod modes; pub(crate) mod params; pub mod exp; diff --git a/ml/neural/src/models/modes.rs b/ml/neural/src/models/modes.rs new file mode 100644 index 00000000..08192212 --- /dev/null +++ b/ml/neural/src/models/modes.rs @@ -0,0 +1,64 @@ +/* + Appellation: modes + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames, VariantNames}; + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum Mode { + Predict, + #[default] + Test, + Train, +} + +impl Mode { + pub fn predict() -> Self { + Self::Predict + } + + pub fn test() -> Self { + Self::Test + } + + pub fn train() -> Self { + Self::Train + } +} + +impl From for Mode { + fn from(mode: usize) -> Self { + match mode % Mode::VARIANTS.len() { + 0 => Self::Predict, + 1 => Self::Test, + _ => Self::Train, + } + } +} + +impl From for usize { + fn from(mode: Mode) -> Self { + mode as usize + } +} diff --git a/ml/neural/src/nn/cnn/mod.rs b/ml/neural/src/nn/cnn/mod.rs index 8d05f4ed..d86fccbc 100644 --- a/ml/neural/src/nn/cnn/mod.rs +++ b/ml/neural/src/nn/cnn/mod.rs @@ -12,6 +12,4 @@ pub(crate) mod model; pub(crate) mod utils {} #[cfg(test)] -mod tests { - -} \ No newline at end of file +mod tests {} diff --git a/ml/neural/src/nn/cnn/model.rs b/ml/neural/src/nn/cnn/model.rs index d4d6a4e1..0c92921e 100644 --- a/ml/neural/src/nn/cnn/model.rs +++ b/ml/neural/src/nn/cnn/model.rs @@ -5,6 +5,4 @@ use ndarray::prelude::Array2; use num::Float; -pub struct CNN { - -} +pub struct CNN {} diff --git a/ml/neural/src/nn/rnn/mod.rs b/ml/neural/src/nn/rnn/mod.rs index 84c9cf10..5f8a8d7f 100644 --- a/ml/neural/src/nn/rnn/mod.rs +++ b/ml/neural/src/nn/rnn/mod.rs @@ -12,6 +12,4 @@ pub(crate) mod model; pub(crate) mod utils {} #[cfg(test)] -mod tests { - -} \ No newline at end of file +mod tests {} diff --git a/ml/neural/src/nn/rnn/model.rs b/ml/neural/src/nn/rnn/model.rs index a2fc3e1b..08a2b48f 100644 --- a/ml/neural/src/nn/rnn/model.rs +++ b/ml/neural/src/nn/rnn/model.rs @@ -5,6 +5,4 @@ use ndarray::prelude::Array2; use num::Float; -pub struct RNN { - -} +pub struct RNN {} diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index c56793b7..1b79c0ab 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -85,14 +85,10 @@ where } } -pub trait Module: Forward, Output=Array> +pub trait Module: Forward, Output = Array> where D: Dimension, T: Float, { type Config; - - - - -} \ No newline at end of file +} diff --git a/ml/optim/src/grad/descent.rs b/ml/optim/src/grad/descent.rs index 45680743..fc4fbb65 100644 --- a/ml/optim/src/grad/descent.rs +++ b/ml/optim/src/grad/descent.rs @@ -73,11 +73,7 @@ where O: Gradient, T: NdFloat + Signed, { - pub fn gradient( - &mut self, - data: &Array2, - targets: &Array2, - ) -> anyhow::Result { + pub fn gradient(&mut self, data: &Array2, targets: &Array2) -> anyhow::Result { let lr = self.gamma(); let ns = T::from(data.shape()[0]).unwrap(); let pred = self.model.forward(data); @@ -127,13 +123,13 @@ mod tests { let mut grad = GradientDescent::new(gamma, model, Sigmoid); let l1 = { - let tmp = grad.gradient(&x, &y,); + let tmp = grad.gradient(&x, &y); assert!(tmp.is_ok()); tmp.unwrap() }; let l2 = { - let tmp = grad.gradient(&x, &y,); + let tmp = grad.gradient(&x, &y); assert!(tmp.is_ok()); tmp.unwrap() }; diff --git a/ml/optim/src/specs.rs b/ml/optim/src/specs.rs index 20ab0830..b1e73252 100644 --- a/ml/optim/src/specs.rs +++ b/ml/optim/src/specs.rs @@ -20,4 +20,3 @@ where { fn autograd(&mut self, loss: &Array) -> Array; } - diff --git a/ml/s4/Cargo.toml b/ml/s4/Cargo.toml index 857fafb7..de8dbb40 100644 --- a/ml/s4/Cargo.toml +++ b/ml/s4/Cargo.toml @@ -25,13 +25,15 @@ test = true [dependencies] concision-core.workspace = true -concision-data = { path = "../../data" } +# concision-data = { path = "../../data" } concision-neural = { path = "../neural" } anyhow.workspace = true +faer-core.workspace = true +faer.workspace = true lazy_static.workspace = true ndarray.workspace = true -ndarray-linalg.workspace = true +# ndarray-linalg.workspace = true ndarray-rand.workspace = true ndarray-stats.workspace = true num.workspace = true diff --git a/ml/s4/src/ssm/mod.rs b/ml/s4/src/ssm/mod.rs index f5c21381..8c836dfb 100644 --- a/ml/s4/src/ssm/mod.rs +++ b/ml/s4/src/ssm/mod.rs @@ -5,15 +5,14 @@ //! # State Space Models (SSM) //! //! -pub use self::{config::*, model::*, utils::*}; +pub use self::{config::*, model::*, params::*, utils::*}; pub(crate) mod config; pub(crate) mod model; +pub(crate) mod params; pub trait StateSpace { fn features(&self) -> usize; - - } pub(crate) mod utils {} @@ -24,6 +23,10 @@ mod tests { #[test] fn test_ssm() { - let model = SSM::create(9); + let step = 0.001; + + let config = SSMConfig::new(true, 9); + let mut model = SSM::::create(config); + assert!(model.descretize(step).is_ok()); } -} \ No newline at end of file +} diff --git a/ml/s4/src/ssm/model.rs b/ml/s4/src/ssm/model.rs index f42dbc92..809cb0c9 100644 --- a/ml/s4/src/ssm/model.rs +++ b/ml/s4/src/ssm/model.rs @@ -3,9 +3,14 @@ Contrib: FL03 */ use super::SSMConfig; -use ndarray::prelude::Array2; -use num::Float; - +use faer::prelude::{FaerMat, IntoFaer, SolverCore}; +use faer::IntoNdarray; +use faer_core::zip::ViewMut; +use faer_core::{ComplexField, Conjugate, SimpleEntity}; +use ndarray::prelude::{Array2, NdFloat}; +// use ndarray_linalg::solve::Inverse; +// use ndarray_linalg::types::Lapack; +use num::{Float, ToPrimitive}; pub struct SSM { config: SSMConfig, @@ -15,13 +20,51 @@ pub struct SSM { pub d: Array2, } -impl SSM where T: Float { +impl SSM +where + T: Float, +{ pub fn create(config: SSMConfig) -> Self { let features = config.features(); - let a = Array2::::zeros(features); + let a = Array2::::zeros((features, features)); let b = Array2::::zeros((features, 1)); let c = Array2::::zeros((1, features)); let d = Array2::::zeros((1, 1)); Self { config, a, b, c, d } } } + +impl SSM +where + T: NdFloat + Conjugate + SimpleEntity, + ::Canonical: ComplexField + SimpleEntity + ToPrimitive, +{ + pub fn descretize(&mut self, step: T) -> anyhow::Result<()> { + let ds = step / T::from(2).unwrap(); + let eye = Array2::::eye(self.config.features()); + let bl = &eye - &self.a * ds; + let mut bi = bl.view().into_faer().qr().inverse(); + let be = { + let arr = &bi.view_mut().into_ndarray(); + arr.mapv(|i| T::from(i).unwrap()) + }; + let ab = (&eye + &self.a * ds); + // let ab = &bl | (&eye + &self.a * ds); + // let bb = &self.b * ds | self.b; + + Ok(()) + } +} + +// impl SSM { + +// pub fn descretize(&mut self, step: f64) -> anyhow::Result<()> { +// let ds = step / 2.0; +// let eye = Array2::::eye(self.config.features()); +// let bl = (&eye - &self.a * ds).inv()?; +// // let ab = &bl | (&eye + &self.a * ds); +// // let bb = &self.b * ds | self.b; + +// Ok(()) +// } +// } diff --git a/ml/s4/src/ssm/params.rs b/ml/s4/src/ssm/params.rs new file mode 100644 index 00000000..22fe77eb --- /dev/null +++ b/ml/s4/src/ssm/params.rs @@ -0,0 +1,35 @@ +/* + Appellation: params + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum SSMParams { + #[default] + A, + B, + C, + D, +} diff --git a/ml/transformers/Cargo.toml b/ml/transformers/Cargo.toml index 82793ee8..6aa804a4 100644 --- a/ml/transformers/Cargo.toml +++ b/ml/transformers/Cargo.toml @@ -25,7 +25,7 @@ test = true [dependencies] concision-core.workspace = true -concision-data = { path = "../../data" } +# concision-data = { path = "../../data" } concision-neural = { path = "../neural" } anyhow.workspace = true From 9e97f3aafbe887cadee25300b9cfd34cdfe443d9 Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 15 Dec 2023 12:30:22 -0600 Subject: [PATCH 084/118] update Signed-off-by: FL03 --- .github/dependabot.yml | 16 ++++++ Cargo.toml | 1 - concision/Cargo.toml | 7 --- concision/src/lib.rs | 4 -- core/src/params/kinds.rs | 5 ++ core/src/params/mod.rs | 5 +- core/src/params/param.rs | 16 +++++- core/src/params/store.rs | 58 ++++++++++++++++++++++ math/Cargo.toml | 42 ---------------- math/src/lib.rs | 20 -------- math/src/primitives.rs | 11 ----- math/src/regress/linear.rs | 6 --- math/src/regress/mod.rs | 12 ----- math/src/specs.rs | 77 ----------------------------- math/src/utils.rs | 23 --------- math/tests/default.rs | 8 --- ml/neural/src/models/exp/modules.rs | 2 + ml/s4/src/ssm/model.rs | 6 +-- 18 files changed, 102 insertions(+), 217 deletions(-) create mode 100644 core/src/params/store.rs delete mode 100644 math/Cargo.toml delete mode 100644 math/src/lib.rs delete mode 100644 math/src/primitives.rs delete mode 100644 math/src/regress/linear.rs delete mode 100644 math/src/regress/mod.rs delete mode 100644 math/src/specs.rs delete mode 100644 math/src/utils.rs delete mode 100644 math/tests/default.rs diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6c555a01..15729339 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -28,10 +28,26 @@ updates: directory: /macros schedule: interval: daily + - package-ecosystem: cargo + directory: /ml/ml + schedule: + interval: daily - package-ecosystem: cargo directory: /ml/neural schedule: interval: daily + - package-ecosystem: cargo + directory: /ml/nlp + schedule: + interval: daily + - package-ecosystem: cargo + directory: /ml/optim + schedule: + interval: daily + - package-ecosystem: cargo + directory: /ml/s4 + schedule: + interval: daily - package-ecosystem: cargo directory: /ml/transformers schedule: diff --git a/Cargo.toml b/Cargo.toml index 87e6bebc..dd740b54 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,7 +45,6 @@ members = [ "data", "derive", "macros", - "math", # "ml/*", "ml/ml", "ml/neural", diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 5668c46d..9dffc331 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -15,7 +15,6 @@ version.workspace = true default = [ "core", "data", - "math", "ml", ] @@ -23,7 +22,6 @@ full = [ "core", "data", "derive", - "math", "ml", ] @@ -44,10 +42,6 @@ macros = [ "concision-macros" ] -math = [ - "concision-math" -] - ml = [ "neural", "nlp", @@ -89,7 +83,6 @@ concision-core = { features = [], optional = true, path = "../core", version = " concision-data = { features = [], optional = true, path = "../data", version = "0.1.12" } concision-derive = { features = [], optional = true, path = "../derive", version = "0.1.12" } concision-macros = { features = [], optional = true, path = "../macros", version = "0.1.12" } -concision-math = { features = [], optional = true, path = "../math", version = "0.1.12" } # concision-ml = { features = [], optional = true, path = "../ml/ml", version = "0.1.12" } concision-neural = { features = [], optional = true, path = "../ml/neural", version = "0.1.12" } diff --git a/concision/src/lib.rs b/concision/src/lib.rs index b46a4885..f3f1bbf6 100644 --- a/concision/src/lib.rs +++ b/concision/src/lib.rs @@ -15,8 +15,6 @@ pub use concision_data as data; pub use concision_derive::*; #[cfg(feature = "macros")] pub use concision_macros::*; -#[cfg(feature = "math")] -pub use concision_math as math; // #[cfg(feature = "ml")] // pub use concision_ml as ml; #[cfg(feature = "neural")] @@ -39,8 +37,6 @@ pub mod prelude { pub use concision_derive::*; #[cfg(feature = "macros")] pub use concision_macros::*; - #[cfg(feature = "math")] - pub use concision_math::prelude::*; // #[cfg(feature = "ml")] // pub use concision_ml::prelude::*; #[cfg(feature = "neural")] diff --git a/core/src/params/kinds.rs b/core/src/params/kinds.rs index 0920bf0d..36158789 100644 --- a/core/src/params/kinds.rs +++ b/core/src/params/kinds.rs @@ -5,6 +5,11 @@ use serde::{Deserialize, Serialize}; use strum::{EnumIs, EnumIter, EnumString, EnumVariantNames}; +pub trait ParamType: Eq + std::hash::Hash { + + fn ptype(&self) -> &str; +} + #[derive( Clone, Debug, diff --git a/core/src/params/mod.rs b/core/src/params/mod.rs index 6ccbc699..b2232a03 100644 --- a/core/src/params/mod.rs +++ b/core/src/params/mod.rs @@ -6,17 +6,18 @@ //! //! ## Overview //! -pub use self::{group::*, kinds::*, param::*, utils::*}; +pub use self::{group::*, kinds::*, param::*, store::*, utils::*}; pub(crate) mod group; pub(crate) mod kinds; pub(crate) mod param; +pub(crate) mod store; use ndarray::prelude::{Array, Dimension, Ix2}; use num::Float; pub trait Param { - fn kind(&self) -> ParamKind; + fn kind(&self) -> &ParamKind; fn name(&self) -> &str; } diff --git a/core/src/params/param.rs b/core/src/params/param.rs index a83755a0..772655a9 100644 --- a/core/src/params/param.rs +++ b/core/src/params/param.rs @@ -2,7 +2,7 @@ Appellation: param Contrib: FL03 */ -use super::ParamKind; +use super::{Param, ParamKind}; use crate::prelude::GenerateRandom; use ndarray::prelude::{Array, Dimension, Ix2}; use ndarray::IntoDimension; @@ -103,3 +103,17 @@ where self } } + +impl Param for Parameter +where + T: Float, + D: Dimension, +{ + fn kind(&self) -> &ParamKind { + &self.kind + } + + fn name(&self) -> &str { + &self.name + } +} diff --git a/core/src/params/store.rs b/core/src/params/store.rs new file mode 100644 index 00000000..cb80cf6e --- /dev/null +++ b/core/src/params/store.rs @@ -0,0 +1,58 @@ +/* + Appellation: store + Contrib: FL03 +*/ +use super::{ParamKind, Parameter}; +use ndarray::prelude::{Dimension, Ix2}; +use num::Float; +use std::collections::HashMap; + +pub struct ParamStore where T: Float, D: Dimension { + store: HashMap>, +} + +impl ParamStore where D: Dimension, T: Float, { + pub fn new() -> Self { + Self { + store: HashMap::new(), + } + } + + pub fn insert(&mut self, param: Parameter) { + self.store.insert(param.kind().clone(), param); + } + +} + +impl Extend> for ParamStore where D: Dimension, T: Float, { + fn extend>>(&mut self, iter: I) { + for param in iter { + self.insert(param); + } + } +} + +impl IntoIterator for ParamStore where D: Dimension, T: Float, { + type Item = (ParamKind, Parameter); + type IntoIter = std::collections::hash_map::IntoIter>; + + fn into_iter(self) -> Self::IntoIter { + self.store.into_iter() + } +} + + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_model_store() { + let (inputs, outputs) = (5, 3); + + let shapes = [(inputs, outputs), (outputs, outputs), (outputs, 1)]; + + let params = ParamStore::::new(); + + } +} diff --git a/math/Cargo.toml b/math/Cargo.toml deleted file mode 100644 index a1080e80..00000000 --- a/math/Cargo.toml +++ /dev/null @@ -1,42 +0,0 @@ -[package] -authors.workspace = true -categories.workspace = true -description.workspace = true -edition.workspace = true -homepage.workspace = true -keywords.workspace = true -license.workspace = true -name = "concision-math" -readme.workspace = true -repository.workspace = true -version.workspace = true - -[features] -default = [] - -[lib] -bench = false -crate-type = ["cdylib", "rlib"] -doctest = false -test = true - -[build-dependencies] - -[dependencies] -anyhow.workspace = true -ndarray.workspace = true -num.workspace = true -serde.workspace = true -serde_json.workspace = true -smart-default.workspace = true -strum.workspace = true - -[dev-dependencies] - -[package.metadata.docs.rs] -all-features = true -rustc-args = ["--cfg", "docsrs"] - -[target.wasm32-unknown-unknown] - -[target.wasm32-wasi] diff --git a/math/src/lib.rs b/math/src/lib.rs deleted file mode 100644 index 9d0fb858..00000000 --- a/math/src/lib.rs +++ /dev/null @@ -1,20 +0,0 @@ -/* - Appellation: math - Contrib: FL03 -*/ -//! # Concision Math -pub use self::{primitives::*, specs::*, utils::*}; - -pub(crate) mod primitives; -pub(crate) mod specs; -pub(crate) mod utils; - -pub mod regress; - -pub mod prelude { - pub use crate::regress::*; - - pub use crate::primitives::*; - pub use crate::specs::*; - pub use crate::utils::*; -} diff --git a/math/src/primitives.rs b/math/src/primitives.rs deleted file mode 100644 index 859023bb..00000000 --- a/math/src/primitives.rs +++ /dev/null @@ -1,11 +0,0 @@ -/* - Appellation: primitives - Contrib: FL03 -*/ -pub use self::{constants::*, statics::*, types::*}; - -mod constants {} - -mod statics {} - -mod types {} diff --git a/math/src/regress/linear.rs b/math/src/regress/linear.rs deleted file mode 100644 index da279fa5..00000000 --- a/math/src/regress/linear.rs +++ /dev/null @@ -1,6 +0,0 @@ -/* - Appellation: linear - Contrib: FL03 -*/ - -pub struct LinearRegression {} diff --git a/math/src/regress/mod.rs b/math/src/regress/mod.rs deleted file mode 100644 index 5d4de7ed..00000000 --- a/math/src/regress/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -/* - Appellation: epochs - Contrib: FL03 -*/ -pub use self::{linear::*, utils::*}; - -pub(crate) mod linear; - -pub(crate) mod utils {} - -#[cfg(test)] -mod tests {} diff --git a/math/src/specs.rs b/math/src/specs.rs deleted file mode 100644 index cc3de9c7..00000000 --- a/math/src/specs.rs +++ /dev/null @@ -1,77 +0,0 @@ -/* - Appellation: specs - Contrib: FL03 -*/ -use ndarray::prelude::Array; -use ndarray::{Dimension, ShapeError}; -use num::Num; -use std::ops; - -pub trait NumOpsAssign: - Sized + ops::AddAssign + ops::DivAssign + ops::MulAssign + ops::SubAssign -{ -} - -impl NumOpsAssign for T where T: ops::AddAssign + ops::DivAssign + ops::MulAssign + ops::SubAssign -{} - -trait Matmul -where - T: Num, - D: Dimension, -{ - fn matmul(&self, other: &Array) -> Result, ShapeError>; - - fn shape(&self) -> D; -} - -// impl Matmul for Array -// where -// T: Num + std::ops::Mul + std::ops::Add + Clone, -// D: Dimension, -// { -// fn matmul(&self, other: &Array) -> Result, ShapeError> { -// let self_shape = self.shape(); -// let other_shape = other.shape(); - -// if self_shape[self.ndim() - 1] != other_shape[self.ndim() - 2] { -// return Err(ShapeError::from_kind(ndarray::ErrorKind::IncompatibleShape)); -// } - -// let mut result = Array::zeros(self_shape); - -// let mut self_shape = self_shape.to_vec(); -// let self_last = self_shape.pop().unwrap(); -// let other_shape = other_shape.to_vec(); - -// let mut iter_self = self.iter(); -// let mut iter_other = other.iter(); - -// for mut row_result in result.outer_iter_mut() { -// for mut col_other in other.inner_iter() { -// let row_self = iter_self.clone(); -// let mut col_other = col_other.clone(); -// let dot = dot_product(&mut row_self, &mut col_other, self_last, &other_shape); -// row_result.assign(&dot); -// } -// iter_self.step_by(self_shape.last().unwrap().index()); -// } - -// Ok(result) -// } - -// fn shape(&self) -> D { -// self.raw_dim() -// } -// } - -#[cfg(test)] -mod tests { - // use super::*; - - #[test] - fn test_product() { - let args = vec![2.0, 4.0, 6.0]; - assert_eq!(args.into_iter().product::(), 48.0); - } -} diff --git a/math/src/utils.rs b/math/src/utils.rs deleted file mode 100644 index c2e898e1..00000000 --- a/math/src/utils.rs +++ /dev/null @@ -1,23 +0,0 @@ -/* - Appellation: utils - Contrib: FL03 -*/ -use ndarray::linalg::Dot; -use ndarray::prelude::{Array, NdFloat}; -use ndarray::Dimension; -use num::FromPrimitive; - -pub fn covariance(ddof: T, x: &Array, y: &Array) -> anyhow::Result> -where - D: Dimension, - T: Default + FromPrimitive + NdFloat, - Array: Dot, Output = Array>, -{ - let x_mean = x.mean().unwrap_or_default(); - let y_mean = y.mean().unwrap_or_default(); - let xs = x - x_mean; - let ys = y - y_mean; - let cov = xs.dot(&ys.t().to_owned()); - let scale = T::one() / (T::from(x.len()).unwrap() - ddof); - Ok(cov * scale) -} diff --git a/math/tests/default.rs b/math/tests/default.rs deleted file mode 100644 index 0cac1eb5..00000000 --- a/math/tests/default.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[cfg(test)] -#[test] -fn compiles() { - let f = |x: usize, y: usize| x + y; - - assert_eq!(f(10, 10), 20); - assert_ne!(f(1, 1), 3); -} diff --git a/ml/neural/src/models/exp/modules.rs b/ml/neural/src/models/exp/modules.rs index 7a7ebd82..f49b7bf8 100644 --- a/ml/neural/src/models/exp/modules.rs +++ b/ml/neural/src/models/exp/modules.rs @@ -25,4 +25,6 @@ where fn modules(&self) -> Vec<&impl Module>; fn name(&self) -> &str; + + fn parameters(&self) -> Vec<&Array2>; } diff --git a/ml/s4/src/ssm/model.rs b/ml/s4/src/ssm/model.rs index 809cb0c9..c2ebec29 100644 --- a/ml/s4/src/ssm/model.rs +++ b/ml/s4/src/ssm/model.rs @@ -43,14 +43,14 @@ where let ds = step / T::from(2).unwrap(); let eye = Array2::::eye(self.config.features()); let bl = &eye - &self.a * ds; + let pos = &eye + &self.a * ds; // positive let mut bi = bl.view().into_faer().qr().inverse(); let be = { let arr = &bi.view_mut().into_ndarray(); arr.mapv(|i| T::from(i).unwrap()) }; - let ab = (&eye + &self.a * ds); - // let ab = &bl | (&eye + &self.a * ds); - // let bb = &self.b * ds | self.b; + let ab = &be.dot(&pos); + let bb = (&self.b * ds).dot(&self.b); Ok(()) } From b4b9fe4c196912c66186fe7769bff8ab5b03a6ae Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 15 Dec 2023 15:20:22 -0600 Subject: [PATCH 085/118] update Signed-off-by: FL03 --- Cargo.toml | 5 +- core/src/params/kinds.rs | 6 +- core/src/params/mod.rs | 19 +++- core/src/params/param.rs | 17 +++- core/src/params/store.rs | 3 + ml/neural/Cargo.toml | 2 +- ml/neural/src/arch/architecture.rs | 4 - ml/neural/src/arch/deep.rs | 68 -------------- ml/neural/src/arch/mod.rs | 89 ------------------- ml/neural/src/arch/shallow.rs | 88 ------------------ ml/neural/src/layers/exp/mod.rs | 2 + ml/neural/src/layers/mod.rs | 2 +- ml/neural/src/lib.rs | 2 - ml/neural/src/models/exp/mod.rs | 18 ++++ ml/neural/src/models/exp/modules.rs | 26 +++--- ml/neural/src/specs.rs | 8 ++ ml/nlp/Cargo.toml | 2 +- ml/optim/Cargo.toml | 2 +- ml/s4/Cargo.toml | 2 +- ml/s4/src/ssm/model.rs | 9 +- ml/transformers/Cargo.toml | 2 +- .../src/transform/{params.rs => config.rs} | 0 ml/transformers/src/transform/mod.rs | 4 +- 23 files changed, 93 insertions(+), 287 deletions(-) delete mode 100644 ml/neural/src/arch/architecture.rs delete mode 100644 ml/neural/src/arch/deep.rs delete mode 100644 ml/neural/src/arch/mod.rs delete mode 100644 ml/neural/src/arch/shallow.rs rename ml/transformers/src/transform/{params.rs => config.rs} (100%) diff --git a/Cargo.toml b/Cargo.toml index dd740b54..da5dc88c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,8 +11,6 @@ repository = "https://github.com/FL03/concision" version = "0.1.12" # TODO - Update the cargo package version [workspace.dependencies] -concision-core = { path = "core", version = "0.1.12" } - computare = { features = ["full"], branch = "v0.1.0", git = "https://github.com/FL03/computare", version = "0.1.0" } anyhow = "1" @@ -21,9 +19,8 @@ faer = { features = ["ndarray"], version = "0.16" } faer-core = "0.16" itertools = { features = [], version = "0.12" } lazy_static = "1" -linfa = { features = [], version = "0.7" } ndarray = { features = ["serde-1"], version = "0.15" } -ndarray-linalg = { features = [], version = "0.16" } +# ndarray-linalg = { features = [], version = "0.16" } ndarray-rand = { features = [], version = "0.14" } ndarray-stats = { features = [], version = "0.5.1" } num = { features = ["serde"], version = "0.4" } diff --git a/core/src/params/kinds.rs b/core/src/params/kinds.rs index 36158789..e53ed35f 100644 --- a/core/src/params/kinds.rs +++ b/core/src/params/kinds.rs @@ -5,11 +5,13 @@ use serde::{Deserialize, Serialize}; use strum::{EnumIs, EnumIter, EnumString, EnumVariantNames}; -pub trait ParamType: Eq + std::hash::Hash { +pub trait ParamType: ToString { - fn ptype(&self) -> &str; + fn kind(&self) -> String; } + + #[derive( Clone, Debug, diff --git a/core/src/params/mod.rs b/core/src/params/mod.rs index b2232a03..4571ef8e 100644 --- a/core/src/params/mod.rs +++ b/core/src/params/mod.rs @@ -64,4 +64,21 @@ where pub(crate) mod utils {} #[cfg(test)] -mod tests {} +mod tests { + use super::*; + use crate::linarr; + use ndarray::linalg::Dot; + use ndarray::prelude::{Ix1, Ix2}; + + #[test] + fn test_parameter() { + let a = linarr::((3,)).unwrap(); + let p = linarr::((3, 3)).unwrap(); + let mut param = Parameter::::new((10, 1), ParamKind::Bias, "bias"); + param.set_params(p.clone()); + + assert_eq!(param.kind(), &ParamKind::Bias); + assert_eq!(param.name(), "bias"); + assert_eq!(param.dot(&a), p.dot(&a)); + } +} diff --git a/core/src/params/param.rs b/core/src/params/param.rs index 772655a9..daa43f8c 100644 --- a/core/src/params/param.rs +++ b/core/src/params/param.rs @@ -6,6 +6,7 @@ use super::{Param, ParamKind}; use crate::prelude::GenerateRandom; use ndarray::prelude::{Array, Dimension, Ix2}; use ndarray::IntoDimension; +use ndarray::linalg::Dot; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; @@ -99,7 +100,8 @@ where T: Float + SampleUniform, { pub fn init_uniform(mut self, dk: T) -> Self { - self.params = Array::uniform_between(dk, self.clone().features); + let dim = self.params.dim(); + self.params = Array::uniform_between(dk, dim); self } } @@ -117,3 +119,16 @@ where &self.name } } + +impl Dot for Parameter +where + Array: Dot, + D: Dimension, + T: Float, +{ + type Output = O; + + fn dot(&self, rhs: &S) -> Self::Output { + self.params.dot(rhs) + } +} diff --git a/core/src/params/store.rs b/core/src/params/store.rs index cb80cf6e..3eb722cf 100644 --- a/core/src/params/store.rs +++ b/core/src/params/store.rs @@ -5,8 +5,10 @@ use super::{ParamKind, Parameter}; use ndarray::prelude::{Dimension, Ix2}; use num::Float; +use serde::{Deserialize, Serialize}; use std::collections::HashMap; +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] pub struct ParamStore where T: Float, D: Dimension { store: HashMap>, } @@ -24,6 +26,7 @@ impl ParamStore where D: Dimension, T: Float, { } + impl Extend> for ParamStore where D: Dimension, T: Float, { fn extend>>(&mut self, iter: I) { for param in iter { diff --git a/ml/neural/Cargo.toml b/ml/neural/Cargo.toml index 011259f5..9ecf31b1 100644 --- a/ml/neural/Cargo.toml +++ b/ml/neural/Cargo.toml @@ -24,7 +24,7 @@ test = true [build-dependencies] [dependencies] -concision-core.workspace = true +concision-core = { path = "../../core", version = "0.1.12" } anyhow.workspace = true ndarray.workspace = true diff --git a/ml/neural/src/arch/architecture.rs b/ml/neural/src/arch/architecture.rs deleted file mode 100644 index ac0460ef..00000000 --- a/ml/neural/src/arch/architecture.rs +++ /dev/null @@ -1,4 +0,0 @@ -/* - Appellation: architecture - Contrib: FL03 -*/ diff --git a/ml/neural/src/arch/deep.rs b/ml/neural/src/arch/deep.rs deleted file mode 100644 index d5eb62c9..00000000 --- a/ml/neural/src/arch/deep.rs +++ /dev/null @@ -1,68 +0,0 @@ -/* - Appellation: network - Contrib: FL03 -*/ -use crate::func::activate::{Activate, Linear}; -use crate::prelude::{Features, Forward, Layer, Parameterized, Stack}; - -use ndarray::prelude::{Array2, NdFloat}; -use num::Float; - -pub struct DeepNetwork -where - T: Float, - I: Activate, - H: Activate, - O: Activate, -{ - pub input: Layer, - pub hidden: Stack, - pub output: Layer, -} - -impl DeepNetwork -where - T: Float, - I: Activate, - H: Activate, - O: Activate, -{ - pub fn new(input: Layer, hidden: Stack, output: Layer) -> Self { - Self { - input, - hidden, - output, - } - } -} -impl DeepNetwork -where - T: Float, - I: Activate + Clone, - H: Activate + Clone, - O: Activate + Clone, -{ - pub fn validate_dims(&self) -> bool { - self.hidden.validate_shapes() - && self.input.features().outputs() == self.hidden.first().unwrap().features().inputs() - && self.output.features().inputs() == self.hidden.last().unwrap().features().outputs() - } -} - -impl Forward> for DeepNetwork -where - T: NdFloat, - I: Activate + Clone, - H: Activate + Clone, - O: Activate + Clone, -{ - type Output = Array2; - - fn forward(&self, args: &Array2) -> Self::Output { - let mut out = self.input.forward(args); - for layer in self.hidden.clone().into_iter() { - out = layer.forward(&out); - } - self.output.forward(&out) - } -} diff --git a/ml/neural/src/arch/mod.rs b/ml/neural/src/arch/mod.rs deleted file mode 100644 index 8d528df1..00000000 --- a/ml/neural/src/arch/mod.rs +++ /dev/null @@ -1,89 +0,0 @@ -/* - Appellation: arch - Contrib: FL03 -*/ -//! # Architecture -//! -//! This module describes the architecture of various components of the neural network. -//! -//! ## Contents -//! -//! ## Verification -//! -//! - Each layer must have the same number of inputs as the previous layer has outputs. -//! - Input Layers should be unbiased -pub use self::{architecture::*, deep::*, shallow::*, utils::*}; - -pub(crate) mod architecture; -pub(crate) mod deep; -pub(crate) mod shallow; - -pub trait NetworkOps {} - -pub trait Arch { - /// Validates the dimensions of the network. Each - fn validate_layers(&self) -> bool; -} - -pub(crate) mod utils {} - -#[cfg(test)] -mod tests { - use super::*; - use crate::core::prelude::linarr; - use crate::func::activate::{ReLU, Sigmoid, Softmax}; - use crate::prelude::{Forward, Layer, LayerShape, Stack}; - use ndarray::prelude::Ix2; - - #[test] - fn test_arch() { - assert!(true); - } - - #[test] - fn test_deep_network() { - let samples = 20; - let (inputs, outputs) = (5, 3); - let shapes = [(outputs, 4), (4, 4), (4, inputs)]; - - // sample data - let x = linarr::((samples, inputs)).unwrap(); - let _y = linarr::((samples, outputs)).unwrap(); - - // layers - let hidden = Stack::::new() - .build_layers(shapes) - .init_layers(true); - let input = Layer::::from(LayerShape::new(inputs, outputs)).init(false); - let output = Layer::::from(LayerShape::new(inputs, outputs)).init(false); - - let network = DeepNetwork::new(input, hidden, output); - assert!(network.validate_dims()); - - let pred = network.forward(&x); - assert_eq!(&pred.dim(), &(samples, outputs)); - } - - #[test] - fn test_shallow_network() { - let samples = 20; - let (inputs, outputs) = (5, 3); - - // sample data - let x = linarr::((samples, inputs)).unwrap(); - let _y = linarr::((samples, outputs)).unwrap(); - - // layers - let in_features = LayerShape::new(inputs, outputs); - let out_features = LayerShape::new(outputs, outputs); - let input = Layer::::from(in_features).init(false); - let hidden = Layer::::from(out_features).init(false); - let output = Layer::::from(out_features).init(false); - - let network = ShallowNetwork::new(input, hidden, output); - assert!(network.validate_dims()); - - let pred = network.forward(&x); - assert_eq!(&pred.dim(), &(samples, outputs)); - } -} diff --git a/ml/neural/src/arch/shallow.rs b/ml/neural/src/arch/shallow.rs deleted file mode 100644 index 8a16ab5c..00000000 --- a/ml/neural/src/arch/shallow.rs +++ /dev/null @@ -1,88 +0,0 @@ -/* - Appellation: shallow - Contrib: FL03 -*/ -use crate::func::activate::{Activate, Linear, ReLU, Softmax}; -use crate::prelude::{Features, Forward, Layer, LayerShape, Parameterized}; - -use ndarray::prelude::{Array2, NdFloat}; -use num::Float; - -pub struct ShallowNetwork -where - T: Float, - I: Activate, - H: Activate, - O: Activate, -{ - pub input: Layer, - pub hidden: Layer, - pub output: Layer, -} - -impl ShallowNetwork -where - T: Float, - I: Activate, - H: Activate, - O: Activate, -{ - pub fn new(input: Layer, hidden: Layer, output: Layer) -> Self { - Self { - input, - hidden, - output, - } - } - - pub fn hidden(&self) -> &Layer { - &self.hidden - } - - pub fn input(&self) -> &Layer { - &self.input - } - - pub fn output(&self) -> &Layer { - &self.output - } - - pub fn validate_dims(&self) -> bool { - self.input.features().outputs() == self.hidden.features().inputs() - && self.hidden.features().outputs() == self.output.features().inputs() - } -} - -impl ShallowNetwork -where - T: Float, - I: Activate + Default, - H: Activate + Default, - O: Activate + Default, -{ - pub fn create(inputs: usize, outputs: usize) -> Self { - let s1 = LayerShape::new(inputs, outputs); - let s2 = LayerShape::new(outputs, outputs); - - let input = Layer::::from(s1); - let hidden = Layer::::from(s2); - let output = Layer::::from(s2); - - Self::new(input, hidden, output) - } -} - -impl Forward> for ShallowNetwork -where - T: NdFloat, - I: Activate, - H: Activate, - O: Activate, -{ - type Output = Array2; - - fn forward(&self, args: &Array2) -> Self::Output { - self.output - .forward(&self.hidden.forward(&self.input.forward(args))) - } -} diff --git a/ml/neural/src/layers/exp/mod.rs b/ml/neural/src/layers/exp/mod.rs index 86c283d2..aecf9766 100644 --- a/ml/neural/src/layers/exp/mod.rs +++ b/ml/neural/src/layers/exp/mod.rs @@ -10,6 +10,8 @@ pub(crate) mod layer; pub(crate) mod sublayer; pub(crate) mod wrapper; + + pub(crate) mod utils {} #[cfg(test)] diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index 611db39f..02d239bf 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -13,7 +13,7 @@ pub(crate) mod stack; pub mod exp; use crate::prelude::{Activate, ActivateDyn, Forward, Node}; -use ndarray::prelude::{Array2, Ix2, NdFloat}; +use ndarray::prelude::{Array2, Ix2,}; // use ndarray::IntoDimension; use num::Float; diff --git a/ml/neural/src/lib.rs b/ml/neural/src/lib.rs index 95b592b8..c5cbbcb1 100644 --- a/ml/neural/src/lib.rs +++ b/ml/neural/src/lib.rs @@ -13,7 +13,6 @@ pub(crate) mod primitives; pub(crate) mod specs; pub(crate) mod utils; -pub mod arch; pub mod errors; pub mod func; pub mod layers; @@ -26,7 +25,6 @@ pub mod params; pub(crate) use concision_core as core; pub mod prelude { - pub use crate::arch::*; pub use crate::errors::*; pub use crate::func::{activate::*, loss::*, prop::*, rms::*}; pub use crate::layers::*; diff --git a/ml/neural/src/models/exp/mod.rs b/ml/neural/src/models/exp/mod.rs index 2d02f8ad..a63d0c95 100644 --- a/ml/neural/src/models/exp/mod.rs +++ b/ml/neural/src/models/exp/mod.rs @@ -8,6 +8,24 @@ pub use self::{modules::*, store::*, utils::*}; pub(crate) mod modules; pub(crate) mod store; +use crate::prelude::Predict; +use num::Float; + +pub trait Model: Predict where T: Float { + type Config; + + fn name(&self) -> &str; + + fn modules(&self) -> &Vec>>; + + fn modules_mut(&mut self) -> &mut Vec>>; + + fn register(&mut self, module: Box>) -> &mut Self { + self.modules_mut().push(module); + self + } +} + pub(crate) mod utils {} #[cfg(test)] diff --git a/ml/neural/src/models/exp/modules.rs b/ml/neural/src/models/exp/modules.rs index f49b7bf8..e21df4b2 100644 --- a/ml/neural/src/models/exp/modules.rs +++ b/ml/neural/src/models/exp/modules.rs @@ -4,27 +4,23 @@ */ //! # Model //! -use crate::prelude::Forward; +use crate::prelude::Predict; use ndarray::prelude::Array2; use num::Float; +use std::collections::HashMap; -pub trait Module: Forward, Output = Array2> -where - T: Float, -{ - type Config; +pub type ModuleParams = HashMap>; - fn add_module(&mut self, module: impl Module); - fn compile(&mut self); - /// Returns a collection of all proceeding [Module]s in the network - fn children(&self) -> &Vec>; - - fn children_mut(&mut self) -> &mut Vec>; - /// Returns a collection of all [Module]s in the network - fn modules(&self) -> Vec<&impl Module>; +pub trait Module: Predict +where + T: Float, +{ fn name(&self) -> &str; - fn parameters(&self) -> Vec<&Array2>; + fn parameters(&self) -> &ModuleParams<&str, T>; + + fn parameters_mut(&mut self) -> &mut ModuleParams<&str, T>; } + diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index 1b79c0ab..b77e1488 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -61,6 +61,14 @@ where } } +impl Predict for Box> where D: Dimension, T: Float, { + type Output = O; + + fn predict(&self, input: &Array) -> BoxResult { + self.as_ref().predict(input) + } +} + pub trait Train where T: Float, diff --git a/ml/nlp/Cargo.toml b/ml/nlp/Cargo.toml index 0955eea6..d0ddf256 100644 --- a/ml/nlp/Cargo.toml +++ b/ml/nlp/Cargo.toml @@ -32,7 +32,7 @@ serde.workspace = true serde_json.workspace = true smart-default.workspace = true strum.workspace = true -tokenizers = { features = [], version = "0.14" } +tokenizers = { features = [], version = "0.15" } [dev-dependencies] computare.workspace = true diff --git a/ml/optim/Cargo.toml b/ml/optim/Cargo.toml index 3275588c..aa66682a 100644 --- a/ml/optim/Cargo.toml +++ b/ml/optim/Cargo.toml @@ -24,7 +24,7 @@ test = true [build-dependencies] [dependencies] -concision-core.workspace = true +concision-core = { path = "../../core", version = "0.1.12" } concision-neural = { path = "../neural" } anyhow.workspace = true diff --git a/ml/s4/Cargo.toml b/ml/s4/Cargo.toml index de8dbb40..79db39e2 100644 --- a/ml/s4/Cargo.toml +++ b/ml/s4/Cargo.toml @@ -24,7 +24,7 @@ test = true [build-dependencies] [dependencies] -concision-core.workspace = true +concision-core = { path = "../../core", version = "0.1.12" } # concision-data = { path = "../../data" } concision-neural = { path = "../neural" } diff --git a/ml/s4/src/ssm/model.rs b/ml/s4/src/ssm/model.rs index c2ebec29..8c73e11f 100644 --- a/ml/s4/src/ssm/model.rs +++ b/ml/s4/src/ssm/model.rs @@ -43,14 +43,13 @@ where let ds = step / T::from(2).unwrap(); let eye = Array2::::eye(self.config.features()); let bl = &eye - &self.a * ds; - let pos = &eye + &self.a * ds; // positive - let mut bi = bl.view().into_faer().qr().inverse(); let be = { - let arr = &bi.view_mut().into_ndarray(); + let mut tmp = bl.view().into_faer().qr().inverse(); + let arr = &tmp.view_mut().into_ndarray(); arr.mapv(|i| T::from(i).unwrap()) }; - let ab = &be.dot(&pos); - let bb = (&self.b * ds).dot(&self.b); + let ab = &be.dot(&(&eye + &self.a * ds)); + let bb = (&self.b * ds).dot(&self.b.t()); Ok(()) } diff --git a/ml/transformers/Cargo.toml b/ml/transformers/Cargo.toml index 6aa804a4..ecab8ede 100644 --- a/ml/transformers/Cargo.toml +++ b/ml/transformers/Cargo.toml @@ -24,7 +24,7 @@ test = true [build-dependencies] [dependencies] -concision-core.workspace = true +concision-core = { path = "../../core", version = "0.1.12" } # concision-data = { path = "../../data" } concision-neural = { path = "../neural" } diff --git a/ml/transformers/src/transform/params.rs b/ml/transformers/src/transform/config.rs similarity index 100% rename from ml/transformers/src/transform/params.rs rename to ml/transformers/src/transform/config.rs diff --git a/ml/transformers/src/transform/mod.rs b/ml/transformers/src/transform/mod.rs index 7510cc93..b3bb0b33 100644 --- a/ml/transformers/src/transform/mod.rs +++ b/ml/transformers/src/transform/mod.rs @@ -3,9 +3,9 @@ Contrib: FL03 */ //! # Transform -pub use self::{params::*, transformer::*, utils::*}; +pub use self::{config::*, transformer::*, utils::*}; -pub(crate) mod params; +pub(crate) mod config; pub(crate) mod transformer; pub(crate) mod utils {} From 26758d67e9d90d3fbff9e00f8d0d02f4c5986164 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 16 Dec 2023 15:06:10 -0600 Subject: [PATCH 086/118] update Signed-off-by: FL03 --- core/src/params/group.rs | 4 +- core/src/params/iter.rs | 8 + core/src/params/kinds.rs | 3 - core/src/params/mod.rs | 11 +- core/src/params/param.rs | 2 +- core/src/params/store.rs | 28 ++- core/src/specs.rs | 4 - ml/neural/src/layers/exp/mod.rs | 2 - ml/neural/src/layers/mod.rs | 2 +- ml/neural/src/models/exp/mod.rs | 16 +- ml/neural/src/models/exp/modules.rs | 105 +++++++++- ml/neural/src/models/exp/store.rs | 10 +- ml/neural/src/specs.rs | 6 +- ml/optim/src/grad/adam.rs | 310 ++++++++++++++++++++++++++++ ml/optim/src/grad/mod.rs | 1 + ml/s4/src/lib.rs | 4 + ml/s4/src/ops/discretize.rs | 41 ++++ ml/s4/src/ops/mod.rs | 10 + ml/s4/src/ssm/config.rs | 47 ++++- ml/s4/src/ssm/mod.rs | 4 +- ml/s4/src/ssm/model.rs | 31 ++- ml/s4/src/ssm/params.rs | 35 ---- ml/s4/src/ssm/params/kinds.rs | 64 ++++++ ml/s4/src/ssm/params/mod.rs | 111 ++++++++++ ml/s4/src/ssm/params/store.rs | 205 ++++++++++++++++++ 25 files changed, 979 insertions(+), 85 deletions(-) create mode 100644 core/src/params/iter.rs create mode 100644 ml/optim/src/grad/adam.rs create mode 100644 ml/s4/src/ops/discretize.rs create mode 100644 ml/s4/src/ops/mod.rs delete mode 100644 ml/s4/src/ssm/params.rs create mode 100644 ml/s4/src/ssm/params/kinds.rs create mode 100644 ml/s4/src/ssm/params/mod.rs create mode 100644 ml/s4/src/ssm/params/store.rs diff --git a/core/src/params/group.rs b/core/src/params/group.rs index 19f678ff..1bbe51a0 100644 --- a/core/src/params/group.rs +++ b/core/src/params/group.rs @@ -103,9 +103,9 @@ where } } -impl Biased for ParamGroup +impl Biased for ParamGroup where - D: Dimension + RemoveAxis, + D: RemoveAxis, T: Float, { fn bias(&self) -> &Array { diff --git a/core/src/params/iter.rs b/core/src/params/iter.rs new file mode 100644 index 00000000..359553ef --- /dev/null +++ b/core/src/params/iter.rs @@ -0,0 +1,8 @@ +/* + Appellation: iter + Contrib: FL03 +*/ + +pub struct Iter; + +pub struct IterMut; diff --git a/core/src/params/kinds.rs b/core/src/params/kinds.rs index e53ed35f..660fa41b 100644 --- a/core/src/params/kinds.rs +++ b/core/src/params/kinds.rs @@ -6,12 +6,9 @@ use serde::{Deserialize, Serialize}; use strum::{EnumIs, EnumIter, EnumString, EnumVariantNames}; pub trait ParamType: ToString { - fn kind(&self) -> String; } - - #[derive( Clone, Debug, diff --git a/core/src/params/mod.rs b/core/src/params/mod.rs index 4571ef8e..cd6d4f25 100644 --- a/core/src/params/mod.rs +++ b/core/src/params/mod.rs @@ -6,9 +6,10 @@ //! //! ## Overview //! -pub use self::{group::*, kinds::*, param::*, store::*, utils::*}; +pub use self::{group::*, iter::*, kinds::*, param::*, store::*}; pub(crate) mod group; +pub(crate) mod iter; pub(crate) mod kinds; pub(crate) mod param; pub(crate) mod store; @@ -28,11 +29,11 @@ where T: Float, { /// Returns an owned reference to the bias of the layer. - fn bias(&self) -> &Array; + fn bias(&self) -> &Array; /// Returns a mutable reference to the bias of the layer. - fn bias_mut(&mut self) -> &mut Array; + fn bias_mut(&mut self) -> &mut Array; /// Sets the bias of the layer. - fn set_bias(&mut self, bias: Array); + fn set_bias(&mut self, bias: Array); } pub trait Weighted @@ -61,8 +62,6 @@ where fn set_params(&mut self, params: Array); } -pub(crate) mod utils {} - #[cfg(test)] mod tests { use super::*; diff --git a/core/src/params/param.rs b/core/src/params/param.rs index daa43f8c..673de647 100644 --- a/core/src/params/param.rs +++ b/core/src/params/param.rs @@ -4,9 +4,9 @@ */ use super::{Param, ParamKind}; use crate::prelude::GenerateRandom; +use ndarray::linalg::Dot; use ndarray::prelude::{Array, Dimension, Ix2}; use ndarray::IntoDimension; -use ndarray::linalg::Dot; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; diff --git a/core/src/params/store.rs b/core/src/params/store.rs index 3eb722cf..d8bf4d3a 100644 --- a/core/src/params/store.rs +++ b/core/src/params/store.rs @@ -9,11 +9,19 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] -pub struct ParamStore where T: Float, D: Dimension { +pub struct ParamStore +where + T: Float, + D: Dimension, +{ store: HashMap>, } -impl ParamStore where D: Dimension, T: Float, { +impl ParamStore +where + D: Dimension, + T: Float, +{ pub fn new() -> Self { Self { store: HashMap::new(), @@ -23,11 +31,13 @@ impl ParamStore where D: Dimension, T: Float, { pub fn insert(&mut self, param: Parameter) { self.store.insert(param.kind().clone(), param); } - } - -impl Extend> for ParamStore where D: Dimension, T: Float, { +impl Extend> for ParamStore +where + D: Dimension, + T: Float, +{ fn extend>>(&mut self, iter: I) { for param in iter { self.insert(param); @@ -35,7 +45,11 @@ impl Extend> for ParamStore where D: Dimension, T: F } } -impl IntoIterator for ParamStore where D: Dimension, T: Float, { +impl IntoIterator for ParamStore +where + D: Dimension, + T: Float, +{ type Item = (ParamKind, Parameter); type IntoIter = std::collections::hash_map::IntoIter>; @@ -44,7 +58,6 @@ impl IntoIterator for ParamStore where D: Dimension, T: Float, { } } - #[cfg(test)] mod tests { use super::*; @@ -56,6 +69,5 @@ mod tests { let shapes = [(inputs, outputs), (outputs, outputs), (outputs, 1)]; let params = ParamStore::::new(); - } } diff --git a/core/src/specs.rs b/core/src/specs.rs index de4005a5..5df18a49 100644 --- a/core/src/specs.rs +++ b/core/src/specs.rs @@ -12,10 +12,6 @@ use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Uniform}; use ndarray_rand::RandomExt; use num::Float; -pub trait Discritize { - fn discritize(&self, step: T) -> Self; -} - pub trait Apply { fn apply(&self, f: F) -> Self where diff --git a/ml/neural/src/layers/exp/mod.rs b/ml/neural/src/layers/exp/mod.rs index aecf9766..86c283d2 100644 --- a/ml/neural/src/layers/exp/mod.rs +++ b/ml/neural/src/layers/exp/mod.rs @@ -10,8 +10,6 @@ pub(crate) mod layer; pub(crate) mod sublayer; pub(crate) mod wrapper; - - pub(crate) mod utils {} #[cfg(test)] diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index 02d239bf..cf28b291 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -13,7 +13,7 @@ pub(crate) mod stack; pub mod exp; use crate::prelude::{Activate, ActivateDyn, Forward, Node}; -use ndarray::prelude::{Array2, Ix2,}; +use ndarray::prelude::{Array2, Ix2}; // use ndarray::IntoDimension; use num::Float; diff --git a/ml/neural/src/models/exp/mod.rs b/ml/neural/src/models/exp/mod.rs index a63d0c95..51aa3c3d 100644 --- a/ml/neural/src/models/exp/mod.rs +++ b/ml/neural/src/models/exp/mod.rs @@ -8,22 +8,30 @@ pub use self::{modules::*, store::*, utils::*}; pub(crate) mod modules; pub(crate) mod store; -use crate::prelude::Predict; +use crate::prelude::Forward; +use ndarray::prelude::Array2; use num::Float; -pub trait Model: Predict where T: Float { +pub trait Model: Forward> +where + T: Float, +{ type Config; - + fn name(&self) -> &str; fn modules(&self) -> &Vec>>; fn modules_mut(&mut self) -> &mut Vec>>; - fn register(&mut self, module: Box>) -> &mut Self { + fn register_module(&mut self, module: Box>) -> &mut Self { self.modules_mut().push(module); self } + + fn get_module(&self, name: &str) -> Option<&Box>> { + self.modules().iter().find(|m| m.name() == name) + } } pub(crate) mod utils {} diff --git a/ml/neural/src/models/exp/modules.rs b/ml/neural/src/models/exp/modules.rs index e21df4b2..4b987e34 100644 --- a/ml/neural/src/models/exp/modules.rs +++ b/ml/neural/src/models/exp/modules.rs @@ -4,23 +4,116 @@ */ //! # Model //! -use crate::prelude::Predict; -use ndarray::prelude::Array2; +use crate::prelude::{Forward, Weighted}; +use ndarray::prelude::{Array2, NdFloat}; use num::Float; use std::collections::HashMap; pub type ModuleParams = HashMap>; +pub struct M(Box>>); - -pub trait Module: Predict +pub trait Module: Forward> where T: Float, { + fn get_param(&self, name: &str) -> Option<&Array2> { + self.parameters().get(name) + } + fn name(&self) -> &str; - fn parameters(&self) -> &ModuleParams<&str, T>; + fn parameters(&self) -> &ModuleParams; + + fn parameters_mut(&mut self) -> &mut ModuleParams; +} + +pub trait ModuleExt: Module +where + T: Float, +{ +} + +pub struct LinearModel { + params: ModuleParams, +} + +impl LinearModel +where + T: Float, +{ + pub fn new() -> Self { + Self { + params: ModuleParams::new(), + } + } + + pub fn biased(&self) -> bool { + self.params.contains_key("bias") + } + + pub fn weighted(&self) -> bool { + self.params.contains_key("weight") + } - fn parameters_mut(&mut self) -> &mut ModuleParams<&str, T>; + pub fn with_weight(mut self, weight: Array2) -> Self { + self.params.insert("weight".to_string(), weight); + self + } + + pub fn with_bias(mut self, bias: Array2) -> Self { + self.params.insert("bias".to_string(), bias); + self + } } +impl Module for LinearModel +where + T: NdFloat, +{ + fn name(&self) -> &str { + "LinearModel" + } + + fn parameters(&self) -> &ModuleParams { + &self.params + } + + fn parameters_mut(&mut self) -> &mut ModuleParams { + &mut self.params + } +} + +impl Weighted for LinearModel +where + T: NdFloat, +{ + fn weights(&self) -> &Array2 { + &self.params["weight"] + } + + fn weights_mut(&mut self) -> &mut Array2 { + self.params.get_mut("weight").unwrap() + } + + fn set_weights(&mut self, weights: Array2) { + self.params.insert("weight".to_string(), weights); + } +} + +impl Forward> for LinearModel +where + T: NdFloat, +{ + type Output = Array2; + + fn forward(&self, args: &Array2) -> Array2 { + if let Some(weight) = self.params.get("weight") { + if let Some(bias) = self.params.get("bias") { + return args.dot(&weight.t()) + bias; + } + return args.dot(&weight.t()); + } + args.clone() + } +} diff --git a/ml/neural/src/models/exp/store.rs b/ml/neural/src/models/exp/store.rs index 392f3311..758755f7 100644 --- a/ml/neural/src/models/exp/store.rs +++ b/ml/neural/src/models/exp/store.rs @@ -12,14 +12,14 @@ use num::Float; use std::collections::HashMap; // use std::ops; -pub struct ModelMap +pub struct ModelStore where T: Float, { store: HashMap>, } -impl ModelMap +impl ModelStore where T: Float, { @@ -72,7 +72,7 @@ where } } -impl ModelMap +impl ModelStore where T: Float + SampleUniform, { @@ -98,7 +98,7 @@ where } } -impl IntoIterator for ModelMap +impl IntoIterator for ModelStore where T: Float, { @@ -120,7 +120,7 @@ mod tests { let shapes = [(inputs, outputs), (outputs, outputs), (outputs, 1)]; - let params = ModelMap::::new().build_layers(shapes).init(true); + let params = ModelStore::::new().build_layers(shapes).init(true); // validate the dimensions of the model params // assert!(params.validate_shapes()); diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index b77e1488..e48fdfd5 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -61,7 +61,11 @@ where } } -impl Predict for Box> where D: Dimension, T: Float, { +impl Predict for Box> +where + D: Dimension, + T: Float, +{ type Output = O; fn predict(&self, input: &Array) -> BoxResult { diff --git a/ml/optim/src/grad/adam.rs b/ml/optim/src/grad/adam.rs new file mode 100644 index 00000000..8da534e1 --- /dev/null +++ b/ml/optim/src/grad/adam.rs @@ -0,0 +1,310 @@ +/* + Appellation: adam + Contrib: FL03 +*/ +//! # Adam (Adaptive Moment Estimation) optimizer +//! +//! The `Adam (Adaptive Moment Estimation)` optimizer is an adaptive learning rate algorithm used +//! in gradient descent and machine learning, such as for training neural networks to solve deep +//! learning problems. Boasting memory-efficient fast convergence rates, it sets and iteratively +//! updates learning rates individually for each model parameter based on the gradient history. +//! +//! ## Algorithm: +//! +//! Given: +//! - α is the learning rate +//! - (β_1, β_2) are the exponential decay rates for moment estimates +//! - ϵ is any small value to prevent division by zero +//! - g_t are the gradients at time step t +//! - m_t are the biased first moment estimates of the gradient at time step t +//! - v_t are the biased second raw moment estimates of the gradient at time step t +//! - θ_t are the model parameters at time step t +//! - t is the time step +//! +//! Required: +//! θ_0 +//! +//! Initialize: +//! m_0 <- 0 +//! v_0 <- 0 +//! t <- 0 +//! +//! while θ_t not converged do +//! m_t = β_1 * m_{t−1} + (1 − β_1) * g_t +//! v_t = β_2 * v_{t−1} + (1 − β_2) * g_t^2 +//! m_hat_t = m_t / 1 - β_1^t +//! v_hat_t = v_t / 1 - β_2^t +//! θ_t = θ_{t-1} − α * m_hat_t / (sqrt(v_hat_t) + ϵ) +//! +//! ## Resources: +//! - Adam: A Method for Stochastic Optimization (by Diederik P. Kingma and Jimmy Ba): +//! - [https://arxiv.org/abs/1412.6980] +//! - PyTorch Adam optimizer: +//! - [https://pytorch.org/docs/stable/generated/torch.optim.Adam.html#torch.optim.Adam] +//! +use ndarray::prelude::{Array1, NdFloat}; +use num::Float; + +pub struct Adam +where + T: Float, +{ + learning_rate: T, // alpha: initial step size for iterative optimization + betas: (T, T), // betas: exponential decay rates for moment estimates + epsilon: T, // epsilon: prevent division by zero + m: Array1, // m: biased first moment estimate of the gradient vector + v: Array1, // v: biased second raw moment estimate of the gradient vector + t: usize, // t: time step +} + +impl Adam +where + T: Float, +{ + pub fn new( + learning_rate: Option, + betas: Option<(T, T)>, + epsilon: Option, + params_len: usize, + ) -> Self { + let gamma = T::from(1e-3).unwrap(); + let betas = betas.unwrap_or((T::from(0.9).unwrap(), T::from(0.999).unwrap())); + let epsilon = epsilon.unwrap_or(T::from(1e-8).unwrap()); + Adam { + learning_rate: learning_rate.unwrap_or(gamma), // typical good default lr + betas, // typical good default decay rates + epsilon, // typical good default epsilon + m: Array1::zeros(params_len), // first moment vector elements all initialized to zero + v: Array1::zeros(params_len), // second moment vector elements all initialized to zero + t: 0, // time step initialized to zero + } + } +} + +impl Adam +where + T: NdFloat, +{ + pub fn step(&mut self, gradients: &Array1) -> Array1 { + let mut model_params = Array1::zeros(gradients.len()); + self.t += 1; + + for i in 0..gradients.len() { + // update biased first moment estimate and second raw moment estimate + self.m[i] = self.betas.0 * self.m[i] + (T::one() - self.betas.0) * gradients[i]; + self.v[i] = self.betas.1 * self.v[i] + (T::one() - self.betas.1) * gradients[i].powi(2); + + // compute bias-corrected first moment estimate and second raw moment estimate + let m_hat = self.m[i] / (T::one() - self.betas.0.powi(self.t as i32)); + let v_hat = self.v[i] / (T::one() - self.betas.1.powi(self.t as i32)); + + // update model parameters + model_params[i] -= self.learning_rate * m_hat / (v_hat.sqrt() + self.epsilon); + } + model_params // return updated model parameters + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ndarray::prelude::array; + + #[test] + fn test_adam_init_default_values() { + let optimizer = Adam::new(None, None, None, 1); + + assert_eq!(optimizer.learning_rate, 0.001); + assert_eq!(optimizer.betas, (0.9, 0.999)); + assert_eq!(optimizer.epsilon, 1e-8); + assert_eq!(optimizer.m, array![0.0]); + assert_eq!(optimizer.v, array![0.0]); + assert_eq!(optimizer.t, 0); + } + + #[test] + fn test_adam_init_custom_lr_value() { + let optimizer = Adam::new(Some(0.9), None, None, 2); + + assert_eq!(optimizer.learning_rate, 0.9); + assert_eq!(optimizer.betas, (0.9, 0.999)); + assert_eq!(optimizer.epsilon, 1e-8); + assert_eq!(optimizer.m, array![0.0, 0.0]); + assert_eq!(optimizer.v, array![0.0, 0.0]); + assert_eq!(optimizer.t, 0); + } + + #[test] + fn test_adam_init_custom_betas_value() { + let optimizer = Adam::new(None, Some((0.8, 0.899)), None, 3); + + assert_eq!(optimizer.learning_rate, 0.001); + assert_eq!(optimizer.betas, (0.8, 0.899)); + assert_eq!(optimizer.epsilon, 1e-8); + assert_eq!(optimizer.m, array![0.0, 0.0, 0.0]); + assert_eq!(optimizer.v, array![0.0, 0.0, 0.0]); + assert_eq!(optimizer.t, 0); + } + + #[test] + fn test_adam_init_custom_epsilon_value() { + let optimizer = Adam::new(None, None, Some(1e-10), 4); + + assert_eq!(optimizer.learning_rate, 0.001); + assert_eq!(optimizer.betas, (0.9, 0.999)); + assert_eq!(optimizer.epsilon, 1e-10); + assert_eq!(optimizer.m, array![0.0, 0.0, 0.0, 0.0]); + assert_eq!(optimizer.v, array![0.0, 0.0, 0.0, 0.0]); + assert_eq!(optimizer.t, 0); + } + + #[test] + fn test_adam_init_all_custom_values() { + let optimizer = Adam::new(Some(1.0), Some((0.001, 0.099)), Some(1e-1), 5); + + assert_eq!(optimizer.learning_rate, 1.0); + assert_eq!(optimizer.betas, (0.001, 0.099)); + assert_eq!(optimizer.epsilon, 1e-1); + assert_eq!(optimizer.m, Array1::::zeros(5)); + assert_eq!(optimizer.v, Array1::::zeros(5)); + assert_eq!(optimizer.t, 0); + } + + #[test] + fn test_adam_step_default_params() { + let gradients = array![-1.0, 2.0, -3.0, 4.0, -5.0, 6.0, -7.0, 8.0]; + + let mut optimizer = Adam::new(None, None, None, 8); + let updated_params = optimizer.step(&gradients); + + assert_eq!( + updated_params, + array![ + 0.0009999999900000003, + -0.000999999995, + 0.0009999999966666666, + -0.0009999999975, + 0.000999999998, + -0.0009999999983333334, + 0.0009999999985714286, + -0.00099999999875 + ] + ); + } + + #[test] + fn test_adam_step_custom_params() { + let gradients = array![9.0, -8.0, 7.0, -6.0, 5.0, -4.0, 3.0, -2.0, 1.0]; + + let mut optimizer = Adam::new(Some(0.005), Some((0.5, 0.599)), Some(1e-5), 9); + let updated_params = optimizer.step(&gradients); + + assert_eq!( + updated_params, + array![ + -0.004999994444450618, + 0.004999993750007813, + -0.004999992857153062, + 0.004999991666680556, + -0.004999990000020001, + 0.004999987500031251, + -0.004999983333388888, + 0.004999975000124999, + -0.0049999500004999945 + ] + ); + } + + #[test] + fn test_adam_step_empty_gradients_array() { + let gradients = Array1::::zeros(0); + + let mut optimizer = Adam::new(None, None, None, 0); + let updated_params = optimizer.step(&gradients); + + assert_eq!(updated_params, Array1::::zeros(0)); + } + + #[ignore] + #[test] + fn test_adam_step_iteratively_until_convergence_with_default_params() { + const CONVERGENCE_THRESHOLD: f64 = 1e-5; + let gradients = array![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + + let mut optimizer = Adam::new(None, None, None, 6); + + let mut model_params = Array1::::zeros(6); + let mut updated_params: Array1 = optimizer.step(&gradients); + + while (updated_params + .iter() + .zip(model_params.iter()) + .map(|(x, y)| x - y) + .collect::>()) + .iter() + .map(|&x| x.powi(2)) + .sum::() + .sqrt() + > CONVERGENCE_THRESHOLD + { + model_params = updated_params; + updated_params = optimizer.step(&gradients); + } + + // assert!(updated_params < Array1::::from_elem(6, CONVERGENCE_THRESHOLD)); + assert_ne!(updated_params, model_params); + assert_eq!( + updated_params, + array![ + -0.0009999999899999931, + -0.0009999999949999929, + -0.0009999999966666597, + -0.0009999999974999929, + -0.0009999999979999927, + -0.0009999999983333263 + ] + ); + } + + #[ignore] + #[test] + fn test_adam_step_iteratively_until_convergence_with_custom_params() { + const CONVERGENCE_THRESHOLD: f64 = 1e-7; + let gradients = array![7.0, -8.0, 9.0, -10.0, 11.0, -12.0, 13.0]; + + let mut optimizer = Adam::new(Some(0.005), Some((0.8, 0.899)), Some(1e-5), 7); + + let mut model_params = Array1::::zeros(7); + let mut updated_params = optimizer.step(&gradients); + + while (updated_params + .iter() + .zip(model_params.iter()) + .map(|(x, y)| x - y) + .collect::>()) + .iter() + .map(|&x| x.powi(2)) + .sum::() + .sqrt() + > CONVERGENCE_THRESHOLD + { + model_params = updated_params; + updated_params = optimizer.step(&gradients); + } + + // assert!(updated_params < vec![CONVERGENCE_THRESHOLD; 7]); + assert_ne!(updated_params, model_params); + assert_eq!( + updated_params, + array![ + -0.004999992857153061, + 0.004999993750007814, + -0.0049999944444506185, + 0.004999995000005001, + -0.004999995454549587, + 0.004999995833336807, + -0.004999996153849113 + ] + ); + } +} diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index 19399f01..671365b2 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -9,6 +9,7 @@ pub(crate) mod descent; pub(crate) mod gradient; pub(crate) mod modes; +pub mod adam; pub mod sgd; pub struct BatchParams { diff --git a/ml/s4/src/lib.rs b/ml/s4/src/lib.rs index c8bd4853..b968910d 100644 --- a/ml/s4/src/lib.rs +++ b/ml/s4/src/lib.rs @@ -11,6 +11,7 @@ pub(crate) mod primitives; pub(crate) mod specs; pub(crate) mod utils; +pub mod ops; pub mod ssm; pub(crate) use concision_core as core; @@ -20,4 +21,7 @@ pub mod prelude { pub use crate::primitives::*; pub use crate::specs::*; pub use crate::utils::*; + + pub use crate::ops::*; + pub use crate::ssm::*; } diff --git a/ml/s4/src/ops/discretize.rs b/ml/s4/src/ops/discretize.rs new file mode 100644 index 00000000..2280dbe5 --- /dev/null +++ b/ml/s4/src/ops/discretize.rs @@ -0,0 +1,41 @@ +/* + Appellation: discretize + Contrib: FL03 +*/ +use faer::prelude::{FaerMat, IntoFaer, SolverCore}; +use faer::IntoNdarray; +use faer_core::zip::ViewMut; +use faer_core::{ComplexField, Conjugate, SimpleEntity}; +use ndarray::prelude::{Array2, NdFloat}; +use num::ToPrimitive; + +pub fn discretize( + a: &Array2, + b: &Array2, + c: &Array2, + _d: &Array2, + step: T, +) -> anyhow::Result<(Array2, Array2, Array2)> +where + T: NdFloat + Conjugate + SimpleEntity, + ::Canonical: ComplexField + SimpleEntity + ToPrimitive, +{ + let ss = step / T::from(2).unwrap(); // half step + let eye = Array2::::eye(a.shape()[0]); + let bl = &eye - a * ss; + let be = { + let mut tmp = bl.view().into_faer().qr().inverse(); + let arr = &tmp.view_mut().into_ndarray(); + arr.mapv(|i| T::from(i).unwrap()) + }; + let ab = be.dot(&(&eye + a * ss)); + let bb = (b * ss).dot(&b.t()); + + Ok((ab, bb, c.clone())) +} + +pub struct Discretized { + pub a: Array2, + pub b: Array2, + pub c: Array2, +} diff --git a/ml/s4/src/ops/mod.rs b/ml/s4/src/ops/mod.rs new file mode 100644 index 00000000..c6ebc5e6 --- /dev/null +++ b/ml/s4/src/ops/mod.rs @@ -0,0 +1,10 @@ +/* + Appellation: ops + Contrib: FL03 +*/ +pub use self::discretize::*; + +pub(crate) mod discretize; + +#[cfg(test)] +mod tests {} diff --git a/ml/s4/src/ssm/config.rs b/ml/s4/src/ssm/config.rs index 2fbb8cd7..b416b46a 100644 --- a/ml/s4/src/ssm/config.rs +++ b/ml/s4/src/ssm/config.rs @@ -2,17 +2,23 @@ Appellation: config Contrib: FL03 */ +use num::Float; use serde::{Deserialize, Serialize}; #[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] pub struct SSMConfig { pub decode: bool, - pub features: usize, + pub features: usize, // n + pub samples: usize, // l_max } impl SSMConfig { - pub fn new(decode: bool, features: usize) -> Self { - Self { decode, features } + pub fn new(decode: bool, features: usize, samples: usize) -> Self { + Self { + decode, + features, + samples, + } } pub fn decode(&self) -> bool { @@ -22,4 +28,39 @@ impl SSMConfig { pub fn features(&self) -> usize { self.features } + + pub fn samples(&self) -> usize { + self.samples + } + + pub fn step(&self) -> T { + T::one() / T::from(self.samples).unwrap() + } + + pub fn set_decode(&mut self, decode: bool) { + self.decode = decode; + } + + pub fn set_features(&mut self, features: usize) { + self.features = features; + } + + pub fn set_samples(&mut self, samples: usize) { + self.samples = samples; + } + + pub fn with_decode(mut self, decode: bool) -> Self { + self.decode = decode; + self + } + + pub fn with_features(mut self, features: usize) -> Self { + self.features = features; + self + } + + pub fn with_samples(mut self, samples: usize) -> Self { + self.samples = samples; + self + } } diff --git a/ml/s4/src/ssm/mod.rs b/ml/s4/src/ssm/mod.rs index 8c836dfb..4d16bcf0 100644 --- a/ml/s4/src/ssm/mod.rs +++ b/ml/s4/src/ssm/mod.rs @@ -25,8 +25,8 @@ mod tests { fn test_ssm() { let step = 0.001; - let config = SSMConfig::new(true, 9); + let config = SSMConfig::new(true, 9, 2); let mut model = SSM::::create(config); - assert!(model.descretize(step).is_ok()); + assert!(model.discretize(step).is_ok()); } } diff --git a/ml/s4/src/ssm/model.rs b/ml/s4/src/ssm/model.rs index 8c73e11f..be91e0a0 100644 --- a/ml/s4/src/ssm/model.rs +++ b/ml/s4/src/ssm/model.rs @@ -7,11 +7,38 @@ use faer::prelude::{FaerMat, IntoFaer, SolverCore}; use faer::IntoNdarray; use faer_core::zip::ViewMut; use faer_core::{ComplexField, Conjugate, SimpleEntity}; -use ndarray::prelude::{Array2, NdFloat}; +use ndarray::prelude::{s, Array1, Array2, ArrayView1, NdFloat}; +use ndarray::IntoDimension; // use ndarray_linalg::solve::Inverse; // use ndarray_linalg::types::Lapack; use num::{Float, ToPrimitive}; +pub type ScanFn = Box Option>; + +pub fn scanner( + a: &Array2, + b: &Array2, + c: &Array2, + u: &Array2, + x0: &Array1, +) -> Array2 +where + T: NdFloat, +{ + let step = |xs: &mut Array1, us: ArrayView1| { + let x1 = a.dot(xs) + b.dot(&us); + let y1 = c.dot(&x1); + Some(y1) + }; + let scan = u.outer_iter().scan(x0.clone(), step).collect::>(); + let shape = [scan.len(), scan[0].len()]; + let mut res = Array2::::zeros(shape.into_dimension()); + for (i, s) in scan.iter().enumerate() { + res.slice_mut(s![i, ..]).assign(s); + } + res +} + pub struct SSM { config: SSMConfig, pub a: Array2, @@ -39,7 +66,7 @@ where T: NdFloat + Conjugate + SimpleEntity, ::Canonical: ComplexField + SimpleEntity + ToPrimitive, { - pub fn descretize(&mut self, step: T) -> anyhow::Result<()> { + pub fn discretize(&mut self, step: T) -> anyhow::Result<()> { let ds = step / T::from(2).unwrap(); let eye = Array2::::eye(self.config.features()); let bl = &eye - &self.a * ds; diff --git a/ml/s4/src/ssm/params.rs b/ml/s4/src/ssm/params.rs deleted file mode 100644 index 22fe77eb..00000000 --- a/ml/s4/src/ssm/params.rs +++ /dev/null @@ -1,35 +0,0 @@ -/* - Appellation: params - Contrib: FL03 -*/ -use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; - -#[derive( - Clone, - Copy, - Debug, - Default, - Deserialize, - Display, - EnumIs, - EnumIter, - EnumString, - EnumVariantNames, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize, -)] -#[repr(usize)] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum SSMParams { - #[default] - A, - B, - C, - D, -} diff --git a/ml/s4/src/ssm/params/kinds.rs b/ml/s4/src/ssm/params/kinds.rs new file mode 100644 index 00000000..bd029ef6 --- /dev/null +++ b/ml/s4/src/ssm/params/kinds.rs @@ -0,0 +1,64 @@ +/* + Appellation: kinds + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames, VariantNames}; + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum SSMParams { + #[default] + A = 0, + B = 1, + C = 2, + D = 3, +} + +impl SSMParams { + pub fn a() -> Self { + Self::A + } + + pub fn b() -> Self { + Self::B + } + + pub fn c() -> Self { + Self::C + } + + pub fn d() -> Self { + Self::D + } +} + +impl From for SSMParams { + fn from(i: usize) -> Self { + match i % SSMParams::VARIANTS.len() { + 0 => Self::A, + 1 => Self::B, + 2 => Self::C, + _ => Self::D, + } + } +} diff --git a/ml/s4/src/ssm/params/mod.rs b/ml/s4/src/ssm/params/mod.rs new file mode 100644 index 00000000..bcbe43ba --- /dev/null +++ b/ml/s4/src/ssm/params/mod.rs @@ -0,0 +1,111 @@ +/* + Appellation: store + Contrib: FL03 +*/ +pub use self::{kinds::*, store::*}; + +pub(crate) mod kinds; +pub(crate) mod store; + +use ndarray::prelude::Array2; +use num::Float; +use std::collections::HashMap; + +pub type SSMMap = HashMap>; + +pub trait SSMParamGroup +where + T: Float, +{ + fn a(&self) -> &Array2; + fn b(&self) -> &Array2; + fn c(&self) -> &Array2; + fn d(&self) -> &Array2; +} + +impl SSMParamGroup for SSMStore +where + T: Float, +{ + fn a(&self) -> &Array2 { + &self.a + } + + fn b(&self) -> &Array2 { + &self.b + } + + fn c(&self) -> &Array2 { + &self.c + } + + fn d(&self) -> &Array2 { + &self.d + } +} + +impl SSMParamGroup for SSMMap +where + T: Float, +{ + fn a(&self) -> &Array2 { + self.get(&SSMParams::A).unwrap() + } + + fn b(&self) -> &Array2 { + self.get(&SSMParams::B).unwrap() + } + + fn c(&self) -> &Array2 { + self.get(&SSMParams::C).unwrap() + } + + fn d(&self) -> &Array2 { + self.get(&SSMParams::D).unwrap() + } +} + +impl SSMParamGroup for &[Array2; 4] +where + T: Float, +{ + fn a(&self) -> &Array2 { + &self[0] + } + + fn b(&self) -> &Array2 { + &self[1] + } + + fn c(&self) -> &Array2 { + &self[2] + } + + fn d(&self) -> &Array2 { + &self[3] + } +} + +impl SSMParamGroup for (Array2, Array2, Array2, Array2) +where + T: Float, +{ + fn a(&self) -> &Array2 { + &self.0 + } + + fn b(&self) -> &Array2 { + &self.1 + } + + fn c(&self) -> &Array2 { + &self.2 + } + + fn d(&self) -> &Array2 { + &self.3 + } +} + +#[cfg(test)] +mod tests {} diff --git a/ml/s4/src/ssm/params/store.rs b/ml/s4/src/ssm/params/store.rs new file mode 100644 index 00000000..da4e210f --- /dev/null +++ b/ml/s4/src/ssm/params/store.rs @@ -0,0 +1,205 @@ +/* + Appellation: store + Contrib: FL03 +*/ +use super::SSMParams; +use ndarray::prelude::Array2; +use num::Float; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::ops; + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct SSMStore +where + T: Float, +{ + pub(crate) a: Array2, + pub(crate) b: Array2, + pub(crate) c: Array2, + pub(crate) d: Array2, +} + +impl SSMStore +where + T: Float, +{ + pub fn new(a: Array2, b: Array2, c: Array2, d: Array2) -> Self { + Self { a, b, c, d } + } + + pub fn from_features(features: usize) -> Self + where + T: Default, + { + let a = Array2::::default((features, features)); + let b = Array2::::default((features, 1)); + let c = Array2::::default((1, features)); + let d = Array2::::default((1, 1)); + Self::new(a, b, c, d) + } + + pub fn ones(features: usize) -> Self { + let a = Array2::::ones((features, features)); + let b = Array2::::ones((features, 1)); + let c = Array2::::ones((1, features)); + let d = Array2::::ones((1, 1)); + Self::new(a, b, c, d) + } + + pub fn zeros(features: usize) -> Self { + let a = Array2::::zeros((features, features)); + let b = Array2::::zeros((features, 1)); + let c = Array2::::zeros((1, features)); + let d = Array2::::zeros((1, 1)); + Self::new(a, b, c, d) + } + + pub fn a(&self) -> &Array2 { + &self.a + } + + pub fn b(&self) -> &Array2 { + &self.b + } + + pub fn c(&self) -> &Array2 { + &self.c + } + + pub fn d(&self) -> &Array2 { + &self.d + } + + pub fn a_mut(&mut self) -> &mut Array2 { + &mut self.a + } + + pub fn b_mut(&mut self) -> &mut Array2 { + &mut self.b + } + + pub fn c_mut(&mut self) -> &mut Array2 { + &mut self.c + } + + pub fn d_mut(&mut self) -> &mut Array2 { + &mut self.d + } +} + +impl ops::Index for SSMStore +where + T: Float, +{ + type Output = Array2; + + fn index(&self, index: SSMParams) -> &Self::Output { + use SSMParams::*; + match index { + A => self.a(), + B => self.b(), + C => self.c(), + D => self.d(), + } + } +} + +impl ops::IndexMut for SSMStore +where + T: Float, +{ + fn index_mut(&mut self, index: SSMParams) -> &mut Self::Output { + use SSMParams::*; + match index { + A => self.a_mut(), + B => self.b_mut(), + C => self.c_mut(), + D => self.d_mut(), + } + } +} + +impl From> for (Array2, Array2, Array2, Array2) +where + T: Float, +{ + fn from(store: SSMStore) -> Self { + (store.a, store.b, store.c, store.d) + } +} + +impl From<(Array2, Array2, Array2, Array2)> for SSMStore +where + T: Float, +{ + fn from((a, b, c, d): (Array2, Array2, Array2, Array2)) -> Self { + Self::new(a, b, c, d) + } +} + +impl From> for HashMap> +where + T: Float, +{ + fn from(store: SSMStore) -> Self { + let mut map = HashMap::new(); + + map.insert(SSMParams::A, store.a); + map.insert(SSMParams::B, store.b); + map.insert(SSMParams::C, store.c); + map.insert(SSMParams::D, store.d); + map + } +} + +impl FromIterator<(SSMParams, Array2)> for SSMStore +where + T: Default + Float, +{ + fn from_iter)>>(iter: I) -> Self { + let tmp = HashMap::>::from_iter(iter); + if tmp.is_empty() { + Self::from_features(1) + } else { + let a = tmp + .get(&SSMParams::A) + .unwrap_or(&Array2::::default((1, 1))) + .clone(); + let b = tmp + .get(&SSMParams::B) + .unwrap_or(&Array2::::default((1, 1))) + .clone(); + let c = tmp + .get(&SSMParams::C) + .unwrap_or(&Array2::::default((1, 1))) + .clone(); + let d = tmp + .get(&SSMParams::D) + .unwrap_or(&Array2::::default((1, 1))) + .clone(); + Self::new(a, b, c, d) + } + } +} + +impl IntoIterator for SSMStore +where + T: Float, +{ + type Item = (SSMParams, Array2); + type IntoIter = std::collections::hash_map::IntoIter>; + + fn into_iter(self) -> Self::IntoIter { + HashMap::from(self).into_iter() + } +} + +// impl<'a, T> IntoIterator for &'a mut SSMStore where T: Float { +// type Item = (&'a SSMParams, &'a mut Array2); +// type IntoIter = std::collections::hash_map::IterMut<'a, SSMParams, Array2>; + +// fn into_iter(self) -> Self::IntoIter { +// HashMap::from(self).iter_mut() +// } +// } From 83cccb433fb38e327cd449957dd5ba5716fe92a0 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 17 Dec 2023 12:33:15 -0600 Subject: [PATCH 087/118] update Signed-off-by: FL03 --- Cargo.toml | 3 +- core/src/params/mod.rs | 28 ++++++++--- ml/neural/src/layers/linear/conv.rs | 4 ++ ml/neural/src/layers/linear/dense.rs | 4 ++ ml/neural/src/layers/linear/mod.rs | 9 ++++ ml/neural/src/layers/mod.rs | 1 + ml/optim/Cargo.toml | 1 + ml/s4/Cargo.toml | 4 +- ml/s4/src/lib.rs | 6 ++- ml/s4/src/model/config.rs | 71 ++++++++++++++++++++++++++++ ml/s4/src/model/mod.rs | 8 ++++ ml/s4/src/model/model.rs | 58 +++++++++++++++++++++++ ml/s4/src/ops/convolve.rs | 6 +++ ml/s4/src/ops/discretize.rs | 6 +++ ml/s4/src/ops/mod.rs | 3 +- ml/s4/src/{ssm => }/params/kinds.rs | 5 +- ml/s4/src/{ssm => }/params/mod.rs | 0 ml/s4/src/{ssm => }/params/store.rs | 30 +++++++++++- ml/s4/src/ssm/config.rs | 2 +- ml/s4/src/ssm/mod.rs | 7 ++- ml/s4/src/ssm/model.rs | 41 +++++----------- ml/s4/src/utils.rs | 54 +++++++++++++++++++++ 22 files changed, 302 insertions(+), 49 deletions(-) create mode 100644 ml/neural/src/layers/linear/conv.rs create mode 100644 ml/neural/src/layers/linear/dense.rs create mode 100644 ml/neural/src/layers/linear/mod.rs create mode 100644 ml/s4/src/model/config.rs create mode 100644 ml/s4/src/model/mod.rs create mode 100644 ml/s4/src/model/model.rs create mode 100644 ml/s4/src/ops/convolve.rs rename ml/s4/src/{ssm => }/params/kinds.rs (87%) rename ml/s4/src/{ssm => }/params/mod.rs (100%) rename ml/s4/src/{ssm => }/params/store.rs (85%) diff --git a/Cargo.toml b/Cargo.toml index da5dc88c..fae03fa1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,11 +14,12 @@ version = "0.1.12" # TODO - Update the cargo package version computare = { features = ["full"], branch = "v0.1.0", git = "https://github.com/FL03/computare", version = "0.1.0" } anyhow = "1" - +approx = "0.5" faer = { features = ["ndarray"], version = "0.16" } faer-core = "0.16" itertools = { features = [], version = "0.12" } lazy_static = "1" +nalgebra = { features = [], version = "0.32" } ndarray = { features = ["serde-1"], version = "0.15" } # ndarray-linalg = { features = [], version = "0.16" } ndarray-rand = { features = [], version = "0.14" } diff --git a/core/src/params/mod.rs b/core/src/params/mod.rs index cd6d4f25..e191d6f8 100644 --- a/core/src/params/mod.rs +++ b/core/src/params/mod.rs @@ -16,6 +16,7 @@ pub(crate) mod store; use ndarray::prelude::{Array, Dimension, Ix2}; use num::Float; +use std::collections::HashMap; pub trait Param { fn kind(&self) -> &ParamKind; @@ -49,17 +50,30 @@ where fn set_weights(&mut self, weights: Array); } -pub trait Params +pub trait Params where D: Dimension, T: Float, + Self: IntoIterator)>, { - /// Returns an owned reference to the parameters of the layer. - fn params(&self) -> &Array; - /// Returns a mutable reference to the parameters of the layer. - fn params_mut(&mut self) -> &mut Array; - /// Sets the parameters of the layer. - fn set_params(&mut self, params: Array); + fn get(&self, param: &K) -> Option<&Array>; + + fn get_mut(&mut self, param: &K) -> Option<&mut Array>; +} + +impl Params for HashMap> +where + D: Dimension, + K: std::cmp::Eq + std::hash::Hash, + T: Float, +{ + fn get(&self, param: &K) -> Option<&Array> { + self.get(param) + } + + fn get_mut(&mut self, param: &K) -> Option<&mut Array> { + self.get_mut(param) + } } #[cfg(test)] diff --git a/ml/neural/src/layers/linear/conv.rs b/ml/neural/src/layers/linear/conv.rs new file mode 100644 index 00000000..db9dd16e --- /dev/null +++ b/ml/neural/src/layers/linear/conv.rs @@ -0,0 +1,4 @@ +/* + Appellation: dense + Contrib: FL03 +*/ diff --git a/ml/neural/src/layers/linear/dense.rs b/ml/neural/src/layers/linear/dense.rs new file mode 100644 index 00000000..db9dd16e --- /dev/null +++ b/ml/neural/src/layers/linear/dense.rs @@ -0,0 +1,4 @@ +/* + Appellation: dense + Contrib: FL03 +*/ diff --git a/ml/neural/src/layers/linear/mod.rs b/ml/neural/src/layers/linear/mod.rs new file mode 100644 index 00000000..1494a09d --- /dev/null +++ b/ml/neural/src/layers/linear/mod.rs @@ -0,0 +1,9 @@ +/* + Appellation: linear + Contrib: FL03 +*/ +//! # Linear Layers +//! + +pub mod conv; +pub mod dense; diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index cf28b291..96a34103 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -11,6 +11,7 @@ pub(crate) mod params; pub(crate) mod stack; pub mod exp; +pub mod linear; use crate::prelude::{Activate, ActivateDyn, Forward, Node}; use ndarray::prelude::{Array2, Ix2}; diff --git a/ml/optim/Cargo.toml b/ml/optim/Cargo.toml index aa66682a..d7839974 100644 --- a/ml/optim/Cargo.toml +++ b/ml/optim/Cargo.toml @@ -28,6 +28,7 @@ concision-core = { path = "../../core", version = "0.1.12" } concision-neural = { path = "../neural" } anyhow.workspace = true +itertools.workspace = true lazy_static.workspace = true ndarray.workspace = true # ndarray-linalg.workspace = true diff --git a/ml/s4/Cargo.toml b/ml/s4/Cargo.toml index 79db39e2..6af189b7 100644 --- a/ml/s4/Cargo.toml +++ b/ml/s4/Cargo.toml @@ -33,17 +33,19 @@ faer-core.workspace = true faer.workspace = true lazy_static.workspace = true ndarray.workspace = true +ndarray-conv = "0.2" # ndarray-linalg.workspace = true ndarray-rand.workspace = true ndarray-stats.workspace = true num.workspace = true +rustfft = { features = [], version = "6" } serde.workspace = true serde_json.workspace = true smart-default.workspace = true strum.workspace = true [dev-dependencies] -approx = "0.5" +approx.workspace = true [package.metadata.docs.rs] all-features = true diff --git a/ml/s4/src/lib.rs b/ml/s4/src/lib.rs index b968910d..2284f0cf 100644 --- a/ml/s4/src/lib.rs +++ b/ml/s4/src/lib.rs @@ -5,13 +5,15 @@ //! # Structured State Space Sequential Models (S4) //! //! -pub use self::{primitives::*, specs::*, utils::*}; +pub use self::{model::*, primitives::*, specs::*, utils::*}; +pub(crate) mod model; pub(crate) mod primitives; pub(crate) mod specs; pub(crate) mod utils; pub mod ops; +pub mod params; pub mod ssm; pub(crate) use concision_core as core; @@ -22,6 +24,8 @@ pub mod prelude { pub use crate::specs::*; pub use crate::utils::*; + pub use crate::model::*; pub use crate::ops::*; + pub use crate::params::*; pub use crate::ssm::*; } diff --git a/ml/s4/src/model/config.rs b/ml/s4/src/model/config.rs new file mode 100644 index 00000000..0496671a --- /dev/null +++ b/ml/s4/src/model/config.rs @@ -0,0 +1,71 @@ +/* + Appellation: config + Contrib: FL03 +*/ +use num::{Complex, Float}; +use serde::{Deserialize, Serialize}; + +pub struct S4Training { + pub lambda: Complex, + pub log_step: f64, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct S4Config { + pub decode: bool, + pub features: usize, // n + pub samples: usize, // l_max +} + +impl S4Config { + pub fn new(decode: bool, features: usize, samples: usize) -> Self { + Self { + decode, + features, + samples, + } + } + + pub fn decode(&self) -> bool { + self.decode + } + + pub fn features(&self) -> usize { + self.features + } + + pub fn samples(&self) -> usize { + self.samples + } + + pub fn step_size(&self) -> T { + T::one() / T::from(self.samples).unwrap() + } + + pub fn set_decode(&mut self, decode: bool) { + self.decode = decode; + } + + pub fn set_features(&mut self, features: usize) { + self.features = features; + } + + pub fn set_samples(&mut self, samples: usize) { + self.samples = samples; + } + + pub fn with_decode(mut self, decode: bool) -> Self { + self.decode = decode; + self + } + + pub fn with_features(mut self, features: usize) -> Self { + self.features = features; + self + } + + pub fn with_samples(mut self, samples: usize) -> Self { + self.samples = samples; + self + } +} diff --git a/ml/s4/src/model/mod.rs b/ml/s4/src/model/mod.rs new file mode 100644 index 00000000..914b6f48 --- /dev/null +++ b/ml/s4/src/model/mod.rs @@ -0,0 +1,8 @@ +/* + Appellation: model + Contrib: FL03 +*/ +pub use self::{config::*, model::*}; + +pub(crate) mod config; +pub(crate) mod model; diff --git a/ml/s4/src/model/model.rs b/ml/s4/src/model/model.rs new file mode 100644 index 00000000..09b97581 --- /dev/null +++ b/ml/s4/src/model/model.rs @@ -0,0 +1,58 @@ +/* + Appellation: model + Contrib: FL03 +*/ +use super::S4Config; +use crate::neural::prelude::Forward; +use crate::prelude::SSMStore; +use ndarray::prelude::{Array1, Array2, NdFloat}; +use ndarray_conv::Conv2DFftExt; +use num::Float; + +use crate::prelude::SSMParams::*; + +pub struct S4 +where + T: Float, +{ + cache: Array1, + config: S4Config, + kernal: Option>, + store: SSMStore, +} + +impl S4 +where + T: Float, +{ + pub fn new(config: S4Config) -> Self + where + T: Default, + { + let n = config.features(); + let cache = Array1::::zeros((n,)); + let kernal = None; + let store = SSMStore::from_features(n); + Self { + cache, + config, + kernal, + store, + } + } +} + +impl Forward> for S4 +where + T: NdFloat, +{ + type Output = Array2; + + fn forward(&self, args: &Array2) -> Self::Output { + if !self.config.decode { + unimplemented!() + } + let scan = self.store.scan(args, &self.cache); + scan + args * &self.store[D] + } +} diff --git a/ml/s4/src/ops/convolve.rs b/ml/s4/src/ops/convolve.rs new file mode 100644 index 00000000..f30b156f --- /dev/null +++ b/ml/s4/src/ops/convolve.rs @@ -0,0 +1,6 @@ +/* + Appellation: convolve + Contrib: FL03 +*/ + +pub fn convolve() {} diff --git a/ml/s4/src/ops/discretize.rs b/ml/s4/src/ops/discretize.rs index 2280dbe5..c0f9f443 100644 --- a/ml/s4/src/ops/discretize.rs +++ b/ml/s4/src/ops/discretize.rs @@ -34,6 +34,12 @@ where Ok((ab, bb, c.clone())) } +pub enum DiscretizeArgs {} + +pub struct Discretize { + pub step: T, +} + pub struct Discretized { pub a: Array2, pub b: Array2, diff --git a/ml/s4/src/ops/mod.rs b/ml/s4/src/ops/mod.rs index c6ebc5e6..3eddf70a 100644 --- a/ml/s4/src/ops/mod.rs +++ b/ml/s4/src/ops/mod.rs @@ -2,8 +2,9 @@ Appellation: ops Contrib: FL03 */ -pub use self::discretize::*; +pub use self::{convolve::*, discretize::*}; +pub(crate) mod convolve; pub(crate) mod discretize; #[cfg(test)] diff --git a/ml/s4/src/ssm/params/kinds.rs b/ml/s4/src/params/kinds.rs similarity index 87% rename from ml/s4/src/ssm/params/kinds.rs rename to ml/s4/src/params/kinds.rs index bd029ef6..5651dd95 100644 --- a/ml/s4/src/ssm/params/kinds.rs +++ b/ml/s4/src/params/kinds.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames, VariantNames}; +use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; #[derive( Clone, @@ -12,6 +12,7 @@ use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames, VariantName Default, Deserialize, Display, + EnumCount, EnumIs, EnumIter, EnumString, @@ -54,7 +55,7 @@ impl SSMParams { impl From for SSMParams { fn from(i: usize) -> Self { - match i % SSMParams::VARIANTS.len() { + match i % SSMParams::COUNT { 0 => Self::A, 1 => Self::B, 2 => Self::C, diff --git a/ml/s4/src/ssm/params/mod.rs b/ml/s4/src/params/mod.rs similarity index 100% rename from ml/s4/src/ssm/params/mod.rs rename to ml/s4/src/params/mod.rs diff --git a/ml/s4/src/ssm/params/store.rs b/ml/s4/src/params/store.rs similarity index 85% rename from ml/s4/src/ssm/params/store.rs rename to ml/s4/src/params/store.rs index da4e210f..92f86804 100644 --- a/ml/s4/src/ssm/params/store.rs +++ b/ml/s4/src/params/store.rs @@ -3,12 +3,17 @@ Contrib: FL03 */ use super::SSMParams; -use ndarray::prelude::Array2; +use crate::core::prelude::GenerateRandom; +use crate::prelude::scanner; +use ndarray::prelude::{Array1, Array2, NdFloat}; +use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::ops; +pub struct B(Box>>); + #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct SSMStore where @@ -88,6 +93,29 @@ where } } +impl SSMStore +where + T: NdFloat, +{ + pub fn scan(&self, u: &Array2, x0: &Array1) -> Array2 { + scanner(&self.a, &self.b, &self.c, u, x0) + } +} + +impl SSMStore +where + T: Float + SampleUniform, +{ + pub fn uniform(features: usize) -> Self { + let dk = T::one() / T::from(features).unwrap().sqrt(); + let a = Array2::::uniform_between(dk, (features, features)); + let b = Array2::::uniform_between(dk, (features, 1)); + let c = Array2::::uniform_between(dk, (1, features)); + let d = Array2::::ones((1, 1)); + Self::new(a, b, c, d) + } +} + impl ops::Index for SSMStore where T: Float, diff --git a/ml/s4/src/ssm/config.rs b/ml/s4/src/ssm/config.rs index b416b46a..69349ada 100644 --- a/ml/s4/src/ssm/config.rs +++ b/ml/s4/src/ssm/config.rs @@ -33,7 +33,7 @@ impl SSMConfig { self.samples } - pub fn step(&self) -> T { + pub fn step_size(&self) -> T { T::one() / T::from(self.samples).unwrap() } diff --git a/ml/s4/src/ssm/mod.rs b/ml/s4/src/ssm/mod.rs index 4d16bcf0..bb1df0da 100644 --- a/ml/s4/src/ssm/mod.rs +++ b/ml/s4/src/ssm/mod.rs @@ -5,17 +5,16 @@ //! # State Space Models (SSM) //! //! -pub use self::{config::*, model::*, params::*, utils::*}; +pub use self::{config::*, model::*}; pub(crate) mod config; pub(crate) mod model; -pub(crate) mod params; pub trait StateSpace { fn features(&self) -> usize; -} -pub(crate) mod utils {} + fn scan(&self, step: f64) -> Result<(), String>; +} #[cfg(test)] mod tests { diff --git a/ml/s4/src/ssm/model.rs b/ml/s4/src/ssm/model.rs index be91e0a0..cf08390b 100644 --- a/ml/s4/src/ssm/model.rs +++ b/ml/s4/src/ssm/model.rs @@ -3,42 +3,14 @@ Contrib: FL03 */ use super::SSMConfig; +use crate::prelude::scanner; use faer::prelude::{FaerMat, IntoFaer, SolverCore}; use faer::IntoNdarray; use faer_core::zip::ViewMut; use faer_core::{ComplexField, Conjugate, SimpleEntity}; -use ndarray::prelude::{s, Array1, Array2, ArrayView1, NdFloat}; -use ndarray::IntoDimension; -// use ndarray_linalg::solve::Inverse; -// use ndarray_linalg::types::Lapack; +use ndarray::prelude::{Array1, Array2, NdFloat}; use num::{Float, ToPrimitive}; -pub type ScanFn = Box Option>; - -pub fn scanner( - a: &Array2, - b: &Array2, - c: &Array2, - u: &Array2, - x0: &Array1, -) -> Array2 -where - T: NdFloat, -{ - let step = |xs: &mut Array1, us: ArrayView1| { - let x1 = a.dot(xs) + b.dot(&us); - let y1 = c.dot(&x1); - Some(y1) - }; - let scan = u.outer_iter().scan(x0.clone(), step).collect::>(); - let shape = [scan.len(), scan[0].len()]; - let mut res = Array2::::zeros(shape.into_dimension()); - for (i, s) in scan.iter().enumerate() { - res.slice_mut(s![i, ..]).assign(s); - } - res -} - pub struct SSM { config: SSMConfig, pub a: Array2, @@ -61,6 +33,15 @@ where } } +impl SSM +where + T: NdFloat, +{ + pub fn scan(&self, u: &Array2, x0: &Array1) -> Array2 { + scanner(&self.a, &self.b, &self.c, u, x0) + } +} + impl SSM where T: NdFloat + Conjugate + SimpleEntity, diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index 752dabaf..8191babf 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -2,3 +2,57 @@ Appellation: utils Contrib: FL03 */ +use ndarray::prelude::{s, Array1, Array2, ArrayView1, Axis, NdFloat}; +use ndarray::IntoDimension; +use rustfft::{FftNum, FftPlanner}; + +pub fn casual_colvolution(a: &Array2, b: &Array2) -> Array2 +where + T: FftNum, +{ + let mut planner = FftPlanner::::new(); + let fft = planner.plan_fft_forward(a.len()); + + a.clone() +} + +pub fn k_convolve(a: &Array2, b: &Array2, c: &Array2, l: usize) -> Array2 +where + T: FftNum, +{ + let b = b.clone().remove_axis(Axis(1)); + let mut res = Array2::::zeros((l, a.shape()[0])); + for i in 0..l { + let mut tmp = a.clone(); + for _ in 0..i { + tmp = tmp.dot(a); + } + let out = c.dot(&tmp.dot(&b)); + res.slice_mut(s![i, ..]).assign(&out); + } + res +} + +pub fn scanner( + a: &Array2, + b: &Array2, + c: &Array2, + u: &Array2, + x0: &Array1, +) -> Array2 +where + T: NdFloat, +{ + let step = |xs: &mut Array1, us: ArrayView1| { + let x1 = a.dot(xs) + b.dot(&us); + let y1 = c.dot(&x1); + Some(y1) + }; + let scan = u.outer_iter().scan(x0.clone(), step).collect::>(); + let shape = [scan.len(), scan[0].len()]; + let mut res = Array2::::zeros(shape.into_dimension()); + for (i, s) in scan.iter().enumerate() { + res.slice_mut(s![i, ..]).assign(s); + } + res +} From 94144c641a6334c74368edba99d4cc79d91a0077 Mon Sep 17 00:00:00 2001 From: FL03 Date: Mon, 18 Dec 2023 15:23:19 -0600 Subject: [PATCH 088/118] update Signed-off-by: FL03 --- Cargo.toml | 5 +- concision/Cargo.toml | 6 + concision/src/lib.rs | 4 + core/src/utils.rs | 16 +- ml/linear/Cargo.toml | 52 +++++ ml/linear/benches/default.rs | 52 +++++ .../dense.rs => linear/src/conv/mod.rs} | 2 +- .../conv.rs => linear/src/dense/mod.rs} | 0 ml/linear/src/lib.rs | 18 ++ ml/linear/src/model/config.rs | 10 + ml/linear/src/model/mod.rs | 7 + ml/linear/src/params/features.rs | 105 +++++++++ ml/linear/src/params/kinds.rs | 84 ++++++++ ml/linear/src/params/mod.rs | 9 + ml/linear/src/params/store.rs | 200 ++++++++++++++++++ ml/linear/tests/default.rs | 8 + ml/neural/src/layers/cmp/mod.rs | 4 +- ml/neural/src/layers/exp/mod.rs | 4 +- ml/neural/src/layers/linear/mod.rs | 9 - ml/neural/src/layers/mod.rs | 1 - ml/neural/src/neurons/mod.rs | 4 +- ml/neural/src/params/group.rs | 6 +- ml/s4/src/model/model.rs | 11 +- ml/s4/src/ops/convolve.rs | 16 ++ ml/s4/src/utils.rs | 34 ++- 25 files changed, 620 insertions(+), 47 deletions(-) create mode 100644 ml/linear/Cargo.toml create mode 100644 ml/linear/benches/default.rs rename ml/{neural/src/layers/linear/dense.rs => linear/src/conv/mod.rs} (61%) rename ml/{neural/src/layers/linear/conv.rs => linear/src/dense/mod.rs} (100%) create mode 100644 ml/linear/src/lib.rs create mode 100644 ml/linear/src/model/config.rs create mode 100644 ml/linear/src/model/mod.rs create mode 100644 ml/linear/src/params/features.rs create mode 100644 ml/linear/src/params/kinds.rs create mode 100644 ml/linear/src/params/mod.rs create mode 100644 ml/linear/src/params/store.rs create mode 100644 ml/linear/tests/default.rs delete mode 100644 ml/neural/src/layers/linear/mod.rs diff --git a/Cargo.toml b/Cargo.toml index fae03fa1..c60f5816 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,9 +41,10 @@ members = [ "concision", "core", "data", - "derive", + "derive", "macros", # "ml/*", + "ml/linear", "ml/ml", "ml/neural", "ml/nlp", @@ -74,4 +75,4 @@ lto = false panic = 'unwind' incremental = false codegen-units = 16 -rpath = false \ No newline at end of file +rpath = false diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 9dffc331..dec77489 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -42,7 +42,12 @@ macros = [ "concision-macros" ] +linear = [ + "concision-linear" +] + ml = [ + "linear", "neural", "nlp", "optim", @@ -85,6 +90,7 @@ concision-derive = { features = [], optional = true, path = "../derive", version concision-macros = { features = [], optional = true, path = "../macros", version = "0.1.12" } # concision-ml = { features = [], optional = true, path = "../ml/ml", version = "0.1.12" } +concision-linear = { features = [], optional = true, path = "../ml/linear", version = "0.1.12" } concision-neural = { features = [], optional = true, path = "../ml/neural", version = "0.1.12" } concision-nlp = { features = [], optional = true, path = "../ml/nlp", version = "0.1.12" } concision-optim = { features = [], optional = true, path = "../ml/optim", version = "0.1.12" } diff --git a/concision/src/lib.rs b/concision/src/lib.rs index f3f1bbf6..a57392dd 100644 --- a/concision/src/lib.rs +++ b/concision/src/lib.rs @@ -13,6 +13,8 @@ pub use concision_core as core; pub use concision_data as data; #[cfg(feature = "derive")] pub use concision_derive::*; +#[cfg(feature = "linear")] +pub use concision_linear as linear; #[cfg(feature = "macros")] pub use concision_macros::*; // #[cfg(feature = "ml")] @@ -35,6 +37,8 @@ pub mod prelude { pub use concision_data::prelude::*; #[cfg(feature = "derive")] pub use concision_derive::*; + #[cfg(feature = "linear")] + pub use concision_linear::prelude::*; #[cfg(feature = "macros")] pub use concision_macros::*; // #[cfg(feature = "ml")] diff --git a/core/src/utils.rs b/core/src/utils.rs index b72e6324..5c45d6f6 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -3,10 +3,24 @@ Contrib: FL03 */ -use ndarray::prelude::{Array, Axis, Dimension}; +use ndarray::prelude::{Array, Array1, Axis, Dimension}; use ndarray::{concatenate, IntoDimension, RemoveAxis, ShapeError}; +use num::cast::AsPrimitive; use num::Float; +pub fn arange(a: T, b: T, h: T) -> Array1 +where + T: AsPrimitive + Float, +{ + let n: usize = ((b - a) / h).as_(); + let mut res = Array1::::zeros(n); + res[0] = a; + for i in 1..n { + res[i] = res[i - 1] + h; + } + res +} + pub fn concat_iter(axis: usize, iter: impl IntoIterator>) -> Array where D: RemoveAxis, diff --git a/ml/linear/Cargo.toml b/ml/linear/Cargo.toml new file mode 100644 index 00000000..ee2a13b6 --- /dev/null +++ b/ml/linear/Cargo.toml @@ -0,0 +1,52 @@ +[package] +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "concision-linear" +readme.workspace = true +repository.workspace = true +version.workspace = true + + +[features] +default = [] + +[lib] +bench = true +crate-type = ["cdylib", "rlib"] +doctest = false +test = true + +[build-dependencies] + +[dependencies] +concision-core = { path = "../../core", version = "0.1.12" } +concision-neural = { path = "../neural", version = "0.1.12" } + +anyhow.workspace = true +ndarray.workspace = true +ndarray-rand.workspace = true +ndarray-stats.workspace = true +num.workspace = true +petgraph.workspace = true +rand.workspace = true +serde.workspace = true +serde_json.workspace = true +smart-default.workspace = true +strum.workspace = true + +[dev-dependencies] +computare.workspace = true + +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] + +[target.wasm32-unknown-unknown] + +[target.wasm32-wasi] + diff --git a/ml/linear/benches/default.rs b/ml/linear/benches/default.rs new file mode 100644 index 00000000..937f2387 --- /dev/null +++ b/ml/linear/benches/default.rs @@ -0,0 +1,52 @@ +// bench.rs +#![feature(test)] + +extern crate test; + +use std::mem::replace; +use test::Bencher; + +// bench: find the `BENCH_SIZE` first terms of the fibonacci sequence +static BENCH_SIZE: usize = 20; + +// recursive fibonacci +fn fibonacci(n: usize) -> u32 { + if n < 2 { + 1 + } else { + fibonacci(n - 1) + fibonacci(n - 2) + } +} + +// iterative fibonacci +struct Fibonacci { + curr: u32, + next: u32, +} + +impl Iterator for Fibonacci { + type Item = u32; + fn next(&mut self) -> Option { + let new_next = self.curr + self.next; + let new_curr = replace(&mut self.next, new_next); + + Some(replace(&mut self.curr, new_curr)) + } +} + +fn fibonacci_sequence() -> Fibonacci { + Fibonacci { curr: 1, next: 1 } +} + +// function to benchmark must be annotated with `#[bench]` +#[bench] +fn recursive_fibonacci(b: &mut Bencher) { + // exact code to benchmark must be passed as a closure to the iter + // method of Bencher + b.iter(|| (0..BENCH_SIZE).map(fibonacci).collect::>()) +} + +#[bench] +fn iterative_fibonacci(b: &mut Bencher) { + b.iter(|| fibonacci_sequence().take(BENCH_SIZE).collect::>()) +} diff --git a/ml/neural/src/layers/linear/dense.rs b/ml/linear/src/conv/mod.rs similarity index 61% rename from ml/neural/src/layers/linear/dense.rs rename to ml/linear/src/conv/mod.rs index db9dd16e..c804f96e 100644 --- a/ml/neural/src/layers/linear/dense.rs +++ b/ml/linear/src/conv/mod.rs @@ -1,4 +1,4 @@ /* - Appellation: dense + Appellation: conv Contrib: FL03 */ diff --git a/ml/neural/src/layers/linear/conv.rs b/ml/linear/src/dense/mod.rs similarity index 100% rename from ml/neural/src/layers/linear/conv.rs rename to ml/linear/src/dense/mod.rs diff --git a/ml/linear/src/lib.rs b/ml/linear/src/lib.rs new file mode 100644 index 00000000..52683095 --- /dev/null +++ b/ml/linear/src/lib.rs @@ -0,0 +1,18 @@ +/* + Appellation: concision-linear + Contrib: FL03 +*/ +//! # concision-linear +//! +//! This library implements the framework for building linear models. +//! + +pub mod conv; +pub mod dense; +pub mod model; +pub mod params; + +pub(crate) use concision_core as core; +pub(crate) use concision_neural as neural; + +pub mod prelude {} diff --git a/ml/linear/src/model/config.rs b/ml/linear/src/model/config.rs new file mode 100644 index 00000000..d11823ee --- /dev/null +++ b/ml/linear/src/model/config.rs @@ -0,0 +1,10 @@ +/* + Appellation: config + Contrib: FL03 +*/ + +pub struct LinearConfig { + pub biased: bool, + pub features: (usize, usize), + pub name: String, +} diff --git a/ml/linear/src/model/mod.rs b/ml/linear/src/model/mod.rs new file mode 100644 index 00000000..88aa46a6 --- /dev/null +++ b/ml/linear/src/model/mod.rs @@ -0,0 +1,7 @@ +/* + Appellation: model + Contrib: FL03 +*/ +pub use self::config::*; + +pub(crate) mod config; diff --git a/ml/linear/src/params/features.rs b/ml/linear/src/params/features.rs new file mode 100644 index 00000000..bc631d23 --- /dev/null +++ b/ml/linear/src/params/features.rs @@ -0,0 +1,105 @@ +/* + Appellation: features + Contrib: FL03 +*/ +use crate::neural::prelude::Features; +use ndarray::prelude::{Dimension, Ix2}; +use ndarray::IntoDimension; +use serde::{Deserialize, Serialize}; + +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] +pub struct LayerShape { + pub inputs: usize, + pub outputs: usize, +} + +impl LayerShape { + pub fn new(inputs: usize, outputs: usize) -> Self { + Self { inputs, outputs } + } + + pub fn from_dimension(shape: impl IntoDimension) -> Self { + let dim = shape.into_dimension(); + let (outputs, inputs) = dim.into_pattern(); + Self::new(inputs, outputs) + } + + pub fn neuron(inputs: usize) -> Self { + Self::new(inputs, 1) + } + + pub fn uniform_scale(&self) -> T { + (T::one() / T::from(self.inputs()).unwrap()).sqrt() + } +} + +impl std::fmt::Display for LayerShape { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "({}, {})", self.inputs, self.outputs) + } +} + +impl Features for LayerShape { + fn inputs(&self) -> usize { + self.inputs + } + + fn outputs(&self) -> usize { + self.outputs + } +} + +impl IntoDimension for LayerShape { + type Dim = Ix2; + + fn into_dimension(self) -> Self::Dim { + ndarray::Ix2(self.outputs, self.inputs) + } +} + +impl From for Ix2 { + fn from(features: LayerShape) -> Self { + features.into_dimension() + } +} + +impl From for ndarray::IxDyn { + fn from(features: LayerShape) -> Self { + ndarray::IxDyn(&[features.outputs, features.inputs]) + } +} + +impl From for [usize; 2] { + fn from(features: LayerShape) -> Self { + [features.outputs, features.inputs] + } +} + +impl From<[usize; 2]> for LayerShape { + fn from(features: [usize; 2]) -> Self { + Self { + inputs: features[1], + outputs: features[0], + } + } +} + +impl From for (usize, usize) { + fn from(features: LayerShape) -> Self { + (features.outputs, features.inputs) + } +} + +impl From<(usize, usize)> for LayerShape { + fn from((inputs, outputs): (usize, usize)) -> Self { + Self { inputs, outputs } + } +} + +impl From for LayerShape { + fn from(inputs: usize) -> Self { + Self { inputs, outputs: 1 } + } +} diff --git a/ml/linear/src/params/kinds.rs b/ml/linear/src/params/kinds.rs new file mode 100644 index 00000000..660fa41b --- /dev/null +++ b/ml/linear/src/params/kinds.rs @@ -0,0 +1,84 @@ +/* + Appellation: kinds + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use strum::{EnumIs, EnumIter, EnumString, EnumVariantNames}; + +pub trait ParamType: ToString { + fn kind(&self) -> String; +} + +#[derive( + Clone, + Debug, + Default, + Deserialize, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[non_exhaustive] +#[repr(usize)] +#[serde(rename_all = "lowercase", tag = "kind")] +#[strum(serialize_all = "lowercase")] +pub enum ParamKind { + #[default] + Bias, + Weight, + Other(String), +} + +impl ParamKind { + pub fn bias() -> Self { + Self::Bias + } + + pub fn weight() -> Self { + Self::Weight + } + + pub fn other(name: impl ToString) -> Self { + Self::Other(name.to_string()) + } +} + +impl std::fmt::Display for ParamKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let content = match self { + ParamKind::Bias => "bias", + ParamKind::Weight => "weight", + ParamKind::Other(name) => name, + }; + write!(f, "{}", content) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + #[test] + fn test_param_kind_map() { + let name = "test"; + let other = ParamKind::other(name); + + let data = [ + (ParamKind::Bias, 0), + (ParamKind::Weight, 1), + (other.clone(), 2), + (ParamKind::other("mask"), 3), + ]; + let store = HashMap::::from_iter(data); + assert_eq!(store.get(&ParamKind::Bias), Some(&0)); + assert_eq!(store.get(&other), Some(&2)); + } +} diff --git a/ml/linear/src/params/mod.rs b/ml/linear/src/params/mod.rs new file mode 100644 index 00000000..e2ce6bd1 --- /dev/null +++ b/ml/linear/src/params/mod.rs @@ -0,0 +1,9 @@ +/* + Appellation: params + Contrib: FL03 +*/ +pub use self::{features::*, kinds::*, store::*}; + +pub(crate) mod features; +pub(crate) mod kinds; +pub(crate) mod store; diff --git a/ml/linear/src/params/store.rs b/ml/linear/src/params/store.rs new file mode 100644 index 00000000..27a2034a --- /dev/null +++ b/ml/linear/src/params/store.rs @@ -0,0 +1,200 @@ +/* + Appellation: params + Contrib: FL03 +*/ +use super::LayerShape; +use crate::core::prelude::GenerateRandom; +// use crate::core::params::{Biased, Weighted}; +use crate::neural::prelude::{Biased, Features, Forward, Node, Weighted}; +use ndarray::linalg::Dot; +use ndarray::prelude::{Array, Array1, Array2, Axis, Dimension, Ix2, NdFloat}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use num::Float; +use serde::{Deserialize, Serialize}; +use std::ops; + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct LayerParams { + bias: Array1, + pub features: LayerShape, + weights: Array2, +} + +impl LayerParams +where + T: Float, +{ + pub fn new(features: LayerShape) -> Self { + Self { + bias: Array1::zeros(features.outputs()), + features, + weights: Array2::zeros(features.out_by_in()), + } + } + + pub fn features(&self) -> &LayerShape { + &self.features + } + + pub fn features_mut(&mut self) -> &mut LayerShape { + &mut self.features + } + + pub fn set_node(&mut self, idx: usize, node: Node) { + self.bias_mut() + .index_axis_mut(Axis(0), idx) + .assign(&node.bias()); + + self.weights_mut() + .index_axis_mut(Axis(0), idx) + .assign(&node.weights()); + } + + pub fn with_bias(mut self, bias: Array1) -> Self { + self.bias = bias; + self + } + + pub fn with_weights(mut self, weights: Array2) -> Self { + self.weights = weights; + self + } +} + +impl LayerParams +where + T: Float + 'static, +{ + pub fn update_with_gradient(&mut self, gamma: T, gradient: &Array2) { + self.weights_mut().scaled_add(-gamma, gradient); + } +} + +impl LayerParams +where + T: NdFloat, +{ + pub fn reset(&mut self) { + self.bias *= T::zero(); + self.weights *= T::zero(); + } +} + +impl LayerParams +where + T: Float + SampleUniform, +{ + pub fn init(mut self, biased: bool) -> Self { + if biased { + self = self.init_bias(); + } + self.init_weight() + } + + pub fn init_bias(mut self) -> Self { + let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); + self.bias = Array1::uniform_between(dk, self.features().outputs()); + self + } + + pub fn init_weight(mut self) -> Self { + let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); + self.weights = Array2::uniform_between(dk, self.features().out_by_in()); + self + } +} + +impl Biased for LayerParams +where + T: Float, +{ + fn bias(&self) -> &Array1 { + &self.bias + } + + fn bias_mut(&mut self) -> &mut Array1 { + &mut self.bias + } + + fn set_bias(&mut self, bias: Array1) { + self.bias = bias; + } +} + +impl Weighted for LayerParams +where + T: Float, +{ + fn set_weights(&mut self, weights: Array2) { + self.weights = weights; + } + + fn weights(&self) -> &Array2 { + &self.weights + } + + fn weights_mut(&mut self) -> &mut Array2 { + &mut self.weights + } +} + +impl Features for LayerParams +where + T: Float, +{ + fn inputs(&self) -> usize { + self.features.inputs() + } + + fn outputs(&self) -> usize { + self.features.outputs() + } +} + +impl Forward> for LayerParams +where + D: Dimension, + T: NdFloat, + Array: Dot, Output = Array> + ops::Add, Output = Array>, +{ + type Output = Array; + + fn forward(&self, input: &Array) -> Self::Output { + input.dot(&self.weights().t().to_owned()) + self.bias().clone() + } +} + +impl IntoIterator for LayerParams +where + T: Float, +{ + type Item = Node; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.weights() + .axis_iter(Axis(0)) + .zip(self.bias().axis_iter(Axis(0))) + .map(|(w, b)| (w.to_owned(), b.to_owned()).into()) + .collect::>() + .into_iter() + } +} + +impl FromIterator> for LayerParams +where + T: Float, +{ + fn from_iter>>(nodes: I) -> Self { + let nodes = nodes.into_iter().collect::>(); + let mut iter = nodes.iter(); + let node = iter.next().unwrap(); + let shape = LayerShape::new(*node.features(), nodes.len()); + let mut params = LayerParams::new(shape); + params.set_node(0, node.clone()); + for (i, node) in iter.into_iter().enumerate() { + params.set_node(i + 1, node.clone()); + } + params + } +} diff --git a/ml/linear/tests/default.rs b/ml/linear/tests/default.rs new file mode 100644 index 00000000..0cac1eb5 --- /dev/null +++ b/ml/linear/tests/default.rs @@ -0,0 +1,8 @@ +#[cfg(test)] +#[test] +fn compiles() { + let f = |x: usize, y: usize| x + y; + + assert_eq!(f(10, 10), 20); + assert_ne!(f(1, 1), 3); +} diff --git a/ml/neural/src/layers/cmp/mod.rs b/ml/neural/src/layers/cmp/mod.rs index 47beedbb..3916a626 100644 --- a/ml/neural/src/layers/cmp/mod.rs +++ b/ml/neural/src/layers/cmp/mod.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ //! # Layers -pub use self::{features::*, kinds::*, utils::*}; +pub use self::{features::*, kinds::*}; pub(crate) mod features; pub(crate) mod kinds; @@ -101,7 +101,5 @@ where } } -pub(crate) mod utils {} - #[cfg(test)] mod tests {} diff --git a/ml/neural/src/layers/exp/mod.rs b/ml/neural/src/layers/exp/mod.rs index 86c283d2..89669e8b 100644 --- a/ml/neural/src/layers/exp/mod.rs +++ b/ml/neural/src/layers/exp/mod.rs @@ -3,14 +3,12 @@ Contrib: FL03 */ //! # Experimental Layers -pub use self::{config::*, layer::*, sublayer::*, utils::*, wrapper::*}; +pub use self::{config::*, layer::*, sublayer::*, wrapper::*}; pub(crate) mod config; pub(crate) mod layer; pub(crate) mod sublayer; pub(crate) mod wrapper; -pub(crate) mod utils {} - #[cfg(test)] mod tests {} diff --git a/ml/neural/src/layers/linear/mod.rs b/ml/neural/src/layers/linear/mod.rs deleted file mode 100644 index 1494a09d..00000000 --- a/ml/neural/src/layers/linear/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -/* - Appellation: linear - Contrib: FL03 -*/ -//! # Linear Layers -//! - -pub mod conv; -pub mod dense; diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index 96a34103..cf28b291 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -11,7 +11,6 @@ pub(crate) mod params; pub(crate) mod stack; pub mod exp; -pub mod linear; use crate::prelude::{Activate, ActivateDyn, Forward, Node}; use ndarray::prelude::{Array2, Ix2}; diff --git a/ml/neural/src/neurons/mod.rs b/ml/neural/src/neurons/mod.rs index 29d9cda9..563df644 100644 --- a/ml/neural/src/neurons/mod.rs +++ b/ml/neural/src/neurons/mod.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ //! # neurons -pub use self::{node::*, perceptron::*, synapse::*, utils::*}; +pub use self::{node::*, perceptron::*, synapse::*}; pub(crate) mod node; pub(crate) mod perceptron; @@ -33,8 +33,6 @@ where fn weights(&self) -> &Array1; } -pub(crate) mod utils {} - #[cfg(test)] mod tests { use super::*; diff --git a/ml/neural/src/params/group.rs b/ml/neural/src/params/group.rs index e59e0471..6c306aed 100644 --- a/ml/neural/src/params/group.rs +++ b/ml/neural/src/params/group.rs @@ -182,7 +182,7 @@ where impl Serialize for ParamGroup where T: Float + Serialize, - D: Dimension + RemoveAxis + Serialize, + D: RemoveAxis + Serialize, ::Smaller: Dimension + Serialize, { fn serialize(&self, serializer: Ser) -> Result @@ -212,9 +212,9 @@ where impl IntoIterator for ParamGroup where - D: Dimension + RemoveAxis, + D: RemoveAxis, T: Float, - ::Smaller: Dimension + RemoveAxis, + ::Smaller: RemoveAxis, { type Item = ( Array, diff --git a/ml/s4/src/model/model.rs b/ml/s4/src/model/model.rs index 09b97581..9e275f78 100644 --- a/ml/s4/src/model/model.rs +++ b/ml/s4/src/model/model.rs @@ -6,8 +6,9 @@ use super::S4Config; use crate::neural::prelude::Forward; use crate::prelude::SSMStore; use ndarray::prelude::{Array1, Array2, NdFloat}; -use ndarray_conv::Conv2DFftExt; +use ndarray_conv::{Conv2DFftExt, PaddingMode, PaddingSize}; use num::Float; +use rustfft::FftNum; use crate::prelude::SSMParams::*; @@ -44,13 +45,17 @@ where impl Forward> for S4 where - T: NdFloat, + T: FftNum + NdFloat, { type Output = Array2; fn forward(&self, args: &Array2) -> Self::Output { if !self.config.decode { - unimplemented!() + let mode = PaddingMode::<2, T>::Const(T::zero()); + let size = PaddingSize::Full; + return args + .conv_2d_fft(&self.kernal.clone().unwrap(), size, mode) + .expect("convolution failed"); } let scan = self.store.scan(args, &self.cache); scan + args * &self.store[D] diff --git a/ml/s4/src/ops/convolve.rs b/ml/s4/src/ops/convolve.rs index f30b156f..2ab15635 100644 --- a/ml/s4/src/ops/convolve.rs +++ b/ml/s4/src/ops/convolve.rs @@ -2,5 +2,21 @@ Appellation: convolve Contrib: FL03 */ +use crate::prelude::powmat; +use ndarray::prelude::{s, Array2, Axis, NdFloat}; pub fn convolve() {} + +pub fn k_convolve(a: &Array2, b: &Array2, c: &Array2, l: usize) -> Array2 +where + T: NdFloat, +{ + let b = b.clone().remove_axis(Axis(1)); + let mut res = Array2::::zeros((l, a.shape()[0])); + for i in 0..l { + let tmp = powmat(a, i); + let out = c.dot(&tmp.dot(&b)); + res.slice_mut(s![i, ..]).assign(&out); + } + res +} diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index 8191babf..6765507c 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -2,10 +2,25 @@ Appellation: utils Contrib: FL03 */ -use ndarray::prelude::{s, Array1, Array2, ArrayView1, Axis, NdFloat}; +use ndarray::prelude::{s, Array1, Array2, ArrayView1, NdFloat}; use ndarray::IntoDimension; +use num::Float; use rustfft::{FftNum, FftPlanner}; +pub fn powmat(a: &Array2, n: usize) -> Array2 +where + T: Float + 'static, +{ + if !a.is_square() { + panic!("Matrix must be square"); + } + let mut res = a.clone(); + for _ in 0..n { + res = res.dot(a); + } + res +} + pub fn casual_colvolution(a: &Array2, b: &Array2) -> Array2 where T: FftNum, @@ -16,23 +31,6 @@ where a.clone() } -pub fn k_convolve(a: &Array2, b: &Array2, c: &Array2, l: usize) -> Array2 -where - T: FftNum, -{ - let b = b.clone().remove_axis(Axis(1)); - let mut res = Array2::::zeros((l, a.shape()[0])); - for i in 0..l { - let mut tmp = a.clone(); - for _ in 0..i { - tmp = tmp.dot(a); - } - let out = c.dot(&tmp.dot(&b)); - res.slice_mut(s![i, ..]).assign(&out); - } - res -} - pub fn scanner( a: &Array2, b: &Array2, From cb475c767f65aef10c9397b62f5e230d7b6c1503 Mon Sep 17 00:00:00 2001 From: FL03 Date: Mon, 18 Dec 2023 15:34:29 -0600 Subject: [PATCH 089/118] update Signed-off-by: FL03 --- core/src/params/group.rs | 24 ++++++++++++++---------- core/src/params/mod.rs | 20 ++++++++++---------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/core/src/params/group.rs b/core/src/params/group.rs index 1bbe51a0..ea68795f 100644 --- a/core/src/params/group.rs +++ b/core/src/params/group.rs @@ -25,7 +25,7 @@ where impl ParamGroup where T: Float, - D: Dimension + RemoveAxis, + D: RemoveAxis, { pub fn new(dim: impl IntoDimension) -> Self { let dim = dim.into_dimension(); @@ -62,7 +62,7 @@ impl ParamGroup where D: Dimension, T: NdFloat, - Self: Biased + Weighted, + Self: Biased + Weighted, { pub fn linear(&self, data: &Array) -> Array where @@ -103,38 +103,42 @@ where } } -impl Biased for ParamGroup +impl Biased for ParamGroup where D: RemoveAxis, T: Float, { - fn bias(&self) -> &Array { + type Dim = D::Smaller; + + fn bias(&self) -> &Array { &self.bias } - fn bias_mut(&mut self) -> &mut Array { + fn bias_mut(&mut self) -> &mut Array { &mut self.bias } - fn set_bias(&mut self, bias: Array) { + fn set_bias(&mut self, bias: Array) { self.bias = bias; } } -impl Weighted for ParamGroup +impl Weighted for ParamGroup where D: Dimension, T: Float, { - fn weights(&self) -> &Array { + type Dim = D; + + fn weights(&self) -> &Array { &self.weights } - fn weights_mut(&mut self) -> &mut Array { + fn weights_mut(&mut self) -> &mut Array { &mut self.weights } - fn set_weights(&mut self, weights: Array) { + fn set_weights(&mut self, weights: Array) { self.weights = weights; } } diff --git a/core/src/params/mod.rs b/core/src/params/mod.rs index e191d6f8..2d4f3a9d 100644 --- a/core/src/params/mod.rs +++ b/core/src/params/mod.rs @@ -24,30 +24,30 @@ pub trait Param { fn name(&self) -> &str; } -pub trait Biased +pub trait Biased where - D: Dimension, T: Float, { + type Dim: Dimension; /// Returns an owned reference to the bias of the layer. - fn bias(&self) -> &Array; + fn bias(&self) -> &Array; /// Returns a mutable reference to the bias of the layer. - fn bias_mut(&mut self) -> &mut Array; + fn bias_mut(&mut self) -> &mut Array; /// Sets the bias of the layer. - fn set_bias(&mut self, bias: Array); + fn set_bias(&mut self, bias: Array); } -pub trait Weighted +pub trait Weighted where - D: Dimension, T: Float, { + type Dim: Dimension; /// Returns an owned reference to the weights of the layer. - fn weights(&self) -> &Array; + fn weights(&self) -> &Array; /// Returns a mutable reference to the weights of the layer. - fn weights_mut(&mut self) -> &mut Array; + fn weights_mut(&mut self) -> &mut Array; /// Sets the weights of the layer. - fn set_weights(&mut self, weights: Array); + fn set_weights(&mut self, weights: Array); } pub trait Params From 775e23c00909af23ff324803814a54fbc7609ca5 Mon Sep 17 00:00:00 2001 From: FL03 Date: Thu, 21 Dec 2023 13:54:54 -0600 Subject: [PATCH 090/118] update Signed-off-by: FL03 --- core/src/utils.rs | 10 +++++++++- ml/s4/src/cmp/cache.rs | 10 ++++++++++ ml/s4/src/cmp/kernal.rs | 31 +++++++++++++++++++++++++++++++ ml/s4/src/cmp/mod.rs | 11 +++++++++++ ml/s4/src/lib.rs | 1 + ml/s4/src/model/model.rs | 27 ++++++++++++++++++++------- ml/s4/src/params/store.rs | 8 ++++++++ ml/s4/src/utils.rs | 15 +++++++++++++++ 8 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 ml/s4/src/cmp/cache.rs create mode 100644 ml/s4/src/cmp/kernal.rs create mode 100644 ml/s4/src/cmp/mod.rs diff --git a/core/src/utils.rs b/core/src/utils.rs index 5c45d6f6..1eacd95b 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ -use ndarray::prelude::{Array, Array1, Axis, Dimension}; +use ndarray::prelude::{Array, Array1, Axis, Dimension, NdFloat}; use ndarray::{concatenate, IntoDimension, RemoveAxis, ShapeError}; use num::cast::AsPrimitive; use num::Float; @@ -21,6 +21,14 @@ where res } +pub fn cauchy_dot(a: &Array, lambda: &Array, omega: &Array) -> T +where + D: Dimension, + T: NdFloat, +{ + (a / (omega - lambda)).sum() +} + pub fn concat_iter(axis: usize, iter: impl IntoIterator>) -> Array where D: RemoveAxis, diff --git a/ml/s4/src/cmp/cache.rs b/ml/s4/src/cmp/cache.rs new file mode 100644 index 00000000..4ec4d32e --- /dev/null +++ b/ml/s4/src/cmp/cache.rs @@ -0,0 +1,10 @@ +/* + Appellation: cache + Contrib: FL03 +*/ +use ndarray::prelude::{Array, Dimension, Ix2}; +// use num::{Complex, Float}; + +pub struct Cache where D: Dimension { + cache: Array, +} \ No newline at end of file diff --git a/ml/s4/src/cmp/kernal.rs b/ml/s4/src/cmp/kernal.rs new file mode 100644 index 00000000..d92a4690 --- /dev/null +++ b/ml/s4/src/cmp/kernal.rs @@ -0,0 +1,31 @@ +/* + Appellation: kernal + Contrib: FL03 +*/ +use ndarray::prelude::Array2; +use num::Float; + +pub struct Kernal { + kernal: Array2, +} + +impl Kernal +where + T: Float, +{ + pub fn new(kernal: Array2) -> Self { + Self { kernal } + } + + pub fn square(features: usize) -> Self + where + T: Default, + { + let kernal = Array2::::default((features, features)); + Self::new(kernal) + } + + pub fn kernal(&self) -> &Array2 { + &self.kernal + } +} \ No newline at end of file diff --git a/ml/s4/src/cmp/mod.rs b/ml/s4/src/cmp/mod.rs new file mode 100644 index 00000000..f8f7e017 --- /dev/null +++ b/ml/s4/src/cmp/mod.rs @@ -0,0 +1,11 @@ +/* + Appellation: cmp + Contrib: FL03 +*/ +//! # Components +//! +//! +pub use self::{cache::*, kernal::*}; + +pub(crate) mod cache; +pub(crate) mod kernal; diff --git a/ml/s4/src/lib.rs b/ml/s4/src/lib.rs index 2284f0cf..6cfcb184 100644 --- a/ml/s4/src/lib.rs +++ b/ml/s4/src/lib.rs @@ -12,6 +12,7 @@ pub(crate) mod primitives; pub(crate) mod specs; pub(crate) mod utils; +pub mod cmp; pub mod ops; pub mod params; pub mod ssm; diff --git a/ml/s4/src/model/model.rs b/ml/s4/src/model/model.rs index 9e275f78..20b520ff 100644 --- a/ml/s4/src/model/model.rs +++ b/ml/s4/src/model/model.rs @@ -16,7 +16,7 @@ pub struct S4 where T: Float, { - cache: Array1, + cache: Array1, // make complex config: S4Config, kernal: Option>, store: SSMStore, @@ -41,6 +41,18 @@ where store, } } + + pub fn cache(&self) -> &Array1 { + &self.cache + } + + pub fn config(&self) -> &S4Config { + &self.config + } + + pub fn config_mut(&mut self) -> &mut S4Config { + &mut self.config + } } impl Forward> for S4 @@ -50,14 +62,15 @@ where type Output = Array2; fn forward(&self, args: &Array2) -> Self::Output { - if !self.config.decode { + let res = if !self.config().decode() { let mode = PaddingMode::<2, T>::Const(T::zero()); let size = PaddingSize::Full; - return args + args .conv_2d_fft(&self.kernal.clone().unwrap(), size, mode) - .expect("convolution failed"); - } - let scan = self.store.scan(args, &self.cache); - scan + args * &self.store[D] + .expect("convolution failed") + } else { + self.store.scanner(args, &self.cache) + }; + res + args * &self.store[D] } } diff --git a/ml/s4/src/params/store.rs b/ml/s4/src/params/store.rs index 92f86804..06799667 100644 --- a/ml/s4/src/params/store.rs +++ b/ml/s4/src/params/store.rs @@ -97,9 +97,17 @@ impl SSMStore where T: NdFloat, { + pub fn scanner(&self, u: &Array2, x0: &Array1) -> Array2 { + scanner(&self.a, &self.b, &self.c, u, x0) + } + pub fn scan(&self, u: &Array2, x0: &Array1) -> Array2 { scanner(&self.a, &self.b, &self.c, u, x0) } + + pub fn scan_complex(&self, u: &Array2, x0: &Array1) -> Array2 { + scanner(&self.a, &self.b, &self.c, u, x0) + } } impl SSMStore diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index 6765507c..fce69717 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -54,3 +54,18 @@ where } res } + +pub fn scan_ssm_step<'a, T>( + a: &'a Array2, + b: &'a Array2, + c: &'a Array2, +) -> impl Fn(&'a mut Array1, ArrayView1<'a, T>) -> Option> +where + T: NdFloat, +{ + |xs, us | { + let x1 = a.dot(xs) + b.dot(&us); + let y1 = c.dot(&x1); + Some(y1) + } +} From 146f8a4fd224d142104fd45600922e1007693574 Mon Sep 17 00:00:00 2001 From: FL03 Date: Thu, 21 Dec 2023 21:19:54 -0600 Subject: [PATCH 091/118] update Signed-off-by: FL03 --- ml/s4/Cargo.toml | 2 + ml/s4/src/cmp/{kernal.rs => kernel.rs} | 6 +-- ml/s4/src/cmp/mod.rs | 4 +- ml/s4/src/model/model.rs | 19 +++++++++- ml/s4/src/ops/discretize.rs | 51 +++++++++++++++++++------- ml/s4/src/ops/gen.rs | 6 +++ ml/s4/src/ops/mod.rs | 3 +- ml/s4/src/params/kinds.rs | 2 + ml/s4/src/params/store.rs | 2 - ml/s4/src/ssm/model.rs | 1 + ml/s4/src/utils.rs | 34 ++++++++++++++++- 11 files changed, 106 insertions(+), 24 deletions(-) rename ml/s4/src/cmp/{kernal.rs => kernel.rs} (85%) create mode 100644 ml/s4/src/ops/gen.rs diff --git a/ml/s4/Cargo.toml b/ml/s4/Cargo.toml index 6af189b7..4e5485d6 100644 --- a/ml/s4/Cargo.toml +++ b/ml/s4/Cargo.toml @@ -32,11 +32,13 @@ anyhow.workspace = true faer-core.workspace = true faer.workspace = true lazy_static.workspace = true +nalgebra.workspace = true ndarray.workspace = true ndarray-conv = "0.2" # ndarray-linalg.workspace = true ndarray-rand.workspace = true ndarray-stats.workspace = true +nshare = "0.9" num.workspace = true rustfft = { features = [], version = "6" } serde.workspace = true diff --git a/ml/s4/src/cmp/kernal.rs b/ml/s4/src/cmp/kernel.rs similarity index 85% rename from ml/s4/src/cmp/kernal.rs rename to ml/s4/src/cmp/kernel.rs index d92a4690..27730137 100644 --- a/ml/s4/src/cmp/kernal.rs +++ b/ml/s4/src/cmp/kernel.rs @@ -1,15 +1,15 @@ /* - Appellation: kernal + Appellation: kernel Contrib: FL03 */ use ndarray::prelude::Array2; use num::Float; -pub struct Kernal { +pub struct Kernel { kernal: Array2, } -impl Kernal +impl Kernel where T: Float, { diff --git a/ml/s4/src/cmp/mod.rs b/ml/s4/src/cmp/mod.rs index f8f7e017..4b417544 100644 --- a/ml/s4/src/cmp/mod.rs +++ b/ml/s4/src/cmp/mod.rs @@ -5,7 +5,7 @@ //! # Components //! //! -pub use self::{cache::*, kernal::*}; +pub use self::{cache::*, kernel::*}; pub(crate) mod cache; -pub(crate) mod kernal; +pub(crate) mod kernel; diff --git a/ml/s4/src/model/model.rs b/ml/s4/src/model/model.rs index 20b520ff..b4ae8a7a 100644 --- a/ml/s4/src/model/model.rs +++ b/ml/s4/src/model/model.rs @@ -7,11 +7,18 @@ use crate::neural::prelude::Forward; use crate::prelude::SSMStore; use ndarray::prelude::{Array1, Array2, NdFloat}; use ndarray_conv::{Conv2DFftExt, PaddingMode, PaddingSize}; -use num::Float; +use num::{Complex, Float}; use rustfft::FftNum; use crate::prelude::SSMParams::*; +pub struct S4Params where T: Float { + pub lambda: Array2>, + + pub c: Array2>, + pub d: Array2, +} + pub struct S4 where T: Float, @@ -55,6 +62,16 @@ where } } +impl S4 +where + T: Float, +{ + pub fn setup(mut self) -> Self { + + self + } +} + impl Forward> for S4 where T: FftNum + NdFloat, diff --git a/ml/s4/src/ops/discretize.rs b/ml/s4/src/ops/discretize.rs index c0f9f443..630448e6 100644 --- a/ml/s4/src/ops/discretize.rs +++ b/ml/s4/src/ops/discretize.rs @@ -5,43 +5,68 @@ use faer::prelude::{FaerMat, IntoFaer, SolverCore}; use faer::IntoNdarray; use faer_core::zip::ViewMut; -use faer_core::{ComplexField, Conjugate, SimpleEntity}; +use faer_core::{Conjugate, SimpleEntity}; use ndarray::prelude::{Array2, NdFloat}; +use nshare::{ToNalgebra, ToNdarray2}; use num::ToPrimitive; -pub fn discretize( +pub fn discretize_nalgebra( + a: &Array2, + b: &Array2, + c: &Array2, + step: f64, +) -> anyhow::Result<(Array2, Array2, Array2)> +{ + let ss = step / 2.0; // half step + let eye = Array2::::eye(a.shape()[0]); + let bl = &eye - a * ss; + let be = { + let tmp = bl.into_nalgebra().lu().try_inverse(); + if let Some(arg) = tmp { + arg.into_ndarray2() + } else { + return Err(anyhow::anyhow!("Could not invert matrix")); + } + }; + let ab = be.dot(&(&eye + a * ss)); + let bb = (b * ss).dot(&b.t()); + + Ok((ab, bb, c.clone())) +} + +pub fn discretize_faer( a: &Array2, b: &Array2, c: &Array2, - _d: &Array2, step: T, -) -> anyhow::Result<(Array2, Array2, Array2)> +) -> (Array2, Array2, Array2) where T: NdFloat + Conjugate + SimpleEntity, - ::Canonical: ComplexField + SimpleEntity + ToPrimitive, + ::Canonical: faer_core::ComplexField + SimpleEntity + ToPrimitive, { let ss = step / T::from(2).unwrap(); // half step let eye = Array2::::eye(a.shape()[0]); let bl = &eye - a * ss; let be = { - let mut tmp = bl.view().into_faer().qr().inverse(); + let mut tmp = bl.view().into_faer().partial_piv_lu().inverse(); let arr = &tmp.view_mut().into_ndarray(); arr.mapv(|i| T::from(i).unwrap()) }; let ab = be.dot(&(&eye + a * ss)); let bb = (b * ss).dot(&b.t()); - Ok((ab, bb, c.clone())) + (ab, bb, c.clone()) +} + +pub trait Discretize { + type Output; + fn discretize(&self, step: impl num::Float) -> Self::Output; } pub enum DiscretizeArgs {} -pub struct Discretize { +pub struct Discretizer { pub step: T, } -pub struct Discretized { - pub a: Array2, - pub b: Array2, - pub c: Array2, -} +pub struct Discretized(T); diff --git a/ml/s4/src/ops/gen.rs b/ml/s4/src/ops/gen.rs new file mode 100644 index 00000000..01ff98fc --- /dev/null +++ b/ml/s4/src/ops/gen.rs @@ -0,0 +1,6 @@ +/* + Appellation: gen + Contrib: FL03 +*/ + +pub fn k_gen() {} \ No newline at end of file diff --git a/ml/s4/src/ops/mod.rs b/ml/s4/src/ops/mod.rs index 3eddf70a..1bc197f8 100644 --- a/ml/s4/src/ops/mod.rs +++ b/ml/s4/src/ops/mod.rs @@ -2,10 +2,11 @@ Appellation: ops Contrib: FL03 */ -pub use self::{convolve::*, discretize::*}; +pub use self::{convolve::*, discretize::*, gen::*}; pub(crate) mod convolve; pub(crate) mod discretize; +pub(crate) mod gen; #[cfg(test)] mod tests {} diff --git a/ml/s4/src/params/kinds.rs b/ml/s4/src/params/kinds.rs index 5651dd95..167965a7 100644 --- a/ml/s4/src/params/kinds.rs +++ b/ml/s4/src/params/kinds.rs @@ -5,6 +5,8 @@ use serde::{Deserialize, Serialize}; use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; + + #[derive( Clone, Copy, diff --git a/ml/s4/src/params/store.rs b/ml/s4/src/params/store.rs index 06799667..25aa0f06 100644 --- a/ml/s4/src/params/store.rs +++ b/ml/s4/src/params/store.rs @@ -12,8 +12,6 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::ops; -pub struct B(Box>>); - #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct SSMStore where diff --git a/ml/s4/src/ssm/model.rs b/ml/s4/src/ssm/model.rs index cf08390b..3e1db444 100644 --- a/ml/s4/src/ssm/model.rs +++ b/ml/s4/src/ssm/model.rs @@ -4,6 +4,7 @@ */ use super::SSMConfig; use crate::prelude::scanner; +use crate::params::SSMStore; use faer::prelude::{FaerMat, IntoFaer, SolverCore}; use faer::IntoNdarray; use faer_core::zip::ViewMut; diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index fce69717..1043b9c2 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -4,9 +4,15 @@ */ use ndarray::prelude::{s, Array1, Array2, ArrayView1, NdFloat}; use ndarray::IntoDimension; -use num::Float; +use num::{Complex, Float}; use rustfft::{FftNum, FftPlanner}; +pub fn cauchy(v: &Array1, omega: &Array1, lambda: &Array1) -> Array1 where T: NdFloat { + let cdot = | b: T | (v / (lambda * T::one().neg() + b)).sum(); + omega.mapv(cdot) + +} + pub fn powmat(a: &Array2, n: usize) -> Array2 where T: Float + 'static, @@ -55,11 +61,35 @@ where res } +pub fn scan_complex( + a: &Array2>, + b: &Array2, + c: &Array2>, + u: &Array2, + x0: &Array1>, +) -> Array2> +where + T: NdFloat, +{ + let step = |xs: &mut Array1>, us: ArrayView1| { + let x1 = a.dot(xs) + b.dot(&us); + let y1 = c.dot(&x1); + Some(y1) + }; + let scan = u.outer_iter().scan(x0.clone(), step).collect::>(); + let shape = [scan.len(), scan[0].len()]; + let mut res = Array2::zeros(shape.into_dimension()); + for (i, s) in scan.iter().enumerate() { + res.slice_mut(s![i, ..]).assign(s); + } + res +} + pub fn scan_ssm_step<'a, T>( a: &'a Array2, b: &'a Array2, c: &'a Array2, -) -> impl Fn(&'a mut Array1, ArrayView1<'a, T>) -> Option> +) -> impl FnMut(&'a mut Array1, ArrayView1<'a, T>) -> Option> where T: NdFloat, { From 6d3d07baf9441bfa3ae372fbc80e664363b41325 Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 22 Dec 2023 07:45:30 -0600 Subject: [PATCH 092/118] update Signed-off-by: FL03 --- core/src/epochs/mod.rs | 12 --- core/src/errors/error.rs | 4 +- core/src/lib.rs | 16 ++-- core/src/params/group.rs | 2 +- core/src/primitives.rs | 1 + core/src/specs.rs | 142 ---------------------------- core/src/specs/arrays.rs | 55 +++++++++++ core/src/specs/base.rs | 31 ++++++ core/src/specs/math.rs | 60 ++++++++++++ core/src/specs/mod.rs | 12 +++ core/src/states/state.rs | 6 +- core/src/step/linspace.rs | 138 --------------------------- core/src/step/mod.rs | 13 --- core/src/step/stepper.rs | 6 -- core/src/{epochs => time}/epoch.rs | 0 core/src/time/mod.rs | 30 ++++++ core/src/utils.rs | 18 ++-- core/src/vars/epsilon.rs | 4 - core/src/vars/mod.rs | 12 --- core/tests/utils.rs | 18 ++-- data/src/lib.rs | 3 + data/src/specs.rs | 2 + data/src/tensors/mod.rs | 14 +-- ml/linear/src/model/mod.rs | 5 +- ml/linear/src/model/module.rs | 131 +++++++++++++++++++++++++ ml/neural/src/cmp/mod.rs | 7 ++ ml/neural/src/lib.rs | 10 +- ml/neural/src/models/exp/modules.rs | 84 ---------------- ml/neural/src/nn/cnn/mod.rs | 4 +- ml/s4/src/cmp/cache.rs | 7 +- ml/s4/src/cmp/kernel.rs | 2 +- ml/s4/src/cmp/mod.rs | 4 +- ml/s4/src/dplr/hippo.rs | 20 ++++ ml/s4/src/dplr/mod.rs | 9 ++ ml/s4/src/lib.rs | 1 + ml/s4/src/model/mod.rs | 3 +- ml/s4/src/model/model.rs | 13 +-- ml/s4/src/model/params.rs | 88 +++++++++++++++++ ml/s4/src/ops/discretize.rs | 3 +- ml/s4/src/ops/gen.rs | 2 +- ml/s4/src/params/kinds.rs | 2 - ml/s4/src/params/store.rs | 8 -- ml/s4/src/specs.rs | 12 +++ ml/s4/src/ssm/model.rs | 77 +++++++++------ ml/s4/src/utils.rs | 12 ++- 45 files changed, 580 insertions(+), 523 deletions(-) delete mode 100644 core/src/epochs/mod.rs delete mode 100644 core/src/specs.rs create mode 100644 core/src/specs/arrays.rs create mode 100644 core/src/specs/base.rs create mode 100644 core/src/specs/math.rs create mode 100644 core/src/specs/mod.rs delete mode 100644 core/src/step/linspace.rs delete mode 100644 core/src/step/mod.rs delete mode 100644 core/src/step/stepper.rs rename core/src/{epochs => time}/epoch.rs (100%) create mode 100644 core/src/time/mod.rs delete mode 100644 core/src/vars/epsilon.rs delete mode 100644 core/src/vars/mod.rs create mode 100644 ml/linear/src/model/module.rs create mode 100644 ml/neural/src/cmp/mod.rs create mode 100644 ml/s4/src/dplr/hippo.rs create mode 100644 ml/s4/src/dplr/mod.rs create mode 100644 ml/s4/src/model/params.rs diff --git a/core/src/epochs/mod.rs b/core/src/epochs/mod.rs deleted file mode 100644 index 25ad6560..00000000 --- a/core/src/epochs/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -/* - Appellation: epochs - Contrib: FL03 -*/ -pub use self::{epoch::*, utils::*}; - -pub(crate) mod epoch; - -pub(crate) mod utils {} - -#[cfg(test)] -mod tests {} diff --git a/core/src/errors/error.rs b/core/src/errors/error.rs index ea8bf8e5..9012cb53 100644 --- a/core/src/errors/error.rs +++ b/core/src/errors/error.rs @@ -54,7 +54,7 @@ pub struct Error { impl Error { pub fn new(kind: Errors, message: String) -> Self { - let ts = crate::now(); + let ts = crate::prelude::now(); Self { kind, message, ts } } @@ -91,7 +91,7 @@ impl Error { } fn on_update(&mut self) { - self.ts = crate::now(); + self.ts = crate::prelude::now(); } } diff --git a/core/src/lib.rs b/core/src/lib.rs index acf905e4..b0a0073c 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -9,13 +9,11 @@ pub(crate) mod primitives; pub(crate) mod specs; pub(crate) mod utils; -pub mod epochs; pub mod errors; pub mod masks; pub mod params; pub mod states; -pub mod step; -pub mod vars; +pub mod time; pub trait Transform { type Output; @@ -26,14 +24,12 @@ pub trait Transform { pub mod prelude { pub use super::Transform; - pub use crate::epochs::*; - pub use crate::errors::*; - pub use crate::masks::*; - pub use crate::states::*; - pub use crate::step::*; - // pub use crate::vars::*; - pub use crate::primitives::*; pub use crate::specs::*; pub use crate::utils::*; + + pub use crate::errors::*; + pub use crate::masks::*; + pub use crate::states::*; + pub use crate::time::*; } diff --git a/core/src/params/group.rs b/core/src/params/group.rs index ea68795f..b5e4138e 100644 --- a/core/src/params/group.rs +++ b/core/src/params/group.rs @@ -62,7 +62,7 @@ impl ParamGroup where D: Dimension, T: NdFloat, - Self: Biased + Weighted, + Self: Biased + Weighted, { pub fn linear(&self, data: &Array) -> Array where diff --git a/core/src/primitives.rs b/core/src/primitives.rs index 540dcf9a..2a96ce14 100644 --- a/core/src/primitives.rs +++ b/core/src/primitives.rs @@ -4,6 +4,7 @@ */ pub use self::{constants::*, statics::*, types::*}; +pub use ndarray::ShapeError; pub use ndarray_rand::rand_distr::uniform::SampleUniform; /// Collection of constants used throughout the system diff --git a/core/src/specs.rs b/core/src/specs.rs deleted file mode 100644 index 5df18a49..00000000 --- a/core/src/specs.rs +++ /dev/null @@ -1,142 +0,0 @@ -/* - Appellation: specs - Contrib: FL03 -*/ -pub use self::math::*; - -use ndarray::prelude::{Array, Axis, Dimension, Ix2}; -use ndarray::IntoDimension; -// use ndarray::linalg::Dot; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Uniform}; -use ndarray_rand::RandomExt; -use num::Float; - -pub trait Apply { - fn apply(&self, f: F) -> Self - where - F: Fn(&T) -> T; -} - -impl Apply for Array -where - D: Dimension, -{ - fn apply(&self, f: F) -> Self - where - F: Fn(&T) -> T, - { - self.map(f) - } -} - -pub trait ApplyTo { - fn apply_to(&self, args: &mut T) -> &mut T; -} - -pub trait As: AsRef + AsMut {} - -impl As for S where S: AsRef + AsMut {} - -pub trait GenerateRandom -where - D: Dimension, - T: Float + SampleUniform, -{ - fn bernoulli( - dim: impl IntoDimension, - p: Option, - ) -> Result, BernoulliError> { - let dist = Bernoulli::new(p.unwrap_or(0.5))?; - Ok(Array::random(dim.into_dimension(), dist)) - } - - fn uniform(axis: usize, dim: impl IntoDimension) -> Array { - let dim = dim.into_dimension(); - let dk = (T::one() / T::from(dim[axis]).unwrap()).sqrt(); - Self::uniform_between(dk, dim) - } - - fn uniform_between(dk: T, dim: impl IntoDimension) -> Array { - Array::random(dim, Uniform::new(-dk, dk)) - } -} - -impl GenerateRandom for Array -where - T: Float + SampleUniform, - D: Dimension, -{ -} - -pub trait IntoAxis { - fn into_axis(self) -> Axis; -} - -impl IntoAxis for S -where - S: AsRef, -{ - fn into_axis(self) -> Axis { - Axis(*self.as_ref()) - } -} - -pub(crate) mod math { - use ndarray::prelude::{Array, Dimension, Ix2, NdFloat}; - use ndarray_rand::rand_distr::uniform::SampleUniform; - use num::{Float, FromPrimitive, One, Signed, Zero}; - use std::ops; - - pub trait BinaryNum: One + Zero {} - - impl BinaryNum for T where T: One + Zero {} - - pub trait FloatExt: FromPrimitive + NdFloat + Signed + SampleUniform {} - - impl FloatExt for T where T: FromPrimitive + NdFloat + Signed + SampleUniform {} - - pub trait Arithmetic: - ops::Add - + ops::Div - + ops::Mul - + ops::Sub - { - } - - impl Arithmetic for A where - A: ops::Add - + ops::Div - + ops::Mul - + ops::Sub - { - } - - pub trait MatrixOps: - Arithmetic, Array> + Sized - where - A: Dimension, - B: Dimension, - { - } - - impl MatrixOps for Array - where - A: Dimension, - B: Dimension, - D: Dimension, - T: Float, - Self: Arithmetic, Array>, - { - } - - impl MatrixOps for &Array - where - A: Dimension, - B: Dimension, - D: Dimension, - T: Float, - Self: Arithmetic, Array>, - { - } -} diff --git a/core/src/specs/arrays.rs b/core/src/specs/arrays.rs new file mode 100644 index 00000000..28e46d65 --- /dev/null +++ b/core/src/specs/arrays.rs @@ -0,0 +1,55 @@ +/* + Appellation: base + Contrib: FL03 +*/ +use ndarray::prelude::{Array, Axis, Dimension, Ix2}; +use ndarray::IntoDimension; +// use ndarray::linalg::Dot; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Uniform}; +use ndarray_rand::RandomExt; +use num::Float; + +pub trait GenerateRandom +where + D: Dimension, + T: Float + SampleUniform, +{ + fn bernoulli( + dim: impl IntoDimension, + p: Option, + ) -> Result, BernoulliError> { + let dist = Bernoulli::new(p.unwrap_or(0.5))?; + Ok(Array::random(dim.into_dimension(), dist)) + } + + fn uniform(axis: usize, dim: impl IntoDimension) -> Array { + let dim = dim.into_dimension(); + let dk = (T::one() / T::from(dim[axis]).unwrap()).sqrt(); + Self::uniform_between(dk, dim) + } + + fn uniform_between(dk: T, dim: impl IntoDimension) -> Array { + Array::random(dim, Uniform::new(-dk, dk)) + } +} + +impl GenerateRandom for Array +where + T: Float + SampleUniform, + D: Dimension, +{ +} + +pub trait IntoAxis { + fn into_axis(self) -> Axis; +} + +impl IntoAxis for S +where + S: AsRef, +{ + fn into_axis(self) -> Axis { + Axis(*self.as_ref()) + } +} diff --git a/core/src/specs/base.rs b/core/src/specs/base.rs new file mode 100644 index 00000000..cc2b42fe --- /dev/null +++ b/core/src/specs/base.rs @@ -0,0 +1,31 @@ +/* + Appellation: base + Contrib: FL03 +*/ +use ndarray::prelude::{Array, Dimension}; + +pub trait Apply { + fn apply(&self, f: F) -> Self + where + F: Fn(&T) -> T; +} + +impl Apply for Array +where + D: Dimension, +{ + fn apply(&self, f: F) -> Self + where + F: Fn(&T) -> T, + { + self.map(f) + } +} + +pub trait ApplyTo { + fn apply_to(&self, args: &mut T) -> &mut T; +} + +pub trait As: AsRef + AsMut {} + +impl As for S where S: AsRef + AsMut {} diff --git a/core/src/specs/math.rs b/core/src/specs/math.rs new file mode 100644 index 00000000..30aff142 --- /dev/null +++ b/core/src/specs/math.rs @@ -0,0 +1,60 @@ +/* + Appellation: math + Contrib: FL03 +*/ +use ndarray::prelude::{Array, Dimension, Ix2, NdFloat}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use num::{Float, FromPrimitive, One, Signed, Zero}; +use std::ops; + +pub trait Binary: One + Zero {} + +impl Binary for T where T: One + Zero {} + +pub trait FloatExt: FromPrimitive + NdFloat + Signed + SampleUniform {} + +impl FloatExt for T where T: FromPrimitive + NdFloat + Signed + SampleUniform {} + +pub trait Arithmetic: + ops::Add + + ops::Div + + ops::Mul + + ops::Sub +{ +} + +impl Arithmetic for A where + A: ops::Add + + ops::Div + + ops::Mul + + ops::Sub +{ +} + +pub trait MatrixOps: + Arithmetic, Array> + Sized +where + A: Dimension, + B: Dimension, +{ +} + +impl MatrixOps for Array +where + A: Dimension, + B: Dimension, + D: Dimension, + T: Float, + Self: Arithmetic, Array>, +{ +} + +impl MatrixOps for &Array +where + A: Dimension, + B: Dimension, + D: Dimension, + T: Float, + Self: Arithmetic, Array>, +{ +} diff --git a/core/src/specs/mod.rs b/core/src/specs/mod.rs new file mode 100644 index 00000000..0f8f5c9c --- /dev/null +++ b/core/src/specs/mod.rs @@ -0,0 +1,12 @@ +/* + Appellation: specs + Contrib: FL03 +*/ +pub use self::{arrays::*, base::*, math::*}; + +pub(crate) mod arrays; +pub(crate) mod base; +pub(crate) mod math; + +#[cfg(test)] +mod tests {} diff --git a/core/src/states/state.rs b/core/src/states/state.rs index 1641712f..c6aba7ab 100644 --- a/core/src/states/state.rs +++ b/core/src/states/state.rs @@ -2,6 +2,7 @@ Appellation: state Contrib: FL03 */ +use crate::time::now; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] @@ -14,11 +15,10 @@ pub struct State { impl State { pub fn new(kind: impl ToString, message: String) -> Self { - let ts = crate::now(); Self { kind: kind.to_string(), message, - ts, + ts: now(), } } @@ -55,7 +55,7 @@ impl State { } fn on_update(&mut self) { - self.ts = crate::now(); + self.ts = now(); } } diff --git a/core/src/step/linspace.rs b/core/src/step/linspace.rs deleted file mode 100644 index 48580a2a..00000000 --- a/core/src/step/linspace.rs +++ /dev/null @@ -1,138 +0,0 @@ -/* - Appellation: linspace - Contrib: FL03 -*/ -use std::ops::{self, Range}; - -fn calculate_step + ops::Sub>( - bounds: Range, - capacity: T, -) -> T { - (bounds.end - bounds.start) / capacity -} - -fn stepsize + ops::Sub>(from: T, to: T, capacity: T) -> T { - (to - from) / capacity -} - -fn _linspace(a: f64, b: f64, capacity: usize) -> Vec { - let stepsize = stepsize(a, b, capacity as f64); - let mut data = Vec::with_capacity(capacity); - let mut current = a; - for _ in 0..capacity { - data.push(current); - current += stepsize; - } - data -} - -pub struct Linspace { - bounds: Range, - capacity: usize, - data: Vec, - stepsize: f64, -} - -impl Linspace { - pub fn new(bounds: Range, capacity: usize) -> Self { - let stepsize = calculate_step(bounds.clone(), capacity as f64); - let mut data = Vec::with_capacity(capacity); - let mut current = bounds.start; - for _ in 0..capacity { - data.push(current); - current += stepsize; - } - Self { - bounds, - capacity, - data, - stepsize, - } - } - - pub fn bounds(&self) -> &Range { - &self.bounds - } - - pub fn capacity(&self) -> usize { - self.capacity - } - - pub fn data(&self) -> &Vec { - &self.data - } - - pub fn stepsize(&self) -> f64 { - self.stepsize - } - - pub fn compute(&mut self) { - let mut current = self.bounds().start; - for i in 0..self.capacity() { - self.data[i] = current; - current += self.stepsize; - } - } - - pub fn update(&mut self) { - self.stepsize = stepsize(self.bounds().start, self.bounds().end, self.capacity as f64); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_linspace_simple() { - let bounds = 0.0..10.0; - let capacity = 10; - - let res = Linspace::new(bounds.clone(), capacity); - assert_eq!(res.bounds(), &bounds); - assert_eq!(res.capacity(), capacity); - assert_eq!( - res.data(), - &vec![0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] - ); - } - - #[test] - fn test_linspace() { - let bounds = 0.0..5.0; - let capacity = 10; - - let res = Linspace::new(bounds.clone(), capacity); - assert_eq!(res.bounds(), &bounds); - assert_eq!(res.capacity(), capacity); - assert_eq!( - res.data(), - &vec![0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5] - ); - } - - #[test] - fn test_linspace_large() { - let bounds = 0.0..10.0; - let capacity = 100; - - let res = Linspace::new(bounds.clone(), capacity); - assert_eq!(res.bounds(), &bounds); - assert_eq!(res.capacity(), capacity); - assert_eq!(res.data()[1], 0.1); - - let bounds = 1.2..10.0; - let capacity = 100; - - let res = Linspace::new(bounds.clone(), capacity); - assert_eq!(res.bounds(), &bounds); - assert_eq!(res.capacity(), capacity); - assert_eq!(res.data()[0], 1.2); - assert_eq!(res.data()[1], 1.288); - let last = { - let tmp = format!("{:.3}", res.data().last().unwrap()); - tmp.parse::().unwrap() - }; - assert_eq!(last, 10.0 - res.stepsize()); - } -} diff --git a/core/src/step/mod.rs b/core/src/step/mod.rs deleted file mode 100644 index 5b0fb408..00000000 --- a/core/src/step/mod.rs +++ /dev/null @@ -1,13 +0,0 @@ -/* - Appellation: step - Contrib: FL03 -*/ -pub use self::{stepper::*, utils::*}; - -pub(crate) mod stepper; - -pub mod linspace; - -pub trait Step {} - -pub(crate) mod utils {} diff --git a/core/src/step/stepper.rs b/core/src/step/stepper.rs deleted file mode 100644 index 07909956..00000000 --- a/core/src/step/stepper.rs +++ /dev/null @@ -1,6 +0,0 @@ -/* - Appellation: stepper - Contrib: FL03 -*/ - -pub struct Stepper {} diff --git a/core/src/epochs/epoch.rs b/core/src/time/epoch.rs similarity index 100% rename from core/src/epochs/epoch.rs rename to core/src/time/epoch.rs diff --git a/core/src/time/mod.rs b/core/src/time/mod.rs new file mode 100644 index 00000000..93daa5af --- /dev/null +++ b/core/src/time/mod.rs @@ -0,0 +1,30 @@ +/* + Appellation: epochs + Contrib: FL03 +*/ +pub use self::utils::*; + +pub mod epoch; + +pub(crate) mod utils { + pub fn now() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_timestamp() { + let period = std::time::Duration::from_secs(1); + let ts = now(); + assert!(ts > 0); + std::thread::sleep(period); + assert!(now() > ts); + } +} diff --git a/core/src/utils.rs b/core/src/utils.rs index 1eacd95b..69d995ea 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ -use ndarray::prelude::{Array, Array1, Axis, Dimension, NdFloat}; +use ndarray::prelude::{Array, Array1, Array2, Axis, Dimension, NdFloat}; use ndarray::{concatenate, IntoDimension, RemoveAxis, ShapeError}; use num::cast::AsPrimitive; use num::Float; @@ -52,9 +52,15 @@ where Array::linspace(T::one(), T::from(n).unwrap(), n).into_shape(dim) } -pub fn now() -> u128 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() +pub fn tril(a: &Array2) -> Array2 +where + T: NdFloat, +{ + let mut out = a.clone(); + for i in 0..a.shape()[0] { + for j in i + 1..a.shape()[1] { + out[[i, j]] = T::zero(); + } + } + out } diff --git a/core/src/vars/epsilon.rs b/core/src/vars/epsilon.rs deleted file mode 100644 index 2055d87f..00000000 --- a/core/src/vars/epsilon.rs +++ /dev/null @@ -1,4 +0,0 @@ -/* - Appellation: epsilon - Contrib: FL03 -*/ diff --git a/core/src/vars/mod.rs b/core/src/vars/mod.rs deleted file mode 100644 index 89a36959..00000000 --- a/core/src/vars/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -/* - Appellation: const - Contrib: FL03 -*/ -pub use self::{epsilon::*, utils::*}; - -pub(crate) mod epsilon; - -pub(crate) mod utils {} - -#[cfg(test)] -mod tests {} diff --git a/core/tests/utils.rs b/core/tests/utils.rs index 318fe9b3..0efdce91 100644 --- a/core/tests/utils.rs +++ b/core/tests/utils.rs @@ -1,20 +1,18 @@ #[cfg(test)] extern crate concision_core; -use concision_core::prelude::{linarr, now}; +use concision_core::prelude::{linarr, now, tril}; use ndarray::prelude::{array, Array2}; #[test] -fn test_linarr() { - let args: Array2 = linarr((2, 3)).unwrap(); - assert_eq!(&args, &array![[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); +fn test_tril() { + let a = linarr::((3, 3)).unwrap(); + let b = array![[1.0, 0.0, 0.0], [4.0, 5.0, 0.0], [7.0, 8.0, 9.0]]; + assert_eq!(b, tril(&a)); } #[test] -fn test_timestamp() { - let period = std::time::Duration::from_secs(1); - let ts = now(); - assert!(ts > 0); - std::thread::sleep(period); - assert!(now() > ts); +fn test_linarr() { + let args: Array2 = linarr((2, 3)).unwrap(); + assert_eq!(&args, &array![[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); } diff --git a/data/src/lib.rs b/data/src/lib.rs index bbf23117..161393fe 100644 --- a/data/src/lib.rs +++ b/data/src/lib.rs @@ -3,6 +3,9 @@ Contrib: FL03 */ //! # Concision Data +//! +#![feature(associated_type_defaults)] + pub use self::{primitives::*, specs::*, utils::*}; pub(crate) mod primitives; diff --git a/data/src/specs.rs b/data/src/specs.rs index be8f7cd5..ba4b3df3 100644 --- a/data/src/specs.rs +++ b/data/src/specs.rs @@ -21,3 +21,5 @@ where self.as_ref().0 } } + +pub trait NdArrayExt {} diff --git a/data/src/tensors/mod.rs b/data/src/tensors/mod.rs index 3c2fd578..b5116995 100644 --- a/data/src/tensors/mod.rs +++ b/data/src/tensors/mod.rs @@ -3,17 +3,17 @@ Contrib: FL03 */ //! # Tensors -pub use self::{tensor::*, utils::*}; +pub use self::tensor::*; pub(crate) mod tensor; use ndarray::prelude::{Array, Dimension, Ix2}; -pub trait NdTensor -where - D: Dimension, -{ - fn data(&self) -> &Array; +pub trait NdTensor { + type Dim: Dimension = Ix2; + + fn tensor(&self) -> &Array; } -pub(crate) mod utils {} +#[cfg(test)] +mod tests {} diff --git a/ml/linear/src/model/mod.rs b/ml/linear/src/model/mod.rs index 88aa46a6..a8eee288 100644 --- a/ml/linear/src/model/mod.rs +++ b/ml/linear/src/model/mod.rs @@ -2,6 +2,9 @@ Appellation: model Contrib: FL03 */ -pub use self::config::*; +//! # Linear Model +//! +pub use self::{config::*, module::*}; pub(crate) mod config; +pub(crate) mod module; diff --git a/ml/linear/src/model/module.rs b/ml/linear/src/model/module.rs new file mode 100644 index 00000000..4fafa7cc --- /dev/null +++ b/ml/linear/src/model/module.rs @@ -0,0 +1,131 @@ +/* + Appellation: module + Contrib: FL03 +*/ +use crate::core::params::{Biased, Weighted}; +use crate::neural::models::exp::{Module, ModuleParams}; +use crate::neural::prelude::Forward; +use ndarray::prelude::{Array2, NdFloat}; +use num::Float; + +pub struct LinearModel { + params: ModuleParams, +} + +impl LinearModel +where + T: Float, +{ + pub fn new() -> Self { + Self { + params: ModuleParams::new(), + } + } + + pub fn biased(&self) -> bool { + self.params.contains_key("bias") + } + + pub fn weighted(&self) -> bool { + self.params.contains_key("weight") + } + + pub fn with_weight(mut self, weight: Array2) -> Self { + self.params.insert("weight".to_string(), weight); + self + } + + pub fn with_bias(mut self, bias: Array2) -> Self { + self.params.insert("bias".to_string(), bias); + self + } +} + +impl Module for LinearModel +where + T: NdFloat, +{ + fn name(&self) -> &str { + "LinearModel" + } + + fn parameters(&self) -> &ModuleParams { + &self.params + } + + fn parameters_mut(&mut self) -> &mut ModuleParams { + &mut self.params + } +} + +impl Biased for LinearModel +where + T: Float, +{ + type Dim = ndarray::Ix2; + + fn bias(&self) -> &Array2 { + &self.params["bias"] + } + + fn bias_mut(&mut self) -> &mut Array2 { + self.params.get_mut("bias").unwrap() + } + + fn set_bias(&mut self, bias: Array2) { + self.params.insert("bias".to_string(), bias); + } +} + +impl Weighted for LinearModel +where + T: Float, +{ + type Dim = ndarray::Ix2; + + fn weights(&self) -> &Array2 { + &self.params["weight"] + } + + fn weights_mut(&mut self) -> &mut Array2 { + self.params.get_mut("weight").unwrap() + } + + fn set_weights(&mut self, weights: Array2) { + self.params.insert("weight".to_string(), weights); + } +} + +impl crate::neural::prelude::Weighted for LinearModel +where + T: NdFloat, +{ + fn weights(&self) -> &Array2 { + &self.params["weight"] + } + + fn weights_mut(&mut self) -> &mut Array2 { + self.params.get_mut("weight").unwrap() + } + + fn set_weights(&mut self, weights: Array2) { + self.params.insert("weight".to_string(), weights); + } +} + +impl Forward> for LinearModel +where + T: NdFloat, +{ + type Output = Array2; + + fn forward(&self, args: &Array2) -> Array2 { + if let Some(weight) = self.params.get("weight") { + if let Some(bias) = self.params.get("bias") { + return args.dot(&weight.t()) + bias; + } + return args.dot(&weight.t()); + } + args.clone() + } +} diff --git a/ml/neural/src/cmp/mod.rs b/ml/neural/src/cmp/mod.rs new file mode 100644 index 00000000..1d44b9a3 --- /dev/null +++ b/ml/neural/src/cmp/mod.rs @@ -0,0 +1,7 @@ +/* + Appellation: cmp + Contrib: FL03 +*/ +//! # Components +//! +//! diff --git a/ml/neural/src/lib.rs b/ml/neural/src/lib.rs index c5cbbcb1..805da18e 100644 --- a/ml/neural/src/lib.rs +++ b/ml/neural/src/lib.rs @@ -13,6 +13,7 @@ pub(crate) mod primitives; pub(crate) mod specs; pub(crate) mod utils; +pub mod cmp; pub mod errors; pub mod func; pub mod layers; @@ -25,6 +26,11 @@ pub mod params; pub(crate) use concision_core as core; pub mod prelude { + pub use crate::primitives::*; + pub use crate::specs::*; + pub use crate::utils::*; + + pub use crate::cmp::*; pub use crate::errors::*; pub use crate::func::{activate::*, loss::*, prop::*, rms::*}; pub use crate::layers::*; @@ -32,8 +38,4 @@ pub mod prelude { pub use crate::nn::*; pub use crate::ops::*; pub use crate::params::*; - - pub use crate::primitives::*; - pub use crate::specs::*; - pub use crate::utils::*; } diff --git a/ml/neural/src/models/exp/modules.rs b/ml/neural/src/models/exp/modules.rs index 4b987e34..c388bfa1 100644 --- a/ml/neural/src/models/exp/modules.rs +++ b/ml/neural/src/models/exp/modules.rs @@ -33,87 +33,3 @@ where T: Float, { } - -pub struct LinearModel { - params: ModuleParams, -} - -impl LinearModel -where - T: Float, -{ - pub fn new() -> Self { - Self { - params: ModuleParams::new(), - } - } - - pub fn biased(&self) -> bool { - self.params.contains_key("bias") - } - - pub fn weighted(&self) -> bool { - self.params.contains_key("weight") - } - - pub fn with_weight(mut self, weight: Array2) -> Self { - self.params.insert("weight".to_string(), weight); - self - } - - pub fn with_bias(mut self, bias: Array2) -> Self { - self.params.insert("bias".to_string(), bias); - self - } -} - -impl Module for LinearModel -where - T: NdFloat, -{ - fn name(&self) -> &str { - "LinearModel" - } - - fn parameters(&self) -> &ModuleParams { - &self.params - } - - fn parameters_mut(&mut self) -> &mut ModuleParams { - &mut self.params - } -} - -impl Weighted for LinearModel -where - T: NdFloat, -{ - fn weights(&self) -> &Array2 { - &self.params["weight"] - } - - fn weights_mut(&mut self) -> &mut Array2 { - self.params.get_mut("weight").unwrap() - } - - fn set_weights(&mut self, weights: Array2) { - self.params.insert("weight".to_string(), weights); - } -} - -impl Forward> for LinearModel -where - T: NdFloat, -{ - type Output = Array2; - - fn forward(&self, args: &Array2) -> Array2 { - if let Some(weight) = self.params.get("weight") { - if let Some(bias) = self.params.get("bias") { - return args.dot(&weight.t()) + bias; - } - return args.dot(&weight.t()); - } - args.clone() - } -} diff --git a/ml/neural/src/nn/cnn/mod.rs b/ml/neural/src/nn/cnn/mod.rs index d86fccbc..7f3d12b3 100644 --- a/ml/neural/src/nn/cnn/mod.rs +++ b/ml/neural/src/nn/cnn/mod.rs @@ -5,11 +5,9 @@ //! # Concurrent Neural Network (CNN) //! //! -pub use self::{model::*, utils::*}; +pub use self::model::*; pub(crate) mod model; -pub(crate) mod utils {} - #[cfg(test)] mod tests {} diff --git a/ml/s4/src/cmp/cache.rs b/ml/s4/src/cmp/cache.rs index 4ec4d32e..3d0fa7e9 100644 --- a/ml/s4/src/cmp/cache.rs +++ b/ml/s4/src/cmp/cache.rs @@ -5,6 +5,9 @@ use ndarray::prelude::{Array, Dimension, Ix2}; // use num::{Complex, Float}; -pub struct Cache where D: Dimension { +pub struct Cache +where + D: Dimension, +{ cache: Array, -} \ No newline at end of file +} diff --git a/ml/s4/src/cmp/kernel.rs b/ml/s4/src/cmp/kernel.rs index 27730137..073f331e 100644 --- a/ml/s4/src/cmp/kernel.rs +++ b/ml/s4/src/cmp/kernel.rs @@ -28,4 +28,4 @@ where pub fn kernal(&self) -> &Array2 { &self.kernal } -} \ No newline at end of file +} diff --git a/ml/s4/src/cmp/mod.rs b/ml/s4/src/cmp/mod.rs index 4b417544..4f09370f 100644 --- a/ml/s4/src/cmp/mod.rs +++ b/ml/s4/src/cmp/mod.rs @@ -3,8 +3,8 @@ Contrib: FL03 */ //! # Components -//! -//! +//! +//! pub use self::{cache::*, kernel::*}; pub(crate) mod cache; diff --git a/ml/s4/src/dplr/hippo.rs b/ml/s4/src/dplr/hippo.rs new file mode 100644 index 00000000..47b7669c --- /dev/null +++ b/ml/s4/src/dplr/hippo.rs @@ -0,0 +1,20 @@ +/* + Appellation: hippo + Contrib: FL03 +*/ +use crate::core::prelude::{linarr, tril}; +use ndarray::prelude::{Array, Array2, Ix2, NdFloat}; +use num::Float; + +pub fn make_hippo(features: usize) -> Array2 +where + T: NdFloat, +{ + let base = linarr::((features, 1)).unwrap(); + let p = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); + let mut a = &p * &p.t(); + a = tril(&a) - &base.diag(); + -a +} + +pub struct HiPPO; diff --git a/ml/s4/src/dplr/mod.rs b/ml/s4/src/dplr/mod.rs new file mode 100644 index 00000000..5d9f53bc --- /dev/null +++ b/ml/s4/src/dplr/mod.rs @@ -0,0 +1,9 @@ +/* + Appellation: dplr + Contrib: FL03 +*/ +//! # Diagonal Plus Low Rank (DPLR) +//! +//! + +pub mod hippo; diff --git a/ml/s4/src/lib.rs b/ml/s4/src/lib.rs index 6cfcb184..349dd913 100644 --- a/ml/s4/src/lib.rs +++ b/ml/s4/src/lib.rs @@ -13,6 +13,7 @@ pub(crate) mod specs; pub(crate) mod utils; pub mod cmp; +pub mod dplr; pub mod ops; pub mod params; pub mod ssm; diff --git a/ml/s4/src/model/mod.rs b/ml/s4/src/model/mod.rs index 914b6f48..e20642e8 100644 --- a/ml/s4/src/model/mod.rs +++ b/ml/s4/src/model/mod.rs @@ -2,7 +2,8 @@ Appellation: model Contrib: FL03 */ -pub use self::{config::*, model::*}; +pub use self::{config::*, model::*, params::*}; pub(crate) mod config; pub(crate) mod model; +pub(crate) mod params; diff --git a/ml/s4/src/model/model.rs b/ml/s4/src/model/model.rs index b4ae8a7a..175aaedb 100644 --- a/ml/s4/src/model/model.rs +++ b/ml/s4/src/model/model.rs @@ -12,13 +12,6 @@ use rustfft::FftNum; use crate::prelude::SSMParams::*; -pub struct S4Params where T: Float { - pub lambda: Array2>, - - pub c: Array2>, - pub d: Array2, -} - pub struct S4 where T: Float, @@ -67,7 +60,6 @@ where T: Float, { pub fn setup(mut self) -> Self { - self } } @@ -82,11 +74,10 @@ where let res = if !self.config().decode() { let mode = PaddingMode::<2, T>::Const(T::zero()); let size = PaddingSize::Full; - args - .conv_2d_fft(&self.kernal.clone().unwrap(), size, mode) + args.conv_2d_fft(&self.kernal.clone().unwrap(), size, mode) .expect("convolution failed") } else { - self.store.scanner(args, &self.cache) + self.store.scan(args, &self.cache) }; res + args * &self.store[D] } diff --git a/ml/s4/src/model/params.rs b/ml/s4/src/model/params.rs new file mode 100644 index 00000000..b12494f8 --- /dev/null +++ b/ml/s4/src/model/params.rs @@ -0,0 +1,88 @@ +/* + Appellation: params + Contrib: FL03 +*/ +use ndarray::prelude::{Array1, Array2, NdFloat}; +use num::{Complex, Float}; + +pub struct S4Store +where + T: Float, +{ + pub a: Array2>, // Lambda + pub b: Array2, + pub c: Array2>, + pub d: Array2, +} + +impl S4Store +where + T: Float, +{ + pub fn new(a: Array2>, b: Array2, c: Array2>, d: Array2) -> Self { + Self { a, b, c, d } + } + + pub fn zeros(features: usize) -> Self { + let a = Array2::>::zeros((features, features)); + let b = Array2::::zeros((features, features)); + let c = Array2::>::zeros((features, features)); + let d = Array2::::zeros((features, features)); + Self { a, b, c, d } + } + + pub fn a(&self) -> &Array2> { + &self.a + } + + pub fn a_mut(&mut self) -> &mut Array2> { + &mut self.a + } + + pub fn b(&self) -> &Array2 { + &self.b + } + + pub fn b_mut(&mut self) -> &mut Array2 { + &mut self.b + } + + pub fn c(&self) -> &Array2> { + &self.c + } + + pub fn c_mut(&mut self) -> &mut Array2> { + &mut self.c + } + + pub fn d(&self) -> &Array2 { + &self.d + } + + pub fn d_mut(&mut self) -> &mut Array2 { + &mut self.d + } +} + +impl S4Store +where + T: NdFloat, +{ + pub fn scan(&self, u: &Array2, x0: &Array1>) -> Array2> { + crate::scan_complex(&self.a, &self.b, &self.c, u, x0) + } +} + +// impl ops::Index for S4Store where T: Float { +// type Output = Array2; + +// fn index(&self, index: SSMParams) -> &Self::Output { +// use SSMParams::*; +// match index { +// A => &self.a, +// B => &self.b, +// C => &self.c, +// D => &self.d, +// } +// } +// } diff --git a/ml/s4/src/ops/discretize.rs b/ml/s4/src/ops/discretize.rs index 630448e6..197a8381 100644 --- a/ml/s4/src/ops/discretize.rs +++ b/ml/s4/src/ops/discretize.rs @@ -15,8 +15,7 @@ pub fn discretize_nalgebra( b: &Array2, c: &Array2, step: f64, -) -> anyhow::Result<(Array2, Array2, Array2)> -{ +) -> anyhow::Result<(Array2, Array2, Array2)> { let ss = step / 2.0; // half step let eye = Array2::::eye(a.shape()[0]); let bl = &eye - a * ss; diff --git a/ml/s4/src/ops/gen.rs b/ml/s4/src/ops/gen.rs index 01ff98fc..bcb59e8b 100644 --- a/ml/s4/src/ops/gen.rs +++ b/ml/s4/src/ops/gen.rs @@ -3,4 +3,4 @@ Contrib: FL03 */ -pub fn k_gen() {} \ No newline at end of file +pub fn k_gen() {} diff --git a/ml/s4/src/params/kinds.rs b/ml/s4/src/params/kinds.rs index 167965a7..5651dd95 100644 --- a/ml/s4/src/params/kinds.rs +++ b/ml/s4/src/params/kinds.rs @@ -5,8 +5,6 @@ use serde::{Deserialize, Serialize}; use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; - - #[derive( Clone, Copy, diff --git a/ml/s4/src/params/store.rs b/ml/s4/src/params/store.rs index 25aa0f06..93d32cb5 100644 --- a/ml/s4/src/params/store.rs +++ b/ml/s4/src/params/store.rs @@ -95,17 +95,9 @@ impl SSMStore where T: NdFloat, { - pub fn scanner(&self, u: &Array2, x0: &Array1) -> Array2 { - scanner(&self.a, &self.b, &self.c, u, x0) - } - pub fn scan(&self, u: &Array2, x0: &Array1) -> Array2 { scanner(&self.a, &self.b, &self.c, u, x0) } - - pub fn scan_complex(&self, u: &Array2, x0: &Array1) -> Array2 { - scanner(&self.a, &self.b, &self.c, u, x0) - } } impl SSMStore diff --git a/ml/s4/src/specs.rs b/ml/s4/src/specs.rs index 1d8faa71..0c990b8d 100644 --- a/ml/s4/src/specs.rs +++ b/ml/s4/src/specs.rs @@ -2,3 +2,15 @@ Appellation: specs Contrib: FL03 */ + +pub trait Scan { + type Output; + + fn scan(&self, args: &T, initial_state: &S) -> Self::Output; +} + +pub trait StateSpace { + type Config; + + fn config(&self) -> &Self::Config; +} diff --git a/ml/s4/src/ssm/model.rs b/ml/s4/src/ssm/model.rs index 3e1db444..a4b6cfa9 100644 --- a/ml/s4/src/ssm/model.rs +++ b/ml/s4/src/ssm/model.rs @@ -3,8 +3,8 @@ Contrib: FL03 */ use super::SSMConfig; -use crate::prelude::scanner; -use crate::params::SSMStore; +use crate::params::SSMParams::*; +use crate::prelude::{scanner, SSMStore}; use faer::prelude::{FaerMat, IntoFaer, SolverCore}; use faer::IntoNdarray; use faer_core::zip::ViewMut; @@ -12,25 +12,55 @@ use faer_core::{ComplexField, Conjugate, SimpleEntity}; use ndarray::prelude::{Array1, Array2, NdFloat}; use num::{Float, ToPrimitive}; -pub struct SSM { +pub struct SSM +where + T: Float, +{ config: SSMConfig, - pub a: Array2, - pub b: Array2, - pub c: Array2, - pub d: Array2, + kernel: Array2, + params: SSMStore, } impl SSM where T: Float, { - pub fn create(config: SSMConfig) -> Self { + pub fn create(config: SSMConfig) -> Self + where + T: Default, + { let features = config.features(); - let a = Array2::::zeros((features, features)); - let b = Array2::::zeros((features, 1)); - let c = Array2::::zeros((1, features)); - let d = Array2::::zeros((1, 1)); - Self { config, a, b, c, d } + let kernel = Array2::::zeros((features, features)); + let params = SSMStore::from_features(features); + Self { + config, + kernel, + params, + } + } + + pub fn config(&self) -> &SSMConfig { + &self.config + } + + pub fn config_mut(&mut self) -> &mut SSMConfig { + &mut self.config + } + + pub fn kernel(&self) -> &Array2 { + &self.kernel + } + + pub fn kernel_mut(&mut self) -> &mut Array2 { + &mut self.kernel + } + + pub fn params(&self) -> &SSMStore { + &self.params + } + + pub fn params_mut(&mut self) -> &mut SSMStore { + &mut self.params } } @@ -39,7 +69,7 @@ where T: NdFloat, { pub fn scan(&self, u: &Array2, x0: &Array1) -> Array2 { - scanner(&self.a, &self.b, &self.c, u, x0) + scanner(&self.params[A], &self.params[B], &self.params[C], u, x0) } } @@ -51,28 +81,15 @@ where pub fn discretize(&mut self, step: T) -> anyhow::Result<()> { let ds = step / T::from(2).unwrap(); let eye = Array2::::eye(self.config.features()); - let bl = &eye - &self.a * ds; + let bl = &eye - &self.params[A] * ds; let be = { let mut tmp = bl.view().into_faer().qr().inverse(); let arr = &tmp.view_mut().into_ndarray(); arr.mapv(|i| T::from(i).unwrap()) }; - let ab = &be.dot(&(&eye + &self.a * ds)); - let bb = (&self.b * ds).dot(&self.b.t()); + let ab = &be.dot(&(&eye + &self.params[A] * ds)); + let bb = (&self.params[B] * ds).dot(&self.params[B].t()); Ok(()) } } - -// impl SSM { - -// pub fn descretize(&mut self, step: f64) -> anyhow::Result<()> { -// let ds = step / 2.0; -// let eye = Array2::::eye(self.config.features()); -// let bl = (&eye - &self.a * ds).inv()?; -// // let ab = &bl | (&eye + &self.a * ds); -// // let bb = &self.b * ds | self.b; - -// Ok(()) -// } -// } diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index 1043b9c2..8706e45a 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -7,10 +7,12 @@ use ndarray::IntoDimension; use num::{Complex, Float}; use rustfft::{FftNum, FftPlanner}; -pub fn cauchy(v: &Array1, omega: &Array1, lambda: &Array1) -> Array1 where T: NdFloat { - let cdot = | b: T | (v / (lambda * T::one().neg() + b)).sum(); +pub fn cauchy(v: &Array1, omega: &Array1, lambda: &Array1) -> Array1 +where + T: NdFloat, +{ + let cdot = |b: T| (v / (lambda * T::one().neg() + b)).sum(); omega.mapv(cdot) - } pub fn powmat(a: &Array2, n: usize) -> Array2 @@ -21,7 +23,7 @@ where panic!("Matrix must be square"); } let mut res = a.clone(); - for _ in 0..n { + for _ in 1..n { res = res.dot(a); } res @@ -93,7 +95,7 @@ pub fn scan_ssm_step<'a, T>( where T: NdFloat, { - |xs, us | { + |xs, us| { let x1 = a.dot(xs) + b.dot(&us); let y1 = c.dot(&x1); Some(y1) From 80a2e3f09121a0eb85c33a01cf40702da2ac75b8 Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 22 Dec 2023 13:11:17 -0600 Subject: [PATCH 093/118] update Signed-off-by: FL03 --- core/src/lib.rs | 5 +- core/src/specs/arrays.rs | 46 +++++++++++++++- core/src/specs/init.rs | 8 +++ core/src/specs/math.rs | 37 ++++++++++++- core/src/specs/mod.rs | 36 ++++++++++++- core/src/utils.rs | 61 ++++++++++++++++++++- core/tests/utils.rs | 45 +++++++++++++--- ml/s4/Cargo.toml | 1 + ml/s4/src/dplr/hippo.rs | 102 ++++++++++++++++++++++++++++++++++-- ml/s4/src/ops/discretize.rs | 35 +++++++++++-- ml/s4/src/ssm/model.rs | 98 +++++++++++++++++++++++++++------- ml/s4/src/utils.rs | 99 +++++++++++++++++++++++++++++++++- 12 files changed, 530 insertions(+), 43 deletions(-) create mode 100644 core/src/specs/init.rs diff --git a/core/src/lib.rs b/core/src/lib.rs index b0a0073c..93f3bc8e 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -6,12 +6,13 @@ pub use self::{primitives::*, specs::*, utils::*}; pub(crate) mod primitives; -pub(crate) mod specs; + pub(crate) mod utils; pub mod errors; pub mod masks; pub mod params; +pub mod specs; pub mod states; pub mod time; @@ -25,11 +26,11 @@ pub mod prelude { pub use super::Transform; pub use crate::primitives::*; - pub use crate::specs::*; pub use crate::utils::*; pub use crate::errors::*; pub use crate::masks::*; + pub use crate::specs::*; pub use crate::states::*; pub use crate::time::*; } diff --git a/core/src/specs/arrays.rs b/core/src/specs/arrays.rs index 28e46d65..7527ad62 100644 --- a/core/src/specs/arrays.rs +++ b/core/src/specs/arrays.rs @@ -2,7 +2,7 @@ Appellation: base Contrib: FL03 */ -use ndarray::prelude::{Array, Axis, Dimension, Ix2}; +use ndarray::prelude::{Array, Axis, Dimension, Ix2, NdFloat}; use ndarray::IntoDimension; // use ndarray::linalg::Dot; use ndarray_rand::rand_distr::uniform::SampleUniform; @@ -10,6 +10,34 @@ use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Uniform}; use ndarray_rand::RandomExt; use num::Float; +pub trait Arange { + fn arange(start: T, stop: T, step: T) -> Self; +} + +impl Arange for Vec +where + T: Float, +{ + fn arange(start: T, stop: T, step: T) -> Self { + let n = ((stop - start) / step).ceil().to_usize().unwrap(); + (0..n).map(|i| start + step * T::from(i).unwrap()).collect() + } +} + +impl Arange for Array +where + T: Float, +{ + fn arange(start: T, stop: T, step: T) -> Self { + let n = ((stop - start) / step).ceil().to_usize().unwrap(); + let mut a = Array::zeros((n, 1)); + for (i, v) in a.iter_mut().enumerate() { + *v = start + step * T::from(i).unwrap(); + } + a + } +} + pub trait GenerateRandom where D: Dimension, @@ -53,3 +81,19 @@ where Axis(*self.as_ref()) } } + +pub trait Inverse: Sized +where + T: Float, +{ + fn inverse(&self) -> Option; +} + +impl Inverse for Array +where + T: NdFloat, +{ + fn inverse(&self) -> Option { + crate::compute_inverse(self) + } +} diff --git a/core/src/specs/init.rs b/core/src/specs/init.rs new file mode 100644 index 00000000..7aa06116 --- /dev/null +++ b/core/src/specs/init.rs @@ -0,0 +1,8 @@ +/* + Appellation: init + Contrib: FL03 +*/ + +pub trait Init { + fn init(&mut self) -> Self; +} diff --git a/core/src/specs/math.rs b/core/src/specs/math.rs index 30aff142..9ef1d0ee 100644 --- a/core/src/specs/math.rs +++ b/core/src/specs/math.rs @@ -4,13 +4,48 @@ */ use ndarray::prelude::{Array, Dimension, Ix2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; -use num::{Float, FromPrimitive, One, Signed, Zero}; +use num::{Complex, Float, FromPrimitive, Num, One, Signed, Zero}; use std::ops; pub trait Binary: One + Zero {} impl Binary for T where T: One + Zero {} +pub trait Conjugate { + fn conj(&self) -> Self; +} + +impl Conjugate for f32 { + fn conj(&self) -> Self { + *self + } +} + +impl Conjugate for f64 { + fn conj(&self) -> Self { + *self + } +} + +impl Conjugate for Complex +where + T: Copy + Num + Signed, +{ + fn conj(&self) -> Self { + Complex::::new(self.re, -self.im) + } +} + +impl Conjugate for Array +where + D: Dimension, + T: Clone + Conjugate, +{ + fn conj(&self) -> Self { + self.mapv(|x| x.conj()) + } +} + pub trait FloatExt: FromPrimitive + NdFloat + Signed + SampleUniform {} impl FloatExt for T where T: FromPrimitive + NdFloat + Signed + SampleUniform {} diff --git a/core/src/specs/mod.rs b/core/src/specs/mod.rs index 0f8f5c9c..dda39a6c 100644 --- a/core/src/specs/mod.rs +++ b/core/src/specs/mod.rs @@ -2,11 +2,43 @@ Appellation: specs Contrib: FL03 */ -pub use self::{arrays::*, base::*, math::*}; +pub use self::{arrays::*, base::*, init::*, math::*}; pub(crate) mod arrays; pub(crate) mod base; +pub(crate) mod init; pub(crate) mod math; +use num::{Complex, Num, Zero}; + +pub trait AsComplex: Num { + fn as_complex(&self) -> Complex; + + fn as_imag(&self) -> Complex; +} + +impl AsComplex for T +where + T: Copy + Num + Zero, +{ + fn as_complex(&self) -> Complex { + Complex::new(*self, T::zero()) + } + + fn as_imag(&self) -> Complex { + Complex::new(T::zero(), *self) + } +} + #[cfg(test)] -mod tests {} +mod tests { + + use super::*; + + #[test] + fn test_as_complex() { + let x = 1.0; + let y = x.as_complex(); + assert_eq!(y, Complex::new(1.0, 0.0)); + } +} diff --git a/core/src/utils.rs b/core/src/utils.rs index 69d995ea..4d754a1e 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ -use ndarray::prelude::{Array, Array1, Array2, Axis, Dimension, NdFloat}; +use ndarray::prelude::{s, Array, Array1, Array2, Axis, Dimension, NdFloat}; use ndarray::{concatenate, IntoDimension, RemoveAxis, ShapeError}; use num::cast::AsPrimitive; use num::Float; @@ -29,6 +29,50 @@ where (a / (omega - lambda)).sum() } +pub fn compute_inverse(matrix: &Array2) -> Option> { + let (rows, cols) = matrix.dim(); + + if rows != cols { + return None; // Matrix must be square for inversion + } + + let identity = Array2::eye(rows); + + // Concatenate the original matrix with an identity matrix + let mut augmented_matrix = Array2::zeros((rows, 2 * cols)); + augmented_matrix.slice_mut(s![.., ..cols]).assign(matrix); + augmented_matrix.slice_mut(s![.., cols..]).assign(&identity); + + // Perform Gaussian elimination to reduce the left half to the identity matrix + for i in 0..rows { + let pivot = augmented_matrix[[i, i]]; + + if pivot == T::zero() { + return None; // Matrix is singular + } + + augmented_matrix + .slice_mut(s![i, ..]) + .mapv_inplace(|x| x / pivot); + + for j in 0..rows { + if i != j { + let am = augmented_matrix.clone(); + let factor = augmented_matrix[[j, i]]; + let rhs = am.slice(s![i, ..]); + augmented_matrix + .slice_mut(s![j, ..]) + .zip_mut_with(&rhs, |x, &y| *x -= y * factor); + } + } + } + + // Extract the inverted matrix from the augmented matrix + let inverted = augmented_matrix.slice(s![.., cols..]); + + Some(inverted.to_owned()) +} + pub fn concat_iter(axis: usize, iter: impl IntoIterator>) -> Array where D: RemoveAxis, @@ -52,6 +96,21 @@ where Array::linspace(T::one(), T::from(n).unwrap(), n).into_shape(dim) } +pub fn linspace(dim: impl IntoDimension) -> Result, ShapeError> +where + D: Dimension, + T: Float, +{ + let dim = dim.into_dimension(); + let n = dim.as_array_view().product(); + Array::linspace(T::zero(), T::from(n - 1).unwrap(), n).into_shape(dim) +} + +pub fn round_to(val: T, decimals: usize) -> T { + let factor = T::from(10).expect("").powi(decimals as i32); + (val * factor).round() / factor +} + pub fn tril(a: &Array2) -> Array2 where T: NdFloat, diff --git a/core/tests/utils.rs b/core/tests/utils.rs index 0efdce91..2674aee2 100644 --- a/core/tests/utils.rs +++ b/core/tests/utils.rs @@ -1,18 +1,51 @@ #[cfg(test)] extern crate concision_core; -use concision_core::prelude::{linarr, now, tril}; +use concision_core as cnc; + +use cnc::prelude::{linarr, tril}; +use cnc::specs::{Conjugate, Inverse}; use ndarray::prelude::{array, Array2}; +use num::Complex; #[test] -fn test_tril() { - let a = linarr::((3, 3)).unwrap(); - let b = array![[1.0, 0.0, 0.0], [4.0, 5.0, 0.0], [7.0, 8.0, 9.0]]; - assert_eq!(b, tril(&a)); +fn test_conj() { + let data = (1..5).map(|x| x as f64).collect::>(); + let a = Array2::from_shape_vec((2, 2), data).unwrap(); + let exp = array![[1.0, 2.0], [3.0, 4.0]]; + assert_eq!(exp, a.conj()); + + let a = array![ + [Complex::new(0.0, 0.0), Complex::new(1.0, 0.25)], + [Complex::new(2.0, 0.5), Complex::new(3.0, 0.0)] + ]; + + let exp = array![ + [Complex::new(0.0, 0.0), Complex::new(1.0, -0.25)], + [Complex::new(2.0, -0.5), Complex::new(3.0, 0.0)] + ]; + + assert_eq!(exp, a.conj()); +} + +#[test] +fn test_inverse() { + let a = array![[1.0, 2.0], [3.0, 4.0]]; + let b = array![[1.0, 2.0, 3.0,], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]]; + let exp = array![[-2.0, 1.0], [1.5, -0.5]]; + assert_eq!(Some(exp), a.inverse()); + assert_eq!(None, b.inverse()); } #[test] fn test_linarr() { - let args: Array2 = linarr((2, 3)).unwrap(); + let args: Array2 = cnc::linarr((2, 3)).unwrap(); assert_eq!(&args, &array![[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); } + +#[test] +fn test_tril() { + let a = linarr::((3, 3)).unwrap(); + let exp = array![[1.0, 0.0, 0.0], [4.0, 5.0, 0.0], [7.0, 8.0, 9.0]]; + assert_eq!(exp, tril(&a)); +} diff --git a/ml/s4/Cargo.toml b/ml/s4/Cargo.toml index 4e5485d6..230c7c20 100644 --- a/ml/s4/Cargo.toml +++ b/ml/s4/Cargo.toml @@ -48,6 +48,7 @@ strum.workspace = true [dev-dependencies] approx.workspace = true +computare.workspace = true [package.metadata.docs.rs] all-features = true diff --git a/ml/s4/src/dplr/hippo.rs b/ml/s4/src/dplr/hippo.rs index 47b7669c..19f5d3bc 100644 --- a/ml/s4/src/dplr/hippo.rs +++ b/ml/s4/src/dplr/hippo.rs @@ -2,9 +2,11 @@ Appellation: hippo Contrib: FL03 */ -use crate::core::prelude::{linarr, tril}; -use ndarray::prelude::{Array, Array2, Ix2, NdFloat}; -use num::Float; +use crate::core::prelude::{linarr, tril, AsComplex, Conjugate}; +use crate::prelude::eig_csym; + +use ndarray::prelude::{Array1, Array2, Ix2, NdFloat}; +use num::{Complex, Float, FromPrimitive}; pub fn make_hippo(features: usize) -> Array2 where @@ -17,4 +19,96 @@ where -a } -pub struct HiPPO; +pub fn make_nplr_hippo(features: usize) -> (Array2, Array1, Array1) +where + T: NdFloat, +{ + let hippo = make_hippo(features); + + let base = Array1::linspace(T::zero(), T::from(features - 1).unwrap(), features); + let p = (&base + T::one() / T::from(2).unwrap()).mapv(T::sqrt); + let b = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); + (hippo, p, b) +} + +pub fn dplr_hippo( + features: usize, +) -> ( + Array2>, + Array2>, + Array2>, +) { + let (a, p, b) = make_nplr_hippo::(features); + let a = a.mapv(|x| x.as_complex()); + let p = p + .mapv(|x| Complex::new(x, 0.0)) + .into_shape((features, 1)) + .unwrap(); + let b = b + .mapv(|x| Complex::new(x, 0.0)) + .into_shape((features, 1)) + .unwrap(); + + // + let s = &a + p.dot(&p.t()); + // + let sd = s.diag(); + + let a = Array2::ones(s.dim()) * sd.mean().expect("Average of diagonal is NaN"); + + // TODO: replace with eigh + let (e, v) = eig_csym(&(&s * Complex::new(0.0, -1.0))); + let e = e.mapv(|x| Complex::new(x, 0.0)); + + let a = a + &e * 1.0.as_imag(); + let p = v.conj().t().dot(&p); + let b = v.conj().t().dot(&b); + (a, p, b) +} + +pub fn make_dplr_hippo(features: usize) -> (Array2, Array2, Array2) +where + T: FromPrimitive + NdFloat, +{ + let (a, p, b) = make_nplr_hippo(features); + // broadcast p into a 2D matrix + let p = p.into_shape((features, 1)).unwrap(); + // + let s = &a + p.dot(&p.t()); + // + let sd = s.diag(); + + let a = Array2::ones(s.dim()) * sd.mean().expect("Average of diagonal is NaN"); + + // TODO: finish up by computing the eigh of s * -1j + // let (e, v) = eig_sym(&ss); + (a, p, b.into_shape((features, 1)).unwrap()) +} + +pub struct HiPPO(Array2); + +impl HiPPO +where + T: Float, +{ + pub fn new(hippo: Array2) -> Self { + Self(hippo) + } + + pub fn hippo(&self) -> &Array2 { + &self.0 + } + + pub fn hippo_mut(&mut self) -> &mut Array2 { + &mut self.0 + } +} + +impl HiPPO +where + T: NdFloat, +{ + pub fn create(features: usize) -> Self { + Self(make_hippo(features)) + } +} diff --git a/ml/s4/src/ops/discretize.rs b/ml/s4/src/ops/discretize.rs index 197a8381..d82deb53 100644 --- a/ml/s4/src/ops/discretize.rs +++ b/ml/s4/src/ops/discretize.rs @@ -2,13 +2,34 @@ Appellation: discretize Contrib: FL03 */ +use crate::core::prelude::Inverse; use faer::prelude::{FaerMat, IntoFaer, SolverCore}; use faer::IntoNdarray; use faer_core::zip::ViewMut; -use faer_core::{Conjugate, SimpleEntity}; +use faer_core::{ComplexField, Conjugate, SimpleEntity}; use ndarray::prelude::{Array2, NdFloat}; use nshare::{ToNalgebra, ToNdarray2}; -use num::ToPrimitive; +use num::{Float, ToPrimitive}; + +pub fn discretize( + a: &Array2, + b: &Array2, + c: &Array2, + step: T, +) -> anyhow::Result<(Array2, Array2, Array2)> +where + T: NdFloat, +{ + let ss = step / T::from(2).unwrap(); // half step + let eye = Array2::::eye(a.shape()[0]); + + let be = (&eye - a * ss).inverse().expect("Could not invert matrix"); + + let ab = be.dot(&(&eye + a * ss)); + let bb = (b * ss).dot(&b.t()); + + Ok((ab, bb, c.clone())) +} pub fn discretize_nalgebra( a: &Array2, @@ -41,7 +62,7 @@ pub fn discretize_faer( ) -> (Array2, Array2, Array2) where T: NdFloat + Conjugate + SimpleEntity, - ::Canonical: faer_core::ComplexField + SimpleEntity + ToPrimitive, + ::Canonical: ComplexField + SimpleEntity + ToPrimitive, { let ss = step / T::from(2).unwrap(); // half step let eye = Array2::::eye(a.shape()[0]); @@ -57,9 +78,13 @@ where (ab, bb, c.clone()) } -pub trait Discretize { +pub trait Discretize +where + T: Float, +{ type Output; - fn discretize(&self, step: impl num::Float) -> Self::Output; + + fn discretize(&self, step: T) -> Self::Output; } pub enum DiscretizeArgs {} diff --git a/ml/s4/src/ssm/model.rs b/ml/s4/src/ssm/model.rs index a4b6cfa9..5f66d397 100644 --- a/ml/s4/src/ssm/model.rs +++ b/ml/s4/src/ssm/model.rs @@ -3,22 +3,67 @@ Contrib: FL03 */ use super::SSMConfig; +use crate::neural::Forward; use crate::params::SSMParams::*; use crate::prelude::{scanner, SSMStore}; -use faer::prelude::{FaerMat, IntoFaer, SolverCore}; -use faer::IntoNdarray; -use faer_core::zip::ViewMut; -use faer_core::{ComplexField, Conjugate, SimpleEntity}; use ndarray::prelude::{Array1, Array2, NdFloat}; -use num::{Float, ToPrimitive}; +use ndarray_conv::{Conv2DFftExt, PaddingMode, PaddingSize}; +use num::Float; +use rustfft::FftNum; + +#[derive(Clone, Debug)] +pub struct Discrete { + pub a: Array2, + pub b: Array2, + pub c: Array2, +} + +impl Discrete +where + T: Float, +{ + pub fn new(a: Array2, b: Array2, c: Array2) -> Self { + Self { a, b, c } + } + + pub fn from_features(features: usize) -> Self + where + T: Default, + { + let a = Array2::::eye(features); + let b = Array2::::zeros((features, 1)); + let c = Array2::::zeros((features, features)); + Self { a, b, c } + } +} + +impl From<(Array2, Array2, Array2)> for Discrete +where + T: Float, +{ + fn from((a, b, c): (Array2, Array2, Array2)) -> Self { + Self { a, b, c } + } +} + +impl From> for (Array2, Array2, Array2) +where + T: Float, +{ + fn from(discrete: Discrete) -> Self { + (discrete.a, discrete.b, discrete.c) + } +} pub struct SSM where T: Float, { + cache: Array1, config: SSMConfig, kernel: Array2, params: SSMStore, + ssm: Discrete, } impl SSM @@ -30,12 +75,16 @@ where T: Default, { let features = config.features(); + + let cache = Array1::::zeros(features); let kernel = Array2::::zeros((features, features)); let params = SSMStore::from_features(features); Self { + cache, config, kernel, params, + ssm: Discrete::from_features(features), } } @@ -75,21 +124,32 @@ where impl SSM where - T: NdFloat + Conjugate + SimpleEntity, - ::Canonical: ComplexField + SimpleEntity + ToPrimitive, + T: NdFloat, { - pub fn discretize(&mut self, step: T) -> anyhow::Result<()> { - let ds = step / T::from(2).unwrap(); - let eye = Array2::::eye(self.config.features()); - let bl = &eye - &self.params[A] * ds; - let be = { - let mut tmp = bl.view().into_faer().qr().inverse(); - let arr = &tmp.view_mut().into_ndarray(); - arr.mapv(|i| T::from(i).unwrap()) - }; - let ab = &be.dot(&(&eye + &self.params[A] * ds)); - let bb = (&self.params[B] * ds).dot(&self.params[B].t()); + pub fn discretize(&mut self, step: T) -> anyhow::Result<&Discrete> { + let discrete = + crate::prelude::discretize(&self.params[A], &self.params[B], &self.params[C], step)?; + + self.ssm = discrete.into(); + Ok(&self.ssm) + } +} - Ok(()) +impl Forward> for SSM +where + T: FftNum + NdFloat, +{ + type Output = Array2; + + fn forward(&self, args: &Array2) -> Array2 { + let res = if !self.config().decode() { + let mode = PaddingMode::<2, T>::Const(T::zero()); + let size = PaddingSize::Full; + args.conv_2d_fft(&self.kernel, size, mode) + .expect("convolution failed") + } else { + self.scan(args, &self.cache) + }; + res + args * &self.params[D] } } diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index 8706e45a..c487548d 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -2,9 +2,16 @@ Appellation: utils Contrib: FL03 */ -use ndarray::prelude::{s, Array1, Array2, ArrayView1, NdFloat}; +use crate::core::prelude::{AsComplex, Conjugate}; +use nalgebra::ComplexField; +use ndarray::prelude::*; use ndarray::IntoDimension; -use num::{Complex, Float}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::Uniform; +use ndarray_rand::RandomExt; +use nshare::{ToNalgebra, ToNdarray1, ToNdarray2}; +use num::traits::FloatConst; +use num::{Complex, Float, Zero}; use rustfft::{FftNum, FftPlanner}; pub fn cauchy(v: &Array1, omega: &Array1, lambda: &Array1) -> Array1 @@ -15,6 +22,29 @@ where omega.mapv(cdot) } +pub fn eig_sym(args: &Array2) -> (Array1, Array2) { + let sym = args.clone().into_nalgebra().symmetric_eigen(); + ( + sym.eigenvalues.into_ndarray1(), + sym.eigenvectors.into_ndarray2(), + ) +} + +pub fn eig_csym(args: &Array2>) -> (Array1, Array2>) { + let sym = args.clone().into_nalgebra().symmetric_eigen(); + let (eig, v) = (sym.eigenvalues, sym.eigenvectors); + (eig.into_ndarray1(), v.into_ndarray2()) +} + +pub fn eigh(args: &Array2) -> (Array1, Array2) { + let na = args.clone().into_nalgebra(); + let sym = na.symmetric_eigen(); + let v = sym.eigenvectors; + let eig = sym.eigenvalues.into_ndarray1(); + let eigval = v.into_ndarray2(); + (eig, eigval) +} + pub fn powmat(a: &Array2, n: usize) -> Array2 where T: Float + 'static, @@ -39,6 +69,22 @@ where a.clone() } +pub fn logstep(a: T, b: T, shape: impl IntoDimension) -> Array +where + D: Dimension, + T: NdFloat + SampleUniform, +{ + Array::random(shape, Uniform::new(a, b)) * (b.ln() - a.ln()) + a.ln() +} + +pub fn logstep_init(a: T, b: T) -> impl Fn(D) -> Array +where + D: Dimension, + T: NdFloat + SampleUniform, +{ + move |shape| Array::random(shape, Uniform::new(a, b)) * (b.ln() - a.ln()) + a.ln() +} + pub fn scanner( a: &Array2, b: &Array2, @@ -101,3 +147,52 @@ where Some(y1) } } + +// fn kernel_dplr(lambda: T, p: &Array2, q: &Array2, b: &Array2, c: &Array2, step: T, l: usize) -> Array1> +// where +// T: Conjugate + FloatConst + FftNum + NdFloat, +// { +// let omega_l: Array1> = (0..l) +// .map(|i| Complex::from_polar(T::one(), -T::PI() * T::from(i).unwrap() / T::from(l).unwrap())) +// .collect(); + +// let aterm = (c.conj(), q.conj()); +// let bterm = (b, p); + +// let g = (T::from(2.0).unwrap() / step) * ((T::one().as_complex() - &omega_l) / (&omega_l + T::one())); +// let c = T::from(2.0).unwrap() / (T::from(1.0).unwrap() + &omega_l); + +// let k00 = cauchy(&aterm.0 * bterm.0, &g, lambda); +// let k01 = cauchy(&aterm.0 * bterm.1, &g, lambda); +// let k10 = cauchy(&aterm.1 * bterm.0, &g, lambda); +// let k11 = cauchy(&aterm.1 * bterm.1, &g, lambda); +// let at_roots = &c * (&k00 - k01 * (T::one() / (&k11 + T::one())) * &k10); +// let mut fft_planner = FftPlanner::new(); +// let fft = fft_planner.plan_fft(l, rustfft::FftDirection::Forward); +// let mut at_roots_complex = Array1::zeros(l); +// for (i, val) in at_roots.iter().enumerate() { +// at_roots_complex[i] = Complex::new(*val, T::zero()); +// } +// let mut out = Array1::zeros(l); +// fft.process(&mut out); +// out +// } + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::round_to; + use approx::assert_relative_eq; + use ndarray::prelude::array; + + #[test] + fn test_eig_sym() { + let a = array![[1.0, 2.0], [2.0, 1.0]]; + let (eig, eigval) = eig_sym(&a); + let exp_eig = array![3.0, -1.0]; + let exp_eigval = array![[0.70710678, -0.70710678], [0.70710678, 0.70710678]]; + + assert_relative_eq!(eig, exp_eig); + assert_relative_eq!(eigval.mapv(|i| round_to(i, 8)), exp_eigval); + } +} From f37dbe04d9fac51fb313e8091fa4665d0163c216 Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 22 Dec 2023 13:34:18 -0600 Subject: [PATCH 094/118] update Signed-off-by: FL03 --- ml/s4/src/ssm/model.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/ml/s4/src/ssm/model.rs b/ml/s4/src/ssm/model.rs index 5f66d397..3841e14c 100644 --- a/ml/s4/src/ssm/model.rs +++ b/ml/s4/src/ssm/model.rs @@ -113,6 +113,19 @@ where } } +impl SSM +where + T: NdFloat, +{ + pub fn setup(&mut self) -> &mut Self { + + self.kernel = crate::ops::k_convolve(&self.params[A], &self.params[B], &self.params[C], self.config().samples()); + + self.ssm = self.discretize(T::from(0.1).unwrap()).expect(""); + self + } +} + impl SSM where T: NdFloat, @@ -126,12 +139,10 @@ impl SSM where T: NdFloat, { - pub fn discretize(&mut self, step: T) -> anyhow::Result<&Discrete> { + pub fn discretize(&self, step: T) -> anyhow::Result> { let discrete = crate::prelude::discretize(&self.params[A], &self.params[B], &self.params[C], step)?; - - self.ssm = discrete.into(); - Ok(&self.ssm) + Ok(discrete.into()) } } From 949d75ba0b6a9a6b4773b83a50906ed83a31b40a Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 24 Dec 2023 11:10:52 -0600 Subject: [PATCH 095/118] update Signed-off-by: FL03 --- core/src/specs/math.rs | 25 +++++++++ core/src/utils.rs | 43 +++++++++++---- ml/s4/examples/sand.rs | 14 +++++ ml/s4/src/dplr/hippo.rs | 110 ++++++++++---------------------------- ml/s4/src/dplr/kinds.rs | 65 ++++++++++++++++++++++ ml/s4/src/dplr/mod.rs | 71 ++++++++++++++++++++++++ ml/s4/src/params/store.rs | 20 +++++++ ml/s4/src/ssm/mod.rs | 2 +- ml/s4/src/ssm/model.rs | 40 +++++++------- ml/s4/src/utils.rs | 33 ++++++++++-- 10 files changed, 307 insertions(+), 116 deletions(-) create mode 100644 ml/s4/examples/sand.rs create mode 100644 ml/s4/src/dplr/kinds.rs diff --git a/core/src/specs/math.rs b/core/src/specs/math.rs index 9ef1d0ee..e6308756 100644 --- a/core/src/specs/math.rs +++ b/core/src/specs/math.rs @@ -93,3 +93,28 @@ where Self: Arithmetic, Array>, { } + +pub trait SquareRoot { + fn sqrt(self) -> Self; +} + +impl SquareRoot for f32 { + fn sqrt(self) -> Self { + f32::sqrt(self) + } +} + +impl SquareRoot for f64 { + fn sqrt(self) -> Self { + f64::sqrt(self) + } +} + +impl SquareRoot for Complex +where + T: Float, +{ + fn sqrt(self) -> Self { + Complex::::sqrt(self) + } +} \ No newline at end of file diff --git a/core/src/utils.rs b/core/src/utils.rs index 4d754a1e..d6a312fe 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -6,7 +6,7 @@ use ndarray::prelude::{s, Array, Array1, Array2, Axis, Dimension, NdFloat}; use ndarray::{concatenate, IntoDimension, RemoveAxis, ShapeError}; use num::cast::AsPrimitive; -use num::Float; +use num::{Float, Num, NumCast, Zero}; pub fn arange(a: T, b: T, h: T) -> Array1 where @@ -106,20 +106,41 @@ where Array::linspace(T::zero(), T::from(n - 1).unwrap(), n).into_shape(dim) } +/// creates a matrix from the given shape filled with numerical elements [0, n) spaced evenly by 1 +pub fn rangespace(dim: impl IntoDimension) -> Array where D: Dimension, T: Num + NumCast { + let dim = dim.into_dimension(); + let iter = (0..dim.size()).map(|i| T::from(i).unwrap()); + Array::from_shape_vec(dim, iter.collect()).unwrap() +} + pub fn round_to(val: T, decimals: usize) -> T { let factor = T::from(10).expect("").powi(decimals as i32); (val * factor).round() / factor } -pub fn tril(a: &Array2) -> Array2 -where - T: NdFloat, -{ - let mut out = a.clone(); - for i in 0..a.shape()[0] { - for j in i + 1..a.shape()[1] { - out[[i, j]] = T::zero(); + /// Returns the upper triangular portion of a matrix. + pub fn triu(a: &Array2) -> Array2 + where + T: Clone + Zero, + { + let mut out = a.clone(); + for i in 0..a.shape()[0] { + for j in 0..i { + out[[i, j]] = T::zero(); + } } + out + } + /// Returns the lower triangular portion of a matrix. + pub fn tril(a: &Array2) -> Array2 + where + T: Clone + Zero, + { + let mut out = a.clone(); + for i in 0..a.shape()[0] { + for j in i + 1..a.shape()[1] { + out[[i, j]] = T::zero(); + } + } + out } - out -} diff --git a/ml/s4/examples/sand.rs b/ml/s4/examples/sand.rs new file mode 100644 index 00000000..06e70af3 --- /dev/null +++ b/ml/s4/examples/sand.rs @@ -0,0 +1,14 @@ +// use concision_core as cnc; +use concision_s4 as s4; + +use s4::randcomplex; + +use ndarray::prelude::Ix2; + +fn main() -> anyhow::Result<()> { + let c = randcomplex::([2, 2]); + + println!("{:?}", &c); + + Ok(()) +} diff --git a/ml/s4/src/dplr/hippo.rs b/ml/s4/src/dplr/hippo.rs index 19f5d3bc..e3a5c1ae 100644 --- a/ml/s4/src/dplr/hippo.rs +++ b/ml/s4/src/dplr/hippo.rs @@ -2,88 +2,28 @@ Appellation: hippo Contrib: FL03 */ -use crate::core::prelude::{linarr, tril, AsComplex, Conjugate}; -use crate::prelude::eig_csym; - -use ndarray::prelude::{Array1, Array2, Ix2, NdFloat}; -use num::{Complex, Float, FromPrimitive}; - -pub fn make_hippo(features: usize) -> Array2 -where - T: NdFloat, -{ - let base = linarr::((features, 1)).unwrap(); - let p = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); - let mut a = &p * &p.t(); - a = tril(&a) - &base.diag(); - -a +use super::utils::*; +use ndarray::prelude::Array2; +use ndarray::ScalarOperand; +use num::Float; +use num::complex::ComplexFloat; + +pub enum HiPPOs { + HiPPO(Array2), + NPLR { + a: Array2, + p: Array2, + b: Array2, + }, + DPLR { + a: Array2, + p: Array2, + b: Array2, + c: Array2, + }, } -pub fn make_nplr_hippo(features: usize) -> (Array2, Array1, Array1) -where - T: NdFloat, -{ - let hippo = make_hippo(features); - let base = Array1::linspace(T::zero(), T::from(features - 1).unwrap(), features); - let p = (&base + T::one() / T::from(2).unwrap()).mapv(T::sqrt); - let b = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); - (hippo, p, b) -} - -pub fn dplr_hippo( - features: usize, -) -> ( - Array2>, - Array2>, - Array2>, -) { - let (a, p, b) = make_nplr_hippo::(features); - let a = a.mapv(|x| x.as_complex()); - let p = p - .mapv(|x| Complex::new(x, 0.0)) - .into_shape((features, 1)) - .unwrap(); - let b = b - .mapv(|x| Complex::new(x, 0.0)) - .into_shape((features, 1)) - .unwrap(); - - // - let s = &a + p.dot(&p.t()); - // - let sd = s.diag(); - - let a = Array2::ones(s.dim()) * sd.mean().expect("Average of diagonal is NaN"); - - // TODO: replace with eigh - let (e, v) = eig_csym(&(&s * Complex::new(0.0, -1.0))); - let e = e.mapv(|x| Complex::new(x, 0.0)); - - let a = a + &e * 1.0.as_imag(); - let p = v.conj().t().dot(&p); - let b = v.conj().t().dot(&b); - (a, p, b) -} - -pub fn make_dplr_hippo(features: usize) -> (Array2, Array2, Array2) -where - T: FromPrimitive + NdFloat, -{ - let (a, p, b) = make_nplr_hippo(features); - // broadcast p into a 2D matrix - let p = p.into_shape((features, 1)).unwrap(); - // - let s = &a + p.dot(&p.t()); - // - let sd = s.diag(); - - let a = Array2::ones(s.dim()) * sd.mean().expect("Average of diagonal is NaN"); - - // TODO: finish up by computing the eigh of s * -1j - // let (e, v) = eig_sym(&ss); - (a, p, b.into_shape((features, 1)).unwrap()) -} pub struct HiPPO(Array2); @@ -106,9 +46,17 @@ where impl HiPPO where - T: NdFloat, + T: ComplexFloat + ScalarOperand, { - pub fn create(features: usize) -> Self { + + pub fn square(features: usize) -> Self { Self(make_hippo(features)) } + + pub fn nplr(features: usize) -> Self { + let (hippo, p, b) = make_nplr_hippo(features); + Self(hippo) + } + + } diff --git a/ml/s4/src/dplr/kinds.rs b/ml/s4/src/dplr/kinds.rs new file mode 100644 index 00000000..c6a44219 --- /dev/null +++ b/ml/s4/src/dplr/kinds.rs @@ -0,0 +1,65 @@ +/* + Appellation: kinds + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumCount, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum Rank { + #[default] + Low +} + + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumCount, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum Mode { + #[default] + Base, + #[strum(serialize = "nplr")] + NPLR, + #[strum(serialize = "dplr")] + DPLR, +} \ No newline at end of file diff --git a/ml/s4/src/dplr/mod.rs b/ml/s4/src/dplr/mod.rs index 5d9f53bc..cc291a64 100644 --- a/ml/s4/src/dplr/mod.rs +++ b/ml/s4/src/dplr/mod.rs @@ -5,5 +5,76 @@ //! # Diagonal Plus Low Rank (DPLR) //! //! +pub use self::{kinds::*, utils::*}; + +pub(crate) mod kinds; pub mod hippo; + +pub struct LowRank { + pub mode: Mode, +} + +pub(crate) mod utils { + use crate::core::prelude::{rangespace, tril, AsComplex, Conjugate, }; + use crate::prelude::eig_csym; + + use ndarray::prelude::{Array1, Array2, Axis}; + use ndarray::ScalarOperand; + use num::Complex; + use num::complex::ComplexFloat; + + + pub fn make_hippo(features: usize) -> Array2 + where + T: ComplexFloat + ScalarOperand, + { + let base = rangespace((features, 1)); + let p = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); + let mut a = &p * &p.t(); + a = tril(&a) - &base.diag(); + -a + } + + pub fn make_nplr_hippo(features: usize) -> (Array2, Array1, Array1) + where + T: ComplexFloat + ScalarOperand, + { + let hippo = make_hippo(features); + + let base = rangespace((features,)); + let p = (&base + T::one() / T::from(2).unwrap()).mapv(T::sqrt); + let b = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); + (hippo, p, b) + } + + pub fn dplr_hippo( + features: usize, + ) -> ( + Array2>, + Array2>, + Array2>, + Array2>, + ) { + let (a, p, b) = make_nplr_hippo::>(features); + let p = p.insert_axis(Axis(1)); + let b = b.insert_axis(Axis(1)); + + // + let s = &a + p.dot(&p.t()); + // + let sd = s.diag(); + + let a = Array2::ones(s.dim()) * sd.mean().expect("Average of diagonal is NaN"); + + // TODO: replace with eigh + let (e, v) = eig_csym(&(&s * (-1.0).as_imag())); + let e = e.mapv(|x| x.as_complex()); + + let a = a + &e * 1.0.as_imag(); + let p = v.conj().t().dot(&p); + let b = v.conj().t().dot(&b); + (a, p, b, v) + } + +} \ No newline at end of file diff --git a/ml/s4/src/params/store.rs b/ml/s4/src/params/store.rs index 93d32cb5..14e12624 100644 --- a/ml/s4/src/params/store.rs +++ b/ml/s4/src/params/store.rs @@ -6,6 +6,8 @@ use super::SSMParams; use crate::core::prelude::GenerateRandom; use crate::prelude::scanner; use ndarray::prelude::{Array1, Array2, NdFloat}; +use ndarray_rand::RandomExt; +use ndarray_rand::rand_distr::{StandardNormal, Distribution}; use ndarray_rand::rand_distr::uniform::SampleUniform; use num::Float; use serde::{Deserialize, Serialize}; @@ -103,7 +105,15 @@ where impl SSMStore where T: Float + SampleUniform, + StandardNormal: Distribution, { + pub fn init(mut self, features: usize) -> Self { + // let (lambda, p, b, _v) = dplr_hippo(features); + self.c = Array2::::random((1, features), StandardNormal); + self.d = Array2::::ones((1, 1)); + self + } + pub fn uniform(features: usize) -> Self { let dk = T::one() / T::from(features).unwrap().sqrt(); let a = Array2::::uniform_between(dk, (features, features)); @@ -155,6 +165,16 @@ where } } +impl<'a, T> From<&'a SSMStore> for (&'a Array2, &'a Array2, &'a Array2, &'a Array2) +where + T: Float, +{ + fn from(store: &'a SSMStore) -> Self { + (&store.a, &store.b, &store.c, &store.d) + } +} + + impl From<(Array2, Array2, Array2, Array2)> for SSMStore where T: Float, diff --git a/ml/s4/src/ssm/mod.rs b/ml/s4/src/ssm/mod.rs index bb1df0da..7a2dec3b 100644 --- a/ml/s4/src/ssm/mod.rs +++ b/ml/s4/src/ssm/mod.rs @@ -25,7 +25,7 @@ mod tests { let step = 0.001; let config = SSMConfig::new(true, 9, 2); - let mut model = SSM::::create(config); + let model = SSM::::create(config).setup(); assert!(model.discretize(step).is_ok()); } } diff --git a/ml/s4/src/ssm/model.rs b/ml/s4/src/ssm/model.rs index 3841e14c..90ee22ae 100644 --- a/ml/s4/src/ssm/model.rs +++ b/ml/s4/src/ssm/model.rs @@ -4,8 +4,8 @@ */ use super::SSMConfig; use crate::neural::Forward; -use crate::params::SSMParams::*; -use crate::prelude::{scanner, SSMStore}; +use crate::params::{SSMParams::*, SSMStore}; +use crate::prelude::{discretize, scanner, k_convolve,}; use ndarray::prelude::{Array1, Array2, NdFloat}; use ndarray_conv::{Conv2DFftExt, PaddingMode, PaddingSize}; use num::Float; @@ -117,11 +117,10 @@ impl SSM where T: NdFloat, { - pub fn setup(&mut self) -> &mut Self { - - self.kernel = crate::ops::k_convolve(&self.params[A], &self.params[B], &self.params[C], self.config().samples()); - - self.ssm = self.discretize(T::from(0.1).unwrap()).expect(""); + pub fn setup(mut self) -> Self { + self.kernel = self.gen_filter(); + + self.ssm = self.discretize(self.config().step_size()).expect(""); self } } @@ -133,17 +132,25 @@ where pub fn scan(&self, u: &Array2, x0: &Array1) -> Array2 { scanner(&self.params[A], &self.params[B], &self.params[C], u, x0) } -} -impl SSM -where - T: NdFloat, -{ + pub fn conv(&self, u: &Array2) -> anyhow::Result> where T: FftNum { + let mode = PaddingMode::<2, T>::Const(T::zero()); + let size = PaddingSize::Full; + if let Some(res) = u.conv_2d_fft(&self.kernel, size, mode) { + Ok(res) + } else { + Err(anyhow::anyhow!("convolution failed")) + } + } + pub fn discretize(&self, step: T) -> anyhow::Result> { - let discrete = - crate::prelude::discretize(&self.params[A], &self.params[B], &self.params[C], step)?; + let discrete = discretize(&self.params[A], &self.params[B], &self.params[C], step)?; Ok(discrete.into()) } + + pub fn gen_filter(&self) -> Array2 { + k_convolve(&self.params[A], &self.params[B], &self.params[C], self.config().samples()) + } } impl Forward> for SSM @@ -154,10 +161,7 @@ where fn forward(&self, args: &Array2) -> Array2 { let res = if !self.config().decode() { - let mode = PaddingMode::<2, T>::Const(T::zero()); - let size = PaddingSize::Full; - args.conv_2d_fft(&self.kernel, size, mode) - .expect("convolution failed") + self.conv(args).expect("convolution failed") } else { self.scan(args, &self.cache) }; diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index c487548d..c5e92557 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -2,18 +2,41 @@ Appellation: utils Contrib: FL03 */ -use crate::core::prelude::{AsComplex, Conjugate}; -use nalgebra::ComplexField; use ndarray::prelude::*; use ndarray::IntoDimension; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::Uniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use ndarray_rand::RandomExt; use nshare::{ToNalgebra, ToNdarray1, ToNdarray2}; -use num::traits::FloatConst; -use num::{Complex, Float, Zero}; +use num::complex::ComplexFloat; +use num::{Complex, Float,}; use rustfft::{FftNum, FftPlanner}; +pub fn stdnorm(shape: impl IntoDimension) -> Array +where + D: Dimension, + StandardNormal: Distribution, +{ + Array::random(shape, StandardNormal) +} + +pub fn randcomplex(shape: impl IntoDimension) -> Array, D> +where + D: Dimension, + T: ComplexFloat, + StandardNormal: Distribution, +{ + let dim = shape.into_dimension(); + let re = Array::random(dim.clone(), StandardNormal); + let im = Array::random(dim.clone(), StandardNormal); + let mut res = Array::zeros(dim); + ndarray::azip!((re in &re, im in &im, res in &mut res) { + *res = Complex::new(*re, *im); + }); + res +} + pub fn cauchy(v: &Array1, omega: &Array1, lambda: &Array1) -> Array1 where T: NdFloat, @@ -82,7 +105,7 @@ where D: Dimension, T: NdFloat + SampleUniform, { - move |shape| Array::random(shape, Uniform::new(a, b)) * (b.ln() - a.ln()) + a.ln() + move |shape| logstep(a, b, shape) } pub fn scanner( From e976fae0a73ea885049d9cc81902cb57a2d3dd70 Mon Sep 17 00:00:00 2001 From: FL03 Date: Tue, 26 Dec 2023 22:30:22 -0600 Subject: [PATCH 096/118] update Signed-off-by: FL03 --- Cargo.toml | 8 ++-- core/src/specs/math.rs | 2 +- core/src/specs/mod.rs | 13 +++++++ core/src/utils.rs | 50 +++++++++++++------------ data/Cargo.toml | 2 +- data/src/lib.rs | 2 +- ml/linear/Cargo.toml | 1 - ml/neural/Cargo.toml | 2 +- ml/neural/src/func/activate/nl/mod.rs | 2 +- ml/neural/src/func/loss/reg/mod.rs | 3 +- ml/neural/src/func/rms.rs | 3 +- ml/neural/src/ops/mod.rs | 2 +- ml/nlp/Cargo.toml | 3 +- ml/nlp/src/embed/context/mod.rs | 4 -- ml/nlp/src/embed/mod.rs | 4 +- ml/nlp/src/embed/words/mod.rs | 4 -- ml/nlp/src/encode/mod.rs | 6 +-- ml/nlp/src/encode/positional.rs | 2 +- ml/nlp/src/lib.rs | 3 ++ ml/s4/Cargo.toml | 5 +-- ml/s4/src/dplr/hippo.rs | 7 +--- ml/s4/src/dplr/kinds.rs | 5 +-- ml/s4/src/dplr/mod.rs | 36 ++++++++++++++++-- ml/s4/src/model/model.rs | 3 +- ml/s4/src/ops/discretize.rs | 53 +-------------------------- ml/s4/src/params/store.rs | 5 +-- ml/s4/src/ssm/model.rs | 16 ++++++-- ml/s4/src/utils.rs | 29 +++++++++++++-- ml/transformers/Cargo.toml | 3 +- ml/transformers/src/transform/mod.rs | 4 +- 30 files changed, 143 insertions(+), 139 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c60f5816..4464ade5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,17 +11,15 @@ repository = "https://github.com/FL03/concision" version = "0.1.12" # TODO - Update the cargo package version [workspace.dependencies] -computare = { features = ["full"], branch = "v0.1.0", git = "https://github.com/FL03/computare", version = "0.1.0" } +# computare = { features = ["full"], branch = "v0.1.0", git = "https://github.com/FL03/computare", version = "0.1.0" } anyhow = "1" approx = "0.5" -faer = { features = ["ndarray"], version = "0.16" } -faer-core = "0.16" itertools = { features = [], version = "0.12" } lazy_static = "1" nalgebra = { features = [], version = "0.32" } -ndarray = { features = ["serde-1"], version = "0.15" } -# ndarray-linalg = { features = [], version = "0.16" } +ndarray = { features = ["blas", "serde-1"], version = "0.15" } +ndarray-linalg = { features = ["intel-mkl-system"], version = "0.16" } ndarray-rand = { features = [], version = "0.14" } ndarray-stats = { features = [], version = "0.5.1" } num = { features = ["serde"], version = "0.4" } diff --git a/core/src/specs/math.rs b/core/src/specs/math.rs index e6308756..2ab59c89 100644 --- a/core/src/specs/math.rs +++ b/core/src/specs/math.rs @@ -117,4 +117,4 @@ where fn sqrt(self) -> Self { Complex::::sqrt(self) } -} \ No newline at end of file +} diff --git a/core/src/specs/mod.rs b/core/src/specs/mod.rs index dda39a6c..f929a11a 100644 --- a/core/src/specs/mod.rs +++ b/core/src/specs/mod.rs @@ -30,6 +30,19 @@ where } } +pub trait RoundTo { + fn round_to(&self, places: usize) -> Self; +} + +impl RoundTo for T +where + T: num::Float, +{ + fn round_to(&self, places: usize) -> Self { + crate::round_to(*self, places) + } +} + #[cfg(test)] mod tests { diff --git a/core/src/utils.rs b/core/src/utils.rs index d6a312fe..cd87120b 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -107,7 +107,11 @@ where } /// creates a matrix from the given shape filled with numerical elements [0, n) spaced evenly by 1 -pub fn rangespace(dim: impl IntoDimension) -> Array where D: Dimension, T: Num + NumCast { +pub fn rangespace(dim: impl IntoDimension) -> Array +where + D: Dimension, + T: Num + NumCast, +{ let dim = dim.into_dimension(); let iter = (0..dim.size()).map(|i| T::from(i).unwrap()); Array::from_shape_vec(dim, iter.collect()).unwrap() @@ -118,29 +122,29 @@ pub fn round_to(val: T, decimals: usize) -> T { (val * factor).round() / factor } - /// Returns the upper triangular portion of a matrix. - pub fn triu(a: &Array2) -> Array2 - where - T: Clone + Zero, - { - let mut out = a.clone(); - for i in 0..a.shape()[0] { - for j in 0..i { - out[[i, j]] = T::zero(); - } +/// Returns the upper triangular portion of a matrix. +pub fn triu(a: &Array2) -> Array2 +where + T: Clone + Zero, +{ + let mut out = a.clone(); + for i in 0..a.shape()[0] { + for j in 0..i { + out[[i, j]] = T::zero(); } - out } - /// Returns the lower triangular portion of a matrix. - pub fn tril(a: &Array2) -> Array2 - where - T: Clone + Zero, - { - let mut out = a.clone(); - for i in 0..a.shape()[0] { - for j in i + 1..a.shape()[1] { - out[[i, j]] = T::zero(); - } + out +} +/// Returns the lower triangular portion of a matrix. +pub fn tril(a: &Array2) -> Array2 +where + T: Clone + Zero, +{ + let mut out = a.clone(); + for i in 0..a.shape()[0] { + for j in i + 1..a.shape()[1] { + out[[i, j]] = T::zero(); } - out } + out +} diff --git a/data/Cargo.toml b/data/Cargo.toml index e78970c2..cb5434c4 100644 --- a/data/Cargo.toml +++ b/data/Cargo.toml @@ -25,7 +25,7 @@ test = true [dependencies] anyhow.workspace = true -linfa = { features = [], version = "0.7" } +# linfa = { features = [], version = "0.7" } ndarray.workspace = true num.workspace = true serde.workspace = true diff --git a/data/src/lib.rs b/data/src/lib.rs index 161393fe..77dc561c 100644 --- a/data/src/lib.rs +++ b/data/src/lib.rs @@ -18,7 +18,7 @@ pub mod flows; pub mod tensors; pub mod prelude { - pub use linfa::dataset::{Dataset, DatasetBase, DatasetView}; + // pub use linfa::dataset::{Dataset, DatasetBase, DatasetView}; pub use crate::datasets::*; pub use crate::df::*; diff --git a/ml/linear/Cargo.toml b/ml/linear/Cargo.toml index ee2a13b6..e5c9124e 100644 --- a/ml/linear/Cargo.toml +++ b/ml/linear/Cargo.toml @@ -40,7 +40,6 @@ smart-default.workspace = true strum.workspace = true [dev-dependencies] -computare.workspace = true [package.metadata.docs.rs] all-features = true diff --git a/ml/neural/Cargo.toml b/ml/neural/Cargo.toml index 9ecf31b1..62110e7e 100644 --- a/ml/neural/Cargo.toml +++ b/ml/neural/Cargo.toml @@ -40,7 +40,7 @@ smart-default.workspace = true strum.workspace = true [dev-dependencies] -computare.workspace = true +# computare.workspace = true [package.metadata.docs.rs] all-features = true diff --git a/ml/neural/src/func/activate/nl/mod.rs b/ml/neural/src/func/activate/nl/mod.rs index bd59a74a..6abf8dab 100644 --- a/ml/neural/src/func/activate/nl/mod.rs +++ b/ml/neural/src/func/activate/nl/mod.rs @@ -70,8 +70,8 @@ pub(crate) mod utils { #[cfg(test)] mod tests { use super::*; + use crate::core::prelude::RoundTo; use crate::prelude::Activate; - use computare::prelude::RoundTo; use ndarray::array; #[test] diff --git a/ml/neural/src/func/loss/reg/mod.rs b/ml/neural/src/func/loss/reg/mod.rs index 5049a0d0..f196fb9d 100644 --- a/ml/neural/src/func/loss/reg/mod.rs +++ b/ml/neural/src/func/loss/reg/mod.rs @@ -67,9 +67,8 @@ pub(crate) mod utils { #[cfg(test)] mod tests { use super::*; - use crate::core::GenerateRandom; + use crate::core::{GenerateRandom, RoundTo}; use crate::func::loss::Loss; - use computare::prelude::RoundTo; use ndarray::prelude::{Array, Ix2}; use ndarray_stats::DeviationExt; diff --git a/ml/neural/src/func/rms.rs b/ml/neural/src/func/rms.rs index 853de250..f42dc3c7 100644 --- a/ml/neural/src/func/rms.rs +++ b/ml/neural/src/func/rms.rs @@ -40,8 +40,7 @@ where #[cfg(test)] mod tests { use super::*; - use crate::core::prelude::linarr; - use computare::prelude::RoundTo; + use crate::core::prelude::{linarr, RoundTo}; use ndarray::prelude::Ix1; #[test] diff --git a/ml/neural/src/ops/mod.rs b/ml/neural/src/ops/mod.rs index e7dcc92e..e8b3d4cc 100644 --- a/ml/neural/src/ops/mod.rs +++ b/ml/neural/src/ops/mod.rs @@ -13,7 +13,7 @@ pub(crate) mod utils {} mod tests { use super::*; use crate::prelude::Forward; - use computare::prelude::RoundTo; + use concision_core::prelude::RoundTo; use ndarray::prelude::{array, Array, Ix2}; #[test] diff --git a/ml/nlp/Cargo.toml b/ml/nlp/Cargo.toml index d0ddf256..bed7d377 100644 --- a/ml/nlp/Cargo.toml +++ b/ml/nlp/Cargo.toml @@ -24,6 +24,7 @@ test = true [build-dependencies] [dependencies] + anyhow.workspace = true finalfusion = "0.18" ndarray.workspace = true @@ -35,7 +36,7 @@ strum.workspace = true tokenizers = { features = [], version = "0.15" } [dev-dependencies] -computare.workspace = true +concision-core = { features = [], path = "../../core", version = "0.1.12" } [package.metadata.docs.rs] all-features = true diff --git a/ml/nlp/src/embed/context/mod.rs b/ml/nlp/src/embed/context/mod.rs index c93d20d4..8ccae58f 100644 --- a/ml/nlp/src/embed/context/mod.rs +++ b/ml/nlp/src/embed/context/mod.rs @@ -4,7 +4,3 @@ */ //! # Contextual Embedding //! - -pub use self::utils::*; - -pub(crate) mod utils {} diff --git a/ml/nlp/src/embed/mod.rs b/ml/nlp/src/embed/mod.rs index f2b80d40..2b414b65 100644 --- a/ml/nlp/src/embed/mod.rs +++ b/ml/nlp/src/embed/mod.rs @@ -11,7 +11,7 @@ //! generated by a model that takes into account the context of the word or phrase //! in question. This module provides a variety of embedding solutions for use in //! natural language processing applications. -pub use self::{embedding::*, utils::*}; +pub use self::embedding::*; pub(crate) mod embedding; @@ -21,5 +21,3 @@ pub mod words; pub trait Embed { fn embed(&self) -> Embedding; } - -pub(crate) mod utils {} diff --git a/ml/nlp/src/embed/words/mod.rs b/ml/nlp/src/embed/words/mod.rs index ca206721..efbd8c89 100644 --- a/ml/nlp/src/embed/words/mod.rs +++ b/ml/nlp/src/embed/words/mod.rs @@ -5,8 +5,4 @@ //! # Word Embedding //! -pub use self::utils::*; - pub mod word2vec; - -pub(crate) mod utils {} diff --git a/ml/nlp/src/encode/mod.rs b/ml/nlp/src/encode/mod.rs index dc114db2..f5dbeeac 100644 --- a/ml/nlp/src/encode/mod.rs +++ b/ml/nlp/src/encode/mod.rs @@ -2,8 +2,6 @@ Appellation: encode Contrib: FL03 */ -pub use self::utils::*; - pub mod positional; use ndarray::prelude::{Array, Array2}; @@ -20,4 +18,6 @@ pub trait EncodeArr { fn encode(&self, data: &Array) -> Array2; } -pub(crate) mod utils {} + +#[cfg(test)] +mod tests {} diff --git a/ml/nlp/src/encode/positional.rs b/ml/nlp/src/encode/positional.rs index 9f343cb6..f647344a 100644 --- a/ml/nlp/src/encode/positional.rs +++ b/ml/nlp/src/encode/positional.rs @@ -143,7 +143,7 @@ impl PositionalEncoderParams { #[cfg(test)] mod tests { use super::*; - use computare::prelude::RoundTo; + use crate::core::prelude::RoundTo; use ndarray::prelude::{array, Array}; #[test] diff --git a/ml/nlp/src/lib.rs b/ml/nlp/src/lib.rs index 9c75e27c..8ddedbe2 100644 --- a/ml/nlp/src/lib.rs +++ b/ml/nlp/src/lib.rs @@ -12,6 +12,9 @@ pub(crate) mod utils; pub mod embed; pub mod encode; +#[cfg(test)] +pub use concision_core as core; + pub mod prelude { pub use crate::embed::*; pub use crate::encode::*; diff --git a/ml/s4/Cargo.toml b/ml/s4/Cargo.toml index 230c7c20..2c99f8b7 100644 --- a/ml/s4/Cargo.toml +++ b/ml/s4/Cargo.toml @@ -29,13 +29,11 @@ concision-core = { path = "../../core", version = "0.1.12" } concision-neural = { path = "../neural" } anyhow.workspace = true -faer-core.workspace = true -faer.workspace = true lazy_static.workspace = true nalgebra.workspace = true ndarray.workspace = true ndarray-conv = "0.2" -# ndarray-linalg.workspace = true +ndarray-linalg.workspace = true ndarray-rand.workspace = true ndarray-stats.workspace = true nshare = "0.9" @@ -48,7 +46,6 @@ strum.workspace = true [dev-dependencies] approx.workspace = true -computare.workspace = true [package.metadata.docs.rs] all-features = true diff --git a/ml/s4/src/dplr/hippo.rs b/ml/s4/src/dplr/hippo.rs index e3a5c1ae..7497721b 100644 --- a/ml/s4/src/dplr/hippo.rs +++ b/ml/s4/src/dplr/hippo.rs @@ -5,8 +5,8 @@ use super::utils::*; use ndarray::prelude::Array2; use ndarray::ScalarOperand; -use num::Float; use num::complex::ComplexFloat; +use num::Float; pub enum HiPPOs { HiPPO(Array2), @@ -23,8 +23,6 @@ pub enum HiPPOs { }, } - - pub struct HiPPO(Array2); impl HiPPO @@ -48,7 +46,6 @@ impl HiPPO where T: ComplexFloat + ScalarOperand, { - pub fn square(features: usize) -> Self { Self(make_hippo(features)) } @@ -57,6 +54,4 @@ where let (hippo, p, b) = make_nplr_hippo(features); Self(hippo) } - - } diff --git a/ml/s4/src/dplr/kinds.rs b/ml/s4/src/dplr/kinds.rs index c6a44219..decab8d7 100644 --- a/ml/s4/src/dplr/kinds.rs +++ b/ml/s4/src/dplr/kinds.rs @@ -29,10 +29,9 @@ use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; #[strum(serialize_all = "lowercase")] pub enum Rank { #[default] - Low + Low, } - #[derive( Clone, Copy, @@ -62,4 +61,4 @@ pub enum Mode { NPLR, #[strum(serialize = "dplr")] DPLR, -} \ No newline at end of file +} diff --git a/ml/s4/src/dplr/mod.rs b/ml/s4/src/dplr/mod.rs index cc291a64..1e361cde 100644 --- a/ml/s4/src/dplr/mod.rs +++ b/ml/s4/src/dplr/mod.rs @@ -16,14 +16,14 @@ pub struct LowRank { } pub(crate) mod utils { - use crate::core::prelude::{rangespace, tril, AsComplex, Conjugate, }; + use crate::core::prelude::{rangespace, tril, AsComplex, Conjugate}; use crate::prelude::eig_csym; use ndarray::prelude::{Array1, Array2, Axis}; use ndarray::ScalarOperand; - use num::Complex; + use ndarray_linalg::{eigh::Eigh, UPLO}; use num::complex::ComplexFloat; - + use num::Complex; pub fn make_hippo(features: usize) -> Array2 where @@ -77,4 +77,32 @@ pub(crate) mod utils { (a, p, b, v) } -} \ No newline at end of file + pub fn make_dplr_hippo( + features: usize, + ) -> ( + Array2>, + Array2>, + Array2>, + Array2>, + ) { + let (a, p, b) = make_nplr_hippo::>(features); + let p = p.insert_axis(Axis(1)); + let b = b.insert_axis(Axis(1)); + + // + let s = &a + p.dot(&p.t()); + // + let sd = s.diag(); + + let a = Array2::ones(s.dim()) * sd.mean().expect("Average of diagonal is NaN"); + + // TODO: replace with eigh + let (e, v) = &(&s * (-1.0).as_imag()).eigh(UPLO::Lower).expect(""); + let e = e.mapv(|x| x.as_complex()); + + let a = a + &e * 1.0.as_imag(); + let p = v.conj().t().dot(&p); + let b = v.conj().t().dot(&b); + (a, p, b, v.clone()) + } +} diff --git a/ml/s4/src/model/model.rs b/ml/s4/src/model/model.rs index 175aaedb..04fcc3e8 100644 --- a/ml/s4/src/model/model.rs +++ b/ml/s4/src/model/model.rs @@ -7,7 +7,8 @@ use crate::neural::prelude::Forward; use crate::prelude::SSMStore; use ndarray::prelude::{Array1, Array2, NdFloat}; use ndarray_conv::{Conv2DFftExt, PaddingMode, PaddingSize}; -use num::{Complex, Float}; +use num::Float; +// use num::complex::{Complex, ComplexFloat}; use rustfft::FftNum; use crate::prelude::SSMParams::*; diff --git a/ml/s4/src/ops/discretize.rs b/ml/s4/src/ops/discretize.rs index d82deb53..f6583bea 100644 --- a/ml/s4/src/ops/discretize.rs +++ b/ml/s4/src/ops/discretize.rs @@ -3,11 +3,9 @@ Contrib: FL03 */ use crate::core::prelude::Inverse; -use faer::prelude::{FaerMat, IntoFaer, SolverCore}; -use faer::IntoNdarray; -use faer_core::zip::ViewMut; -use faer_core::{ComplexField, Conjugate, SimpleEntity}; + use ndarray::prelude::{Array2, NdFloat}; + use nshare::{ToNalgebra, ToNdarray2}; use num::{Float, ToPrimitive}; @@ -31,53 +29,6 @@ where Ok((ab, bb, c.clone())) } -pub fn discretize_nalgebra( - a: &Array2, - b: &Array2, - c: &Array2, - step: f64, -) -> anyhow::Result<(Array2, Array2, Array2)> { - let ss = step / 2.0; // half step - let eye = Array2::::eye(a.shape()[0]); - let bl = &eye - a * ss; - let be = { - let tmp = bl.into_nalgebra().lu().try_inverse(); - if let Some(arg) = tmp { - arg.into_ndarray2() - } else { - return Err(anyhow::anyhow!("Could not invert matrix")); - } - }; - let ab = be.dot(&(&eye + a * ss)); - let bb = (b * ss).dot(&b.t()); - - Ok((ab, bb, c.clone())) -} - -pub fn discretize_faer( - a: &Array2, - b: &Array2, - c: &Array2, - step: T, -) -> (Array2, Array2, Array2) -where - T: NdFloat + Conjugate + SimpleEntity, - ::Canonical: ComplexField + SimpleEntity + ToPrimitive, -{ - let ss = step / T::from(2).unwrap(); // half step - let eye = Array2::::eye(a.shape()[0]); - let bl = &eye - a * ss; - let be = { - let mut tmp = bl.view().into_faer().partial_piv_lu().inverse(); - let arr = &tmp.view_mut().into_ndarray(); - arr.mapv(|i| T::from(i).unwrap()) - }; - let ab = be.dot(&(&eye + a * ss)); - let bb = (b * ss).dot(&b.t()); - - (ab, bb, c.clone()) -} - pub trait Discretize where T: Float, diff --git a/ml/s4/src/params/store.rs b/ml/s4/src/params/store.rs index 14e12624..82739e35 100644 --- a/ml/s4/src/params/store.rs +++ b/ml/s4/src/params/store.rs @@ -6,9 +6,9 @@ use super::SSMParams; use crate::core::prelude::GenerateRandom; use crate::prelude::scanner; use ndarray::prelude::{Array1, Array2, NdFloat}; -use ndarray_rand::RandomExt; -use ndarray_rand::rand_distr::{StandardNormal, Distribution}; use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; +use ndarray_rand::RandomExt; use num::Float; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -174,7 +174,6 @@ where } } - impl From<(Array2, Array2, Array2, Array2)> for SSMStore where T: Float, diff --git a/ml/s4/src/ssm/model.rs b/ml/s4/src/ssm/model.rs index 90ee22ae..af65165d 100644 --- a/ml/s4/src/ssm/model.rs +++ b/ml/s4/src/ssm/model.rs @@ -5,7 +5,7 @@ use super::SSMConfig; use crate::neural::Forward; use crate::params::{SSMParams::*, SSMStore}; -use crate::prelude::{discretize, scanner, k_convolve,}; +use crate::prelude::{discretize, k_convolve, scanner}; use ndarray::prelude::{Array1, Array2, NdFloat}; use ndarray_conv::{Conv2DFftExt, PaddingMode, PaddingSize}; use num::Float; @@ -117,7 +117,7 @@ impl SSM where T: NdFloat, { - pub fn setup(mut self) -> Self { + pub fn setup(mut self) -> Self { self.kernel = self.gen_filter(); self.ssm = self.discretize(self.config().step_size()).expect(""); @@ -133,7 +133,10 @@ where scanner(&self.params[A], &self.params[B], &self.params[C], u, x0) } - pub fn conv(&self, u: &Array2) -> anyhow::Result> where T: FftNum { + pub fn conv(&self, u: &Array2) -> anyhow::Result> + where + T: FftNum, + { let mode = PaddingMode::<2, T>::Const(T::zero()); let size = PaddingSize::Full; if let Some(res) = u.conv_2d_fft(&self.kernel, size, mode) { @@ -149,7 +152,12 @@ where } pub fn gen_filter(&self) -> Array2 { - k_convolve(&self.params[A], &self.params[B], &self.params[C], self.config().samples()) + k_convolve( + &self.params[A], + &self.params[B], + &self.params[C], + self.config().samples(), + ) } } diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index c5e92557..8d45c331 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -5,12 +5,11 @@ use ndarray::prelude::*; use ndarray::IntoDimension; use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::Uniform; -use ndarray_rand::rand_distr::{Distribution, StandardNormal}; +use ndarray_rand::rand_distr::{Distribution, StandardNormal, Uniform}; use ndarray_rand::RandomExt; use nshare::{ToNalgebra, ToNdarray1, ToNdarray2}; use num::complex::ComplexFloat; -use num::{Complex, Float,}; +use num::{Complex, Float}; use rustfft::{FftNum, FftPlanner}; pub fn stdnorm(shape: impl IntoDimension) -> Array @@ -132,6 +131,30 @@ where res } +pub fn scan_ssm( + a: &Array2, + b: &Array2, + c: &Array2, + u: &Array2, + x0: &Array1, +) -> Array2 +where + T: ComplexFloat + 'static, +{ + let step = |xs: &mut Array1, us: ArrayView1| { + let x1 = a.dot(xs) + b.dot(&us); + let y1 = c.dot(&x1); + Some(y1) + }; + let scan = u.outer_iter().scan(x0.clone(), step).collect::>(); + let shape = [scan.len(), scan[0].len()]; + let mut res = Array2::::zeros(shape.into_dimension()); + for (i, s) in scan.iter().enumerate() { + res.slice_mut(s![i, ..]).assign(s); + } + res +} + pub fn scan_complex( a: &Array2>, b: &Array2, diff --git a/ml/transformers/Cargo.toml b/ml/transformers/Cargo.toml index ecab8ede..69bb9d6f 100644 --- a/ml/transformers/Cargo.toml +++ b/ml/transformers/Cargo.toml @@ -26,12 +26,11 @@ test = true [dependencies] concision-core = { path = "../../core", version = "0.1.12" } # concision-data = { path = "../../data" } -concision-neural = { path = "../neural" } +concision-neural = { path = "../neural", version = "0.1.12" } anyhow.workspace = true lazy_static.workspace = true ndarray.workspace = true -# ndarray-linalg.workspace = true ndarray-rand.workspace = true ndarray-stats.workspace = true num.workspace = true diff --git a/ml/transformers/src/transform/mod.rs b/ml/transformers/src/transform/mod.rs index b3bb0b33..c408f6eb 100644 --- a/ml/transformers/src/transform/mod.rs +++ b/ml/transformers/src/transform/mod.rs @@ -3,13 +3,11 @@ Contrib: FL03 */ //! # Transform -pub use self::{config::*, transformer::*, utils::*}; +pub use self::{config::*, transformer::*}; pub(crate) mod config; pub(crate) mod transformer; -pub(crate) mod utils {} - #[cfg(test)] mod tests { // use super::*; From 4e92962ede9d94675b49d6729c10114fa95f7d37 Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 29 Dec 2023 12:06:34 -0600 Subject: [PATCH 097/118] update Signed-off-by: FL03 --- .github/workflows/rust.yml | 51 +++++++++++++++++++++++--- Cargo.toml | 13 ++----- concision/Cargo.toml | 47 +++++++++++++++++++++--- core/Cargo.toml | 8 +++- core/src/errors/mod.rs | 10 ++++- core/src/masks/mod.rs | 20 +++++++++- core/src/primitives.rs | 4 +- core/src/specs/init.rs | 4 ++ core/src/states/mod.rs | 5 ++- data/Cargo.toml | 35 ++++++++++++++++-- macros/Cargo.toml | 5 ++- ml/linear/Cargo.toml | 12 ++++-- ml/linear/src/lib.rs | 4 +- ml/ml/Cargo.toml | 73 ------------------------------------- ml/ml/benches/default.rs | 52 -------------------------- ml/ml/examples/nn.rs | 7 ---- ml/ml/src/lib.rs | 32 ---------------- ml/ml/tests/default.rs | 8 ---- ml/neural/Cargo.toml | 13 ++++--- ml/nlp/Cargo.toml | 7 +++- ml/optim/Cargo.toml | 14 ++++--- ml/s4/Cargo.toml | 52 +++++++++++++++++++++----- ml/s4/src/dplr/mod.rs | 37 ++----------------- ml/s4/src/ops/discretize.rs | 1 - ml/s4/src/primitives.rs | 2 +- ml/s4/src/utils.rs | 72 ++++++++++++++++++++---------------- ml/s4/tests/blas.rs | 26 +++++++++++++ ml/transformers/Cargo.toml | 10 +++-- 28 files changed, 324 insertions(+), 300 deletions(-) delete mode 100644 ml/ml/Cargo.toml delete mode 100644 ml/ml/benches/default.rs delete mode 100644 ml/ml/examples/nn.rs delete mode 100644 ml/ml/src/lib.rs delete mode 100644 ml/ml/tests/default.rs create mode 100644 ml/s4/tests/blas.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index caac03ae..8d797a73 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -24,7 +24,7 @@ permissions: write-all jobs: build: - name: Build and Test + name: Build strategy: matrix: platform: [ ubuntu-latest ] @@ -35,9 +35,9 @@ jobs: - uses: actions/checkout@v4 - name: setup (langspace) run: | - rustup update rustup default ${{ matrix.toolchain }} - - name: Build + rustup update + - name: Build (release) run: cargo build -r -v --workspace - name: Cache build uses: actions/cache@v3 @@ -47,8 +47,47 @@ jobs: ~/.cargo/registry ~/.cargo/git target/release - - name: Test - run: cargo test --all-features -r -v --workspace + + bench: + name: Bench + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: setup (langspace) + run: | + rustup default nightly + rustup update - name: Bench - if: matrix.toolchain == 'nightly' run: cargo bench --all -v + test: + name: Test + strategy: + matrix: + platform: [ ubuntu-latest ] + target: [ wasm32-unknown-unknown, wasm32-wasi ] + toolchain: [ stable, nightly ] + runs-on: ${{ matrix.platform }} + steps: + - uses: actions/checkout@v4 + - name: setup (langspace) + run: | + rustup default ${{ matrix.toolchain }} + rustup update + - name: Test + run: cargo test --features full -v --workspace + blas: + name: Test (blas) + strategy: + matrix: + platform: [ ubuntu-latest ] + target: [ wasm32-unknown-unknown, wasm32-wasi ] + toolchain: [ stable, nightly ] + runs-on: ${{ matrix.platform }} + steps: + - uses: actions/checkout@v4 + - name: setup (langspace) + run: | + rustup default ${{ matrix.toolchain }} + rustup update + - name: Test + run: cargo test --features blas -v --workspace diff --git a/Cargo.toml b/Cargo.toml index 4464ade5..78364f23 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,16 +15,13 @@ version = "0.1.12" # TODO - Update the cargo package version anyhow = "1" approx = "0.5" -itertools = { features = [], version = "0.12" } lazy_static = "1" -nalgebra = { features = [], version = "0.32" } -ndarray = { features = ["blas", "serde-1"], version = "0.15" } -ndarray-linalg = { features = ["intel-mkl-system"], version = "0.16" } +# ndarray = { features = ["serde-1"], version = "0.15" } +# ndarray-linalg = { features = [], version = "0.16" } ndarray-rand = { features = [], version = "0.14" } -ndarray-stats = { features = [], version = "0.5.1" } +ndarray-stats = { features = [], version = "0.5" } num = { features = ["serde"], version = "0.4" } -petgraph = { features = ["serde-1"], version = "0.6" } -rand = "0.8" + serde = { features = ["derive"], version = "1" } serde_json = "1" smart-default = "0.7" @@ -41,9 +38,7 @@ members = [ "data", "derive", "macros", - # "ml/*", "ml/linear", - "ml/ml", "ml/neural", "ml/nlp", "ml/optim", diff --git a/concision/Cargo.toml b/concision/Cargo.toml index dec77489..52a72a08 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -51,7 +51,6 @@ ml = [ "neural", "nlp", "optim", - "s4", "transformers", ] @@ -68,16 +67,53 @@ optim = [ ] s4 = [ - "concision-s4" + "blas", + "concision-s4", ] transformers = [ "concision-transformers" ] +blas = [ + "concision-core/blas", + "concision-data/blas", + "concision-linear/blas", + "concision-neural/blas", + "concision-nlp/blas", + "concision-optim/blas", + "concision-s4/blas", + "concision-transformers/blas", +] + +intel-mkl-system = [ + "concision-s4/intel-mkl-system", +] + +intel-mkl-static = [ + "concision-s4/intel-mkl-static", + +] + +netlib-system = [ + "concision-s4/netlib-system", +] + +netlib-static = [ + "concision-s4/netlib-static", +] + +openblas-system = [ + "concision-s4/openblas-system", +] + +openblas-static = [ + "concision-s4/openblas-static", +] + [lib] bench = true -crate-type = ["cdylib", "rlib"] +crate-type = ["rlib"] doctest = false test = true @@ -89,7 +125,6 @@ concision-data = { features = [], optional = true, path = "../data", version = " concision-derive = { features = [], optional = true, path = "../derive", version = "0.1.12" } concision-macros = { features = [], optional = true, path = "../macros", version = "0.1.12" } -# concision-ml = { features = [], optional = true, path = "../ml/ml", version = "0.1.12" } concision-linear = { features = [], optional = true, path = "../ml/linear", version = "0.1.12" } concision-neural = { features = [], optional = true, path = "../ml/neural", version = "0.1.12" } concision-nlp = { features = [], optional = true, path = "../ml/nlp", version = "0.1.12" } @@ -98,8 +133,8 @@ concision-s4 = { features = [], optional = true, path = "../ml/s4", version = "0 concision-transformers = { features = [], optional = true, path = "../ml/transformers", version = "0.1.12" } [dev-dependencies] -anyhow.workspace = true -ndarray.workspace = true +anyhow = "1" +ndarray = "0.15" [package.metadata.docs.rs] all-features = true diff --git a/core/Cargo.toml b/core/Cargo.toml index 729c84d5..4a523883 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -14,9 +14,13 @@ version.workspace = true [features] default = [] +blas = [ + "ndarray/blas", +] + [lib] bench = false -crate-type = ["cdylib", "rlib"] +crate-type = ["rlib"] doctest = false test = true @@ -25,7 +29,7 @@ test = true [dependencies] anyhow.workspace = true chrono = "0.4" -ndarray.workspace = true +ndarray = { features = ["serde-1"], version = "0.15" } ndarray-rand.workspace = true num.workspace = true serde.workspace = true diff --git a/core/src/errors/mod.rs b/core/src/errors/mod.rs index 267dace8..bb5c8ff8 100644 --- a/core/src/errors/mod.rs +++ b/core/src/errors/mod.rs @@ -6,4 +6,12 @@ pub use self::{error::*, utils::*}; pub(crate) mod error; -pub(crate) mod utils {} +pub(crate) mod utils { + + pub fn random_err() -> String { + String::new() + } +} + +#[cfg(test)] +mod tests {} diff --git a/core/src/masks/mod.rs b/core/src/masks/mod.rs index d196bd78..6748f63b 100644 --- a/core/src/masks/mod.rs +++ b/core/src/masks/mod.rs @@ -7,7 +7,25 @@ pub use self::{mask::*, utils::*}; pub(crate) mod mask; -pub(crate) mod utils {} +pub(crate) mod utils { + use super::Mask; + use ndarray::prelude::Array2; + use ndarray_rand::rand_distr::uniform::SampleUniform; + use ndarray_rand::rand_distr::Uniform; + use ndarray_rand::RandomExt; + use num::Float; + + pub fn rmask_uniform(features: usize) -> Mask + where + T: Float + SampleUniform, + { + let ds = (T::one() / T::from(features).unwrap()).sqrt(); + Mask::from(Array2::::random( + (features, features), + Uniform::new(-ds, ds), + )) + } +} #[cfg(test)] mod tests {} diff --git a/core/src/primitives.rs b/core/src/primitives.rs index 2a96ce14..920c63cc 100644 --- a/core/src/primitives.rs +++ b/core/src/primitives.rs @@ -8,7 +8,9 @@ pub use ndarray::ShapeError; pub use ndarray_rand::rand_distr::uniform::SampleUniform; /// Collection of constants used throughout the system -mod constants {} +mod constants { + pub const DEFAULT_MODEL_SIZE: usize = 2048; +} /// Collection of static references used throughout mod statics {} diff --git a/core/src/specs/init.rs b/core/src/specs/init.rs index 7aa06116..6fc1ee0b 100644 --- a/core/src/specs/init.rs +++ b/core/src/specs/init.rs @@ -6,3 +6,7 @@ pub trait Init { fn init(&mut self) -> Self; } + +pub trait Rand {} + +pub trait RandComplex {} diff --git a/core/src/states/mod.rs b/core/src/states/mod.rs index 561bedb4..49b512c1 100644 --- a/core/src/states/mod.rs +++ b/core/src/states/mod.rs @@ -2,10 +2,11 @@ Appellation: states Contrib: FL03 */ -pub use self::{state::*, utils::*}; +pub use self::state::*; pub(crate) mod state; pub mod weighted; -pub(crate) mod utils {} +#[cfg(test)] +mod tests {} diff --git a/data/Cargo.toml b/data/Cargo.toml index cb5434c4..3aaf713e 100644 --- a/data/Cargo.toml +++ b/data/Cargo.toml @@ -14,10 +14,39 @@ version.workspace = true [features] default = [] +blas = [ + "ndarray/blas", +] + +# intel-mkl-system = [ +# "linfa/intel-mkl-system", +# ] + +# intel-mkl-static = [ +# "linfa/intel-mkl-static", +# ] + +# netlib-system = [ +# "linfa/netlib-system", +# ] + +# netlib-static = [ +# "linfa/netlib-static", +# ] + +# openblas-system = [ +# "linfa/openblas-system", +# ] + +# openblas-static = [ +# "linfa/openblas-static", +# ] + + [lib] bench = true -crate-type = ["cdylib", "rlib"] +crate-type = ["rlib"] doctest = false test = true @@ -25,8 +54,8 @@ test = true [dependencies] anyhow.workspace = true -# linfa = { features = [], version = "0.7" } -ndarray.workspace = true +linfa = { features = ["serde"], version = "0.7" } +ndarray = { features = ["serde-1"], version = "0.15" } num.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/macros/Cargo.toml b/macros/Cargo.toml index 07ce181c..092e3d2d 100644 --- a/macros/Cargo.toml +++ b/macros/Cargo.toml @@ -14,14 +14,15 @@ version.workspace = true [features] default = [] + [lib] bench = false -crate-type = ["cdylib", "rlib"] +crate-type = ["rlib"] doctest = false test = false [dependencies] -ndarray.workspace = true +ndarray = { features = [], version = "0.15" } proc-macro2 = "1" quote = "1" syn = { features = ["full"], version = "2" } diff --git a/ml/linear/Cargo.toml b/ml/linear/Cargo.toml index e5c9124e..13ff773a 100644 --- a/ml/linear/Cargo.toml +++ b/ml/linear/Cargo.toml @@ -15,9 +15,15 @@ version.workspace = true [features] default = [] +blas = [ + "ndarray/blas", + "concision-core/blas", + "concision-neural/blas", +] + [lib] bench = true -crate-type = ["cdylib", "rlib"] +crate-type = ["rlib"] doctest = false test = true @@ -28,12 +34,10 @@ concision-core = { path = "../../core", version = "0.1.12" } concision-neural = { path = "../neural", version = "0.1.12" } anyhow.workspace = true -ndarray.workspace = true +ndarray = { features = ["serde-1"], version = "0.15" } ndarray-rand.workspace = true ndarray-stats.workspace = true num.workspace = true -petgraph.workspace = true -rand.workspace = true serde.workspace = true serde_json.workspace = true smart-default.workspace = true diff --git a/ml/linear/src/lib.rs b/ml/linear/src/lib.rs index 52683095..9a1908a6 100644 --- a/ml/linear/src/lib.rs +++ b/ml/linear/src/lib.rs @@ -15,4 +15,6 @@ pub mod params; pub(crate) use concision_core as core; pub(crate) use concision_neural as neural; -pub mod prelude {} +pub mod prelude { + pub use crate::model::*; +} diff --git a/ml/ml/Cargo.toml b/ml/ml/Cargo.toml deleted file mode 100644 index 29eb6a86..00000000 --- a/ml/ml/Cargo.toml +++ /dev/null @@ -1,73 +0,0 @@ -[package] -authors.workspace = true -categories.workspace = true -description.workspace = true -edition.workspace = true -homepage.workspace = true -keywords.workspace = true -license.workspace = true -name = "concision-ml" -readme.workspace = true -repository.workspace = true -version.workspace = true - -[features] -default = [ - "neural", - "optim", -] - -full = [ - "neural", - "nlp", - "optim", - "s4", - "transformers", -] - -neural = [ - "concision-neural" -] - -nlp = [ - "concision-nlp" -] - -optim = [ - "concision-optim", -] - -s4 = [ - "concision-s4" -] - -transformers = [ - "concision-transformers" -] - -[lib] -bench = true -crate-type = ["cdylib", "rlib"] -doctest = false -test = true - -[build-dependencies] - -[dependencies] -concision-neural = { features = [], optional = true, path = "../neural", version = "0.1.12" } -concision-nlp = { features = [], optional = true, path = "../nlp", version = "0.1.12" } -concision-optim = { features = [], optional = true, path = "../optim", version = "0.1.12" } -concision-s4 = { features = [], optional = true, path = "../s4", version = "0.1.12" } -concision-transformers = { features = [], optional = true, path = "../transformers", version = "0.1.12" } - -[dev-dependencies] -anyhow.workspace = true -ndarray.workspace = true - -[package.metadata.docs.rs] -all-features = true -rustc-args = ["--cfg", "docsrs"] - -[target.wasm32-unknown-unknown] - -[target.wasm32-wasi] \ No newline at end of file diff --git a/ml/ml/benches/default.rs b/ml/ml/benches/default.rs deleted file mode 100644 index 937f2387..00000000 --- a/ml/ml/benches/default.rs +++ /dev/null @@ -1,52 +0,0 @@ -// bench.rs -#![feature(test)] - -extern crate test; - -use std::mem::replace; -use test::Bencher; - -// bench: find the `BENCH_SIZE` first terms of the fibonacci sequence -static BENCH_SIZE: usize = 20; - -// recursive fibonacci -fn fibonacci(n: usize) -> u32 { - if n < 2 { - 1 - } else { - fibonacci(n - 1) + fibonacci(n - 2) - } -} - -// iterative fibonacci -struct Fibonacci { - curr: u32, - next: u32, -} - -impl Iterator for Fibonacci { - type Item = u32; - fn next(&mut self) -> Option { - let new_next = self.curr + self.next; - let new_curr = replace(&mut self.next, new_next); - - Some(replace(&mut self.curr, new_curr)) - } -} - -fn fibonacci_sequence() -> Fibonacci { - Fibonacci { curr: 1, next: 1 } -} - -// function to benchmark must be annotated with `#[bench]` -#[bench] -fn recursive_fibonacci(b: &mut Bencher) { - // exact code to benchmark must be passed as a closure to the iter - // method of Bencher - b.iter(|| (0..BENCH_SIZE).map(fibonacci).collect::>()) -} - -#[bench] -fn iterative_fibonacci(b: &mut Bencher) { - b.iter(|| fibonacci_sequence().take(BENCH_SIZE).collect::>()) -} diff --git a/ml/ml/examples/nn.rs b/ml/ml/examples/nn.rs deleted file mode 100644 index f9d1e06a..00000000 --- a/ml/ml/examples/nn.rs +++ /dev/null @@ -1,7 +0,0 @@ -extern crate concision_ml; - -fn main() -> anyhow::Result<()> { - println!("Welcome to concision!"); - - Ok(()) -} diff --git a/ml/ml/src/lib.rs b/ml/ml/src/lib.rs deleted file mode 100644 index d89f407e..00000000 --- a/ml/ml/src/lib.rs +++ /dev/null @@ -1,32 +0,0 @@ -/* - Appellation: concision-ml - Contrib: FL03 -*/ -//! # concision-ml -//! -//! Concision aims to be a complete machine learning library written in pure Rust. -//! - -#[cfg(feature = "neural")] -pub use concision_neural as neural; -#[cfg(feature = "nlp")] -pub use concision_nlp as nlp; -#[cfg(feature = "optim")] -pub use concision_optim as optim; -#[cfg(feature = "s4")] -pub use concision_s4 as s4; -#[cfg(feature = "transformers")] -pub use concision_transformers as transformers; - -pub mod prelude { - #[cfg(feature = "neural")] - pub use concision_neural::prelude::*; - #[cfg(feature = "nlp")] - pub use concision_nlp::prelude::*; - #[cfg(feature = "optim")] - pub use concision_optim::prelude::*; - #[cfg(feature = "s4")] - pub use concision_s4::prelude::*; - #[cfg(feature = "transformers")] - pub use concision_transformers::prelude::*; -} diff --git a/ml/ml/tests/default.rs b/ml/ml/tests/default.rs deleted file mode 100644 index 0cac1eb5..00000000 --- a/ml/ml/tests/default.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[cfg(test)] -#[test] -fn compiles() { - let f = |x: usize, y: usize| x + y; - - assert_eq!(f(10, 10), 20); - assert_ne!(f(1, 1), 3); -} diff --git a/ml/neural/Cargo.toml b/ml/neural/Cargo.toml index 62110e7e..d70fd817 100644 --- a/ml/neural/Cargo.toml +++ b/ml/neural/Cargo.toml @@ -15,9 +15,14 @@ version.workspace = true [features] default = [] +blas = [ + "ndarray/blas", + "concision-core/blas", +] + [lib] bench = true -crate-type = ["cdylib", "rlib"] +crate-type = ["rlib"] doctest = false test = true @@ -27,13 +32,11 @@ test = true concision-core = { path = "../../core", version = "0.1.12" } anyhow.workspace = true -ndarray.workspace = true -# ndarray-linalg.workspace = true +ndarray = { features = ["serde-1"], version = "0.15" } ndarray-rand.workspace = true ndarray-stats.workspace = true num.workspace = true -petgraph.workspace = true -rand.workspace = true +petgraph = { features = ["serde-1"], version = "0.6" } serde.workspace = true serde_json.workspace = true smart-default.workspace = true diff --git a/ml/nlp/Cargo.toml b/ml/nlp/Cargo.toml index bed7d377..6afdc42a 100644 --- a/ml/nlp/Cargo.toml +++ b/ml/nlp/Cargo.toml @@ -14,10 +14,13 @@ version.workspace = true [features] default = [] +blas = [ + "ndarray/blas", +] [lib] bench = true -crate-type = ["cdylib", "rlib"] +crate-type = ["rlib"] doctest = false test = true @@ -27,7 +30,7 @@ test = true anyhow.workspace = true finalfusion = "0.18" -ndarray.workspace = true +ndarray = { features = ["serde-1"], version = "0.15" } num.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/ml/optim/Cargo.toml b/ml/optim/Cargo.toml index d7839974..f847e421 100644 --- a/ml/optim/Cargo.toml +++ b/ml/optim/Cargo.toml @@ -14,10 +14,15 @@ version.workspace = true [features] default = [] +blas = [ + "ndarray/blas", + "concision-core/blas", + "concision-neural/blas", +] [lib] bench = true -crate-type = ["cdylib", "rlib"] +crate-type = ["rlib"] doctest = false test = true @@ -28,14 +33,13 @@ concision-core = { path = "../../core", version = "0.1.12" } concision-neural = { path = "../neural" } anyhow.workspace = true -itertools.workspace = true +itertools = { features = [], version = "0.12" } lazy_static.workspace = true -ndarray.workspace = true -# ndarray-linalg.workspace = true +ndarray = { features = ["serde-1"], version = "0.15" } ndarray-rand.workspace = true ndarray-stats.workspace = true num.workspace = true -rand.workspace = true +rand = "0.8" serde.workspace = true serde_json.workspace = true smart-default.workspace = true diff --git a/ml/s4/Cargo.toml b/ml/s4/Cargo.toml index 2c99f8b7..8c67013a 100644 --- a/ml/s4/Cargo.toml +++ b/ml/s4/Cargo.toml @@ -12,31 +12,65 @@ repository.workspace = true version.workspace = true [features] -default = [] +default = ["blas"] + +blas = [ + # "ndarray/blas", + # "concision-core/blas", + # "concision-neural/blas", +] + +intel-mkl-system = [ + "blas", + "ndarray-linalg/intel-mkl-system", +] + +intel-mkl-static = [ + "blas", + "ndarray-linalg/intel-mkl-static", +] + +netlib-system = [ + "blas", + "ndarray-linalg/netlib-system", +] + +netlib-static = [ + "blas", + "ndarray-linalg/netlib-static", +] + +openblas-system = [ + "blas", + "ndarray-linalg/openblas-system", +] + +openblas-static = [ + "blas", + "ndarray-linalg/openblas-static", +] + [lib] bench = true -crate-type = ["cdylib", "rlib"] +crate-type = ["rlib"] doctest = false test = true [build-dependencies] [dependencies] -concision-core = { path = "../../core", version = "0.1.12" } -# concision-data = { path = "../../data" } -concision-neural = { path = "../neural" } +concision-core = { features = ["blas"], path = "../../core", version = "0.1.12" } +concision-neural = { features = ["blas"], path = "../neural" } anyhow.workspace = true lazy_static.workspace = true -nalgebra.workspace = true -ndarray.workspace = true +ndarray = { features = ["blas", "serde-1"], version = "0.15" } ndarray-conv = "0.2" -ndarray-linalg.workspace = true +ndarray-linalg = { features = [], version = "0.16" } ndarray-rand.workspace = true ndarray-stats.workspace = true -nshare = "0.9" num.workspace = true rustfft = { features = [], version = "6" } serde.workspace = true diff --git a/ml/s4/src/dplr/mod.rs b/ml/s4/src/dplr/mod.rs index 1e361cde..c6c92f90 100644 --- a/ml/s4/src/dplr/mod.rs +++ b/ml/s4/src/dplr/mod.rs @@ -16,12 +16,12 @@ pub struct LowRank { } pub(crate) mod utils { - use crate::core::prelude::{rangespace, tril, AsComplex, Conjugate}; - use crate::prelude::eig_csym; + use crate::core::prelude::{rangespace, AsComplex, Conjugate}; use ndarray::prelude::{Array1, Array2, Axis}; use ndarray::ScalarOperand; - use ndarray_linalg::{eigh::Eigh, UPLO}; + use ndarray_linalg::eigh::Eigh; + use ndarray_linalg::{IntoTriangular, UPLO}; use num::complex::ComplexFloat; use num::Complex; @@ -32,7 +32,7 @@ pub(crate) mod utils { let base = rangespace((features, 1)); let p = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); let mut a = &p * &p.t(); - a = tril(&a) - &base.diag(); + a = &a.into_triangular(UPLO::Lower) - &base.diag(); -a } @@ -48,35 +48,6 @@ pub(crate) mod utils { (hippo, p, b) } - pub fn dplr_hippo( - features: usize, - ) -> ( - Array2>, - Array2>, - Array2>, - Array2>, - ) { - let (a, p, b) = make_nplr_hippo::>(features); - let p = p.insert_axis(Axis(1)); - let b = b.insert_axis(Axis(1)); - - // - let s = &a + p.dot(&p.t()); - // - let sd = s.diag(); - - let a = Array2::ones(s.dim()) * sd.mean().expect("Average of diagonal is NaN"); - - // TODO: replace with eigh - let (e, v) = eig_csym(&(&s * (-1.0).as_imag())); - let e = e.mapv(|x| x.as_complex()); - - let a = a + &e * 1.0.as_imag(); - let p = v.conj().t().dot(&p); - let b = v.conj().t().dot(&b); - (a, p, b, v) - } - pub fn make_dplr_hippo( features: usize, ) -> ( diff --git a/ml/s4/src/ops/discretize.rs b/ml/s4/src/ops/discretize.rs index f6583bea..a963f67e 100644 --- a/ml/s4/src/ops/discretize.rs +++ b/ml/s4/src/ops/discretize.rs @@ -6,7 +6,6 @@ use crate::core::prelude::Inverse; use ndarray::prelude::{Array2, NdFloat}; -use nshare::{ToNalgebra, ToNdarray2}; use num::{Float, ToPrimitive}; pub fn discretize( diff --git a/ml/s4/src/primitives.rs b/ml/s4/src/primitives.rs index 00daa8c3..7db8f95d 100644 --- a/ml/s4/src/primitives.rs +++ b/ml/s4/src/primitives.rs @@ -2,7 +2,7 @@ Appellation: primitives Contrib: FL03 */ -pub use self::{constants::*, statics::*, types::*}; +pub use self::constants::*; mod constants { /// The default model size for S4 models diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index 8d45c331..3d97fb87 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -7,7 +7,6 @@ use ndarray::IntoDimension; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, StandardNormal, Uniform}; use ndarray_rand::RandomExt; -use nshare::{ToNalgebra, ToNdarray1, ToNdarray2}; use num::complex::ComplexFloat; use num::{Complex, Float}; use rustfft::{FftNum, FftPlanner}; @@ -44,28 +43,28 @@ where omega.mapv(cdot) } -pub fn eig_sym(args: &Array2) -> (Array1, Array2) { - let sym = args.clone().into_nalgebra().symmetric_eigen(); - ( - sym.eigenvalues.into_ndarray1(), - sym.eigenvectors.into_ndarray2(), - ) -} +// pub fn eig_sym(args: &Array2) -> (Array1, Array2) { +// let sym = args.clone().into_nalgebra().symmetric_eigen(); +// ( +// sym.eigenvalues.into_ndarray1(), +// sym.eigenvectors.into_ndarray2(), +// ) +// } -pub fn eig_csym(args: &Array2>) -> (Array1, Array2>) { - let sym = args.clone().into_nalgebra().symmetric_eigen(); - let (eig, v) = (sym.eigenvalues, sym.eigenvectors); - (eig.into_ndarray1(), v.into_ndarray2()) -} +// pub fn eig_csym(args: &Array2>) -> (Array1, Array2>) { +// let sym = args.clone().into_nalgebra().symmetric_eigen(); +// let (eig, v) = (sym.eigenvalues, sym.eigenvectors); +// (eig.into_ndarray1(), v.into_ndarray2()) +// } -pub fn eigh(args: &Array2) -> (Array1, Array2) { - let na = args.clone().into_nalgebra(); - let sym = na.symmetric_eigen(); - let v = sym.eigenvectors; - let eig = sym.eigenvalues.into_ndarray1(); - let eigval = v.into_ndarray2(); - (eig, eigval) -} +// pub fn eigh(args: &Array2) -> (Array1, Array2) { +// let na = args.clone().into_nalgebra(); +// let sym = na.symmetric_eigen(); +// let v = sym.eigenvectors; +// let eig = sym.eigenvalues.into_ndarray1(); +// let eigval = v.into_ndarray2(); +// (eig, eigval) +// } pub fn powmat(a: &Array2, n: usize) -> Array2 where @@ -187,7 +186,7 @@ pub fn scan_ssm_step<'a, T>( where T: NdFloat, { - |xs, us| { + move |xs, us| { let x1 = a.dot(xs) + b.dot(&us); let y1 = c.dot(&x1); Some(y1) @@ -227,18 +226,29 @@ where #[cfg(test)] mod tests { use super::*; - use crate::core::round_to; - use approx::assert_relative_eq; - use ndarray::prelude::array; + use ndarray::prelude::{array, Array1, Array2}; + use ndarray_linalg::eig::Eig; + use num::complex::{Complex, ComplexFloat}; + + fn round_to(a: T, precision: usize) -> T { + let factor = T::from(10).unwrap().powi(precision as i32); + (a * factor).round() / factor + } #[test] - fn test_eig_sym() { + fn test_eig() { let a = array![[1.0, 2.0], [2.0, 1.0]]; - let (eig, eigval) = eig_sym(&a); - let exp_eig = array![3.0, -1.0]; - let exp_eigval = array![[0.70710678, -0.70710678], [0.70710678, 0.70710678]]; + let (eig, eigval): (Array1>, Array2>) = a.eig().unwrap(); + + let eig = eig.mapv(|i| Complex::new(i.re().round(), i.im().round())); + let eigval = eigval.mapv(|i| Complex::new(round_to(i.re(), 8), round_to(i.im(), 8))); + + let exp_eig: Array1> = array![3.0, -1.0].mapv(|i| Complex::new(i, 0.0)); + let exp_eigval: Array2> = + array![[0.70710678, -0.70710678], [0.70710678, 0.70710678]] + .mapv(|i| Complex::new(i, 0.0)); - assert_relative_eq!(eig, exp_eig); - assert_relative_eq!(eigval.mapv(|i| round_to(i, 8)), exp_eigval); + assert_eq!(eig, exp_eig); + assert_eq!(eigval, exp_eigval); } } diff --git a/ml/s4/tests/blas.rs b/ml/s4/tests/blas.rs new file mode 100644 index 00000000..132a8874 --- /dev/null +++ b/ml/s4/tests/blas.rs @@ -0,0 +1,26 @@ +#[cfg(test)] +use ndarray::prelude::{array, Array1, Array2}; +use ndarray_linalg::eig::Eig; +use num::complex::{Complex, ComplexFloat}; +use num::Float; + +fn round_to(a: T, precision: usize) -> T { + let factor = T::from(10).unwrap().powi(precision as i32); + (a * factor).round() / factor +} + +#[test] +fn test_eig() { + let a = array![[1.0, 2.0], [2.0, 1.0]]; + let (eig, eigval): (Array1>, Array2>) = a.eig().unwrap(); + + let eig = eig.mapv(|i| Complex::new(i.re().round(), i.im().round())); + let eigval = eigval.mapv(|i| Complex::new(round_to(i.re(), 8), round_to(i.im(), 8))); + + let exp_eig: Array1> = array![3.0, -1.0].mapv(|i| Complex::new(i, 0.0)); + let exp_eigval: Array2> = + array![[0.70710678, -0.70710678], [0.70710678, 0.70710678]].mapv(|i| Complex::new(i, 0.0)); + + assert_eq!(eig, exp_eig); + assert_eq!(eigval, exp_eigval); +} diff --git a/ml/transformers/Cargo.toml b/ml/transformers/Cargo.toml index 69bb9d6f..182aa22a 100644 --- a/ml/transformers/Cargo.toml +++ b/ml/transformers/Cargo.toml @@ -14,10 +14,15 @@ version.workspace = true [features] default = [] +blas = [ + "ndarray/blas", + "concision-core/blas", + "concision-neural/blas", +] [lib] bench = true -crate-type = ["cdylib", "rlib"] +crate-type = ["rlib"] doctest = false test = true @@ -25,12 +30,11 @@ test = true [dependencies] concision-core = { path = "../../core", version = "0.1.12" } -# concision-data = { path = "../../data" } concision-neural = { path = "../neural", version = "0.1.12" } anyhow.workspace = true lazy_static.workspace = true -ndarray.workspace = true +ndarray = { features = ["serde-1"], version = "0.15" } ndarray-rand.workspace = true ndarray-stats.workspace = true num.workspace = true From 142fb598339f111d542cb3dbd785bcb373fdff3d Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 29 Dec 2023 15:03:04 -0600 Subject: [PATCH 098/118] update Signed-off-by: FL03 --- core/src/specs/arrays.rs | 2 + core/src/utils.rs | 17 +++- ml/s4/src/model/model.rs | 5 +- ml/s4/src/model/params.rs | 38 ++++---- ml/s4/src/ops/mod.rs | 3 +- ml/s4/src/ops/scan.rs | 31 +++++++ ml/s4/src/params/mod.rs | 16 ++-- ml/s4/src/params/store.rs | 32 ++++--- ml/s4/src/utils.rs | 176 ++++++++++++++------------------------ ml/s4/tests/utils.rs | 22 +++++ 10 files changed, 191 insertions(+), 151 deletions(-) create mode 100644 ml/s4/src/ops/scan.rs create mode 100644 ml/s4/tests/utils.rs diff --git a/core/src/specs/arrays.rs b/core/src/specs/arrays.rs index 7527ad62..8058d3b2 100644 --- a/core/src/specs/arrays.rs +++ b/core/src/specs/arrays.rs @@ -97,3 +97,5 @@ where crate::compute_inverse(self) } } + +// pub trait Stack diff --git a/core/src/utils.rs b/core/src/utils.rs index cd87120b..a44e8911 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ -use ndarray::prelude::{s, Array, Array1, Array2, Axis, Dimension, NdFloat}; +use ndarray::prelude::*; use ndarray::{concatenate, IntoDimension, RemoveAxis, ShapeError}; use num::cast::AsPrimitive; use num::{Float, Num, NumCast, Zero}; @@ -122,6 +122,21 @@ pub fn round_to(val: T, decimals: usize) -> T { (val * factor).round() / factor } +pub fn stack_iter(iter: impl IntoIterator>) -> Array2 +where + T: Clone + Num, +{ + let mut iter = iter.into_iter(); + let first = iter.next().unwrap(); + let shape = [iter.size_hint().0 + 1, first.len()]; + let mut res = Array2::::zeros(shape); + res.slice_mut(s![0, ..]).assign(&first); + for (i, s) in iter.enumerate() { + res.slice_mut(s![i + 1, ..]).assign(&s); + } + res +} + /// Returns the upper triangular portion of a matrix. pub fn triu(a: &Array2) -> Array2 where diff --git a/ml/s4/src/model/model.rs b/ml/s4/src/model/model.rs index 04fcc3e8..2c043593 100644 --- a/ml/s4/src/model/model.rs +++ b/ml/s4/src/model/model.rs @@ -7,6 +7,7 @@ use crate::neural::prelude::Forward; use crate::prelude::SSMStore; use ndarray::prelude::{Array1, Array2, NdFloat}; use ndarray_conv::{Conv2DFftExt, PaddingMode, PaddingSize}; +use ndarray_linalg::Scalar; use num::Float; // use num::complex::{Complex, ComplexFloat}; use rustfft::FftNum; @@ -67,7 +68,7 @@ where impl Forward> for S4 where - T: FftNum + NdFloat, + T: FftNum + NdFloat + Scalar, { type Output = Array2; @@ -78,7 +79,7 @@ where args.conv_2d_fft(&self.kernal.clone().unwrap(), size, mode) .expect("convolution failed") } else { - self.store.scan(args, &self.cache) + self.store.scan(args, &self.cache).expect("scan failed") }; res + args * &self.store[D] } diff --git a/ml/s4/src/model/params.rs b/ml/s4/src/model/params.rs index b12494f8..0aa8108d 100644 --- a/ml/s4/src/model/params.rs +++ b/ml/s4/src/model/params.rs @@ -2,40 +2,41 @@ Appellation: params Contrib: FL03 */ -use ndarray::prelude::{Array1, Array2, NdFloat}; -use num::{Complex, Float}; +use ndarray::prelude::{Array1, Array2, ArrayView1, NdFloat}; +use num::complex::{Complex, ComplexFloat}; +use num::Float; pub struct S4Store where - T: Float, + T: ComplexFloat, { - pub a: Array2>, // Lambda + pub a: Array2, // Lambda pub b: Array2, - pub c: Array2>, + pub c: Array2, pub d: Array2, } impl S4Store where - T: Float, + T: ComplexFloat, { - pub fn new(a: Array2>, b: Array2, c: Array2>, d: Array2) -> Self { + pub fn new(a: Array2, b: Array2, c: Array2, d: Array2) -> Self { Self { a, b, c, d } } pub fn zeros(features: usize) -> Self { - let a = Array2::>::zeros((features, features)); + let a = Array2::::zeros((features, features)); let b = Array2::::zeros((features, features)); - let c = Array2::>::zeros((features, features)); + let c = Array2::::zeros((features, features)); let d = Array2::::zeros((features, features)); Self { a, b, c, d } } - pub fn a(&self) -> &Array2> { + pub fn a(&self) -> &Array2 { &self.a } - pub fn a_mut(&mut self) -> &mut Array2> { + pub fn a_mut(&mut self) -> &mut Array2 { &mut self.a } @@ -47,11 +48,11 @@ where &mut self.b } - pub fn c(&self) -> &Array2> { + pub fn c(&self) -> &Array2 { &self.c } - pub fn c_mut(&mut self) -> &mut Array2> { + pub fn c_mut(&mut self) -> &mut Array2 { &mut self.c } @@ -66,10 +67,15 @@ where impl S4Store where - T: NdFloat, + T: ComplexFloat + 'static, { - pub fn scan(&self, u: &Array2, x0: &Array1>) -> Array2> { - crate::scan_complex(&self.a, &self.b, &self.c, u, x0) + pub fn scan(&self, u: &Array2, x0: &Array1) -> Array2 { + let step = |xs: &mut Array1, us: ArrayView1| { + let x1 = self.a.dot(xs) + self.b.t().dot(&us); + let y1 = self.c.dot(&x1.t()); + Some(y1) + }; + crate::stack_arrays(u.outer_iter().scan(x0.clone(), step)) } } diff --git a/ml/s4/src/ops/mod.rs b/ml/s4/src/ops/mod.rs index 1bc197f8..3e9ca368 100644 --- a/ml/s4/src/ops/mod.rs +++ b/ml/s4/src/ops/mod.rs @@ -2,11 +2,12 @@ Appellation: ops Contrib: FL03 */ -pub use self::{convolve::*, discretize::*, gen::*}; +pub use self::{convolve::*, discretize::*, gen::*, scan::*}; pub(crate) mod convolve; pub(crate) mod discretize; pub(crate) mod gen; +pub(crate) mod scan; #[cfg(test)] mod tests {} diff --git a/ml/s4/src/ops/scan.rs b/ml/s4/src/ops/scan.rs new file mode 100644 index 00000000..361bfa09 --- /dev/null +++ b/ml/s4/src/ops/scan.rs @@ -0,0 +1,31 @@ +/* + Appellation: scan + Contrib: FL03 +*/ +use crate::params::SSMStore; +use crate::prelude::stack_arrays; +use num::Float; + +pub struct Scan<'a, T = f64> +where + T: Float, +{ + model: &'a mut SSMStore, +} + +impl<'a, T> Scan<'a, T> +where + T: Float, +{ + pub fn new(model: &'a mut SSMStore) -> Self { + Self { model } + } + + pub fn model(&self) -> &SSMStore { + self.model + } + + pub fn model_mut(&mut self) -> &mut SSMStore { + self.model + } +} diff --git a/ml/s4/src/params/mod.rs b/ml/s4/src/params/mod.rs index bcbe43ba..249db5f1 100644 --- a/ml/s4/src/params/mod.rs +++ b/ml/s4/src/params/mod.rs @@ -7,15 +7,15 @@ pub use self::{kinds::*, store::*}; pub(crate) mod kinds; pub(crate) mod store; -use ndarray::prelude::Array2; -use num::Float; +use ndarray::prelude::{Array, Array1, Array2, Ix2}; +use num::Num; use std::collections::HashMap; -pub type SSMMap = HashMap>; +pub type SSMMap = HashMap>; pub trait SSMParamGroup where - T: Float, + T: Num, { fn a(&self) -> &Array2; fn b(&self) -> &Array2; @@ -25,7 +25,7 @@ where impl SSMParamGroup for SSMStore where - T: Float, + T: Num, { fn a(&self) -> &Array2 { &self.a @@ -46,7 +46,7 @@ where impl SSMParamGroup for SSMMap where - T: Float, + T: Num, { fn a(&self) -> &Array2 { self.get(&SSMParams::A).unwrap() @@ -67,7 +67,7 @@ where impl SSMParamGroup for &[Array2; 4] where - T: Float, + T: Num, { fn a(&self) -> &Array2 { &self[0] @@ -88,7 +88,7 @@ where impl SSMParamGroup for (Array2, Array2, Array2, Array2) where - T: Float, + T: Num, { fn a(&self) -> &Array2 { &self.0 diff --git a/ml/s4/src/params/store.rs b/ml/s4/src/params/store.rs index 82739e35..403cb940 100644 --- a/ml/s4/src/params/store.rs +++ b/ml/s4/src/params/store.rs @@ -4,21 +4,21 @@ */ use super::SSMParams; use crate::core::prelude::GenerateRandom; -use crate::prelude::scanner; -use ndarray::prelude::{Array1, Array2, NdFloat}; +use crate::prelude::stack_arrays; +use ndarray::prelude::{Array1, Array2, ArrayView1, NdFloat}; +use ndarray_linalg::error::LinalgError; +use ndarray_linalg::{vstack, Scalar}; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use ndarray_rand::RandomExt; -use num::Float; +use num::complex::ComplexFloat; +use num::{Float, Num}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::ops; #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub struct SSMStore -where - T: Float, -{ +pub struct SSMStore { pub(crate) a: Array2, pub(crate) b: Array2, pub(crate) c: Array2, @@ -27,7 +27,7 @@ where impl SSMStore where - T: Float, + T: Clone + Num, { pub fn new(a: Array2, b: Array2, c: Array2, d: Array2) -> Self { Self { a, b, c, d } @@ -95,10 +95,20 @@ where impl SSMStore where - T: NdFloat, + T: Scalar, { - pub fn scan(&self, u: &Array2, x0: &Array1) -> Array2 { - scanner(&self.a, &self.b, &self.c, u, x0) + pub fn scan(&self, u: &Array2, x0: &Array1) -> Result, LinalgError> { + let step = |xs: &mut Array1, us: ArrayView1| { + let x1 = self.a().dot(xs) + self.b().t().dot(&us); + let y1 = self.c().dot(&x1.t()); + Some(y1) + }; + vstack( + u.outer_iter() + .scan(x0.clone(), step) + .collect::>() + .as_slice(), + ) } } diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index 3d97fb87..9af5551f 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -2,13 +2,16 @@ Appellation: utils Contrib: FL03 */ +use crate::core::prelude::{AsComplex, Conjugate}; use ndarray::prelude::*; -use ndarray::IntoDimension; +use ndarray::{IntoDimension, ScalarOperand}; +use ndarray_linalg::Scalar; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, StandardNormal, Uniform}; use ndarray_rand::RandomExt; -use num::complex::ComplexFloat; -use num::{Complex, Float}; +use num::complex::{Complex, ComplexFloat}; +use num::traits::FloatConst; +use num::{Float, Num, Signed}; use rustfft::{FftNum, FftPlanner}; pub fn stdnorm(shape: impl IntoDimension) -> Array @@ -35,9 +38,24 @@ where res } -pub fn cauchy(v: &Array1, omega: &Array1, lambda: &Array1) -> Array1 +pub fn cauchy(v: &Array, omega: &Array, lambda: &Array) -> Array where - T: NdFloat, + D: Dimension, + T: Clone + Num + ScalarOperand + Signed, +{ + let cdot = |b: T| (v / (lambda * T::one().neg() + b)).sum(); + omega.mapv(cdot) +} + +pub fn cauchy_complex( + v: &Array, + omega: &Array, + lambda: &Array, +) -> Array +where + D: Dimension, + S: Dimension, + T: ComplexFloat + ScalarOperand, { let cdot = |b: T| (v / (lambda * T::one().neg() + b)).sum(); omega.mapv(cdot) @@ -117,8 +135,8 @@ where T: NdFloat, { let step = |xs: &mut Array1, us: ArrayView1| { - let x1 = a.dot(xs) + b.dot(&us); - let y1 = c.dot(&x1); + let x1 = a.dot(xs) + b.t().dot(&us); + let y1 = c.dot(&x1.t()); Some(y1) }; let scan = u.outer_iter().scan(x0.clone(), step).collect::>(); @@ -130,125 +148,59 @@ where res } -pub fn scan_ssm( - a: &Array2, - b: &Array2, - c: &Array2, - u: &Array2, - x0: &Array1, -) -> Array2 +pub fn stack_arrays(iter: impl IntoIterator>) -> Array2 where - T: ComplexFloat + 'static, + T: Clone + Num, { - let step = |xs: &mut Array1, us: ArrayView1| { - let x1 = a.dot(xs) + b.dot(&us); - let y1 = c.dot(&x1); - Some(y1) - }; - let scan = u.outer_iter().scan(x0.clone(), step).collect::>(); - let shape = [scan.len(), scan[0].len()]; - let mut res = Array2::::zeros(shape.into_dimension()); - for (i, s) in scan.iter().enumerate() { + let iter = Vec::from_iter(iter); + let mut res = Array2::::zeros((iter.len(), iter.first().unwrap().len())); + for (i, s) in iter.iter().enumerate() { res.slice_mut(s![i, ..]).assign(s); } res } -pub fn scan_complex( - a: &Array2>, +pub fn kernel_dplr( + lambda: &Array2, + p: &Array2, + q: &Array2, b: &Array2, - c: &Array2>, - u: &Array2, - x0: &Array1>, -) -> Array2> + c: &Array2, + step: T, + l: usize, +) -> Array1::Real>> where - T: NdFloat, + T: AsComplex + ComplexFloat + Conjugate + FloatConst + Scalar + ScalarOperand, + ::Real: NdFloat + Num + Signed + num::FromPrimitive + num::Zero, + ::Complex: ComplexFloat, { - let step = |xs: &mut Array1>, us: ArrayView1| { - let x1 = a.dot(xs) + b.dot(&us); - let y1 = c.dot(&x1); - Some(y1) + let omega_l = { + let f = | i: usize | { + T::from(i).unwrap() + * T::from(Complex::new(T::one(), -T::PI() / T::from(l).unwrap())).unwrap() + }; + Array::from_iter((0..l).map(f)) }; - let scan = u.outer_iter().scan(x0.clone(), step).collect::>(); - let shape = [scan.len(), scan[0].len()]; - let mut res = Array2::zeros(shape.into_dimension()); - for (i, s) in scan.iter().enumerate() { - res.slice_mut(s![i, ..]).assign(s); - } - res -} - -pub fn scan_ssm_step<'a, T>( - a: &'a Array2, - b: &'a Array2, - c: &'a Array2, -) -> impl FnMut(&'a mut Array1, ArrayView1<'a, T>) -> Option> -where - T: NdFloat, -{ - move |xs, us| { - let x1 = a.dot(xs) + b.dot(&us); - let y1 = c.dot(&x1); - Some(y1) - } -} + let aterm = (c.conj(), q.conj()); + let bterm = (b, p); -// fn kernel_dplr(lambda: T, p: &Array2, q: &Array2, b: &Array2, c: &Array2, step: T, l: usize) -> Array1> -// where -// T: Conjugate + FloatConst + FftNum + NdFloat, -// { -// let omega_l: Array1> = (0..l) -// .map(|i| Complex::from_polar(T::one(), -T::PI() * T::from(i).unwrap() / T::from(l).unwrap())) -// .collect(); - -// let aterm = (c.conj(), q.conj()); -// let bterm = (b, p); - -// let g = (T::from(2.0).unwrap() / step) * ((T::one().as_complex() - &omega_l) / (&omega_l + T::one())); -// let c = T::from(2.0).unwrap() / (T::from(1.0).unwrap() + &omega_l); - -// let k00 = cauchy(&aterm.0 * bterm.0, &g, lambda); -// let k01 = cauchy(&aterm.0 * bterm.1, &g, lambda); -// let k10 = cauchy(&aterm.1 * bterm.0, &g, lambda); -// let k11 = cauchy(&aterm.1 * bterm.1, &g, lambda); -// let at_roots = &c * (&k00 - k01 * (T::one() / (&k11 + T::one())) * &k10); -// let mut fft_planner = FftPlanner::new(); -// let fft = fft_planner.plan_fft(l, rustfft::FftDirection::Forward); -// let mut at_roots_complex = Array1::zeros(l); -// for (i, val) in at_roots.iter().enumerate() { -// at_roots_complex[i] = Complex::new(*val, T::zero()); -// } -// let mut out = Array1::zeros(l); -// fft.process(&mut out); -// out -// } - -#[cfg(test)] -mod tests { - use super::*; - use ndarray::prelude::{array, Array1, Array2}; - use ndarray_linalg::eig::Eig; - use num::complex::{Complex, ComplexFloat}; + let two = T::from(2).unwrap(); - fn round_to(a: T, precision: usize) -> T { - let factor = T::from(10).unwrap().powi(precision as i32); - (a * factor).round() / factor - } + let g = ((&omega_l * T::one().neg() + T::one()) / (&omega_l + T::one())) * (two / step); + let c = (&omega_l + T::one()).mapv(|i| two / i); - #[test] - fn test_eig() { - let a = array![[1.0, 2.0], [2.0, 1.0]]; - let (eig, eigval): (Array1>, Array2>) = a.eig().unwrap(); + let k00 = cauchy_complex(&(&aterm.0 * bterm.0), &g, lambda); + let k01 = cauchy_complex(&(&aterm.0 * bterm.1), &g, lambda); + let k10 = cauchy_complex(&(&aterm.1 * bterm.0), &g, lambda); + let k11 = cauchy_complex(&(&aterm.1 * bterm.1), &g, lambda); - let eig = eig.mapv(|i| Complex::new(i.re().round(), i.im().round())); - let eigval = eigval.mapv(|i| Complex::new(round_to(i.re(), 8), round_to(i.im(), 8))); + let at_roots = &c * (&k00 - k01 * (&k11 + T::one()).mapv(|i| T::one() / i) * &k10); - let exp_eig: Array1> = array![3.0, -1.0].mapv(|i| Complex::new(i, 0.0)); - let exp_eigval: Array2> = - array![[0.70710678, -0.70710678], [0.70710678, 0.70710678]] - .mapv(|i| Complex::new(i, 0.0)); - - assert_eq!(eig, exp_eig); - assert_eq!(eigval, exp_eigval); - } + let mut fft_planner = FftPlanner::new(); + let fft = fft_planner.plan_fft_inverse(l); + let mut buffer = at_roots + .mapv(|i| Complex::new(i.re(), i.im())) + .into_raw_vec(); + fft.process(buffer.as_mut_slice()); + Array::from_vec(buffer) } diff --git a/ml/s4/tests/utils.rs b/ml/s4/tests/utils.rs new file mode 100644 index 00000000..8876b845 --- /dev/null +++ b/ml/s4/tests/utils.rs @@ -0,0 +1,22 @@ +#[cfg(test)] +extern crate concision_s4; +use concision_s4 as s4; + +use ndarray::prelude::*; +use s4::prelude::{scanner, SSMStore}; + +#[test] +fn test_scan() { + let features = 2; + + let u = Array2::ones((10, features)); + let x0 = Array1::ones(features); + + let ssm = SSMStore::::from_features(features); + let (a, b, c, _d) = ssm.clone().into(); + let scan1 = scanner(&a, &b, &c, &u, &x0); + + let scan2 = ssm.scan(&u, &x0).unwrap(); + + assert_eq!(scan1, scan2); +} From 61c7a39f53e569c22a08bb5cfa0f56b72fcc701f Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 30 Dec 2023 11:30:35 -0600 Subject: [PATCH 099/118] Add dependencies and update code Signed-off-by: FL03 --- core/Cargo.toml | 1 + core/src/errors/error.rs | 41 +- core/src/errors/kinds.rs | 77 ++++ core/src/errors/mod.rs | 10 +- core/src/id/identity.rs | 10 + core/src/id/mod.rs | 25 ++ core/src/lib.rs | 2 + core/src/params/group.rs | 2 + core/src/params/param.rs | 2 + core/src/params/store.rs | 25 ++ core/src/specs/arrays.rs | 111 +++++- core/src/specs/mod.rs | 14 + core/src/utils.rs | 39 +- data/Cargo.toml | 1 + data/src/lib.rs | 3 +- data/src/misc/dtype.rs | 11 + data/src/misc/mod.rs | 8 + data/src/specs.rs | 30 +- ml/linear/src/params/store.rs | 2 + ml/neural/src/layers/exp/layer.rs | 9 +- ml/neural/src/layers/layer.rs | 2 + ml/neural/src/layers/params.rs | 2 + ml/neural/src/layers/stack.rs | 5 +- ml/neural/src/models/exp/mod.rs | 4 +- ml/neural/src/models/exp/modules.rs | 4 +- ml/neural/src/models/exp/store.rs | 2 + ml/neural/src/models/mod.rs | 4 +- ml/neural/src/models/model.rs | 2 + ml/neural/src/models/params.rs | 2 + ml/neural/src/neurons/node.rs | 2 + ml/neural/src/neurons/perceptron.rs | 2 + ml/neural/src/params/bias.rs | 359 ------------------ ml/neural/src/params/group.rs | 6 +- ml/neural/src/params/mod.rs | 6 +- ml/neural/src/params/weight.rs | 62 --- ml/optim/src/optimizer.rs | 11 - ml/optim/src/params/mod.rs | 4 - ml/s4/src/dplr/hippo.rs | 3 +- ml/s4/src/dplr/mod.rs | 29 +- ml/s4/src/model/params.rs | 18 +- ml/s4/src/ops/discretize.rs | 63 ++- ml/s4/src/ops/scan.rs | 2 +- ml/s4/src/params/kinds.rs | 31 ++ ml/s4/src/params/store.rs | 4 +- ml/s4/src/ssm/model.rs | 28 +- ml/s4/src/utils.rs | 18 +- ml/transformers/src/attention/head.rs | 2 + .../src/attention/multi/attention.rs | 2 + ml/transformers/src/attention/weights.rs | 2 + 49 files changed, 516 insertions(+), 588 deletions(-) create mode 100644 core/src/errors/kinds.rs create mode 100644 core/src/id/identity.rs create mode 100644 core/src/id/mod.rs create mode 100644 data/src/misc/dtype.rs create mode 100644 data/src/misc/mod.rs delete mode 100644 ml/neural/src/params/bias.rs delete mode 100644 ml/neural/src/params/weight.rs diff --git a/core/Cargo.toml b/core/Cargo.toml index 4a523883..f792da56 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -32,6 +32,7 @@ chrono = "0.4" ndarray = { features = ["serde-1"], version = "0.15" } ndarray-rand.workspace = true num.workspace = true +rand = { features = [], version = "0.8" } serde.workspace = true serde_json.workspace = true smart-default.workspace = true diff --git a/core/src/errors/error.rs b/core/src/errors/error.rs index 9012cb53..edaf1f18 100644 --- a/core/src/errors/error.rs +++ b/core/src/errors/error.rs @@ -2,47 +2,8 @@ Appellation: error Contrib: FL03 */ +use super::Errors; use serde::{Deserialize, Serialize}; -use smart_default::SmartDefault; -use strum::{Display, EnumIs, EnumIter, EnumVariantNames}; - -#[derive( - Clone, - Debug, - Deserialize, - Display, - EnumIs, - EnumIter, - EnumVariantNames, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize, - SmartDefault, -)] -#[non_exhaustive] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum Errors { - Async, - Codec, - Connection, - Custom(String), - Data, - Dimension, - #[default] - Error, - Execution, - IO, - Null, - Parse, - Process, - Runtime, - Syntax, - Unknown, -} #[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] #[serde(rename_all = "lowercase")] diff --git a/core/src/errors/kinds.rs b/core/src/errors/kinds.rs new file mode 100644 index 00000000..56809bb2 --- /dev/null +++ b/core/src/errors/kinds.rs @@ -0,0 +1,77 @@ +/* + Appellation: error + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use smart_default::SmartDefault; +use strum::{Display, EnumCount, EnumIs, EnumIter, EnumVariantNames}; + +#[derive( + Clone, + Debug, + Deserialize, + Display, + EnumCount, + EnumIs, + EnumIter, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, + SmartDefault, +)] +#[non_exhaustive] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum Errors { + Async, + Codec, + Connection, + Custom(String), + Data, + Dimension, + #[default] + Error, + Execution, + IO, + Null, + Parse, + Process, + Runtime, + Syntax, + Unknown, +} + +#[derive( + Clone, + Debug, + Deserialize, + Display, + EnumCount, + EnumIs, + EnumIter, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, + SmartDefault, +)] +#[non_exhaustive] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum ProcessError { + #[default] + Error, + Execution, + IO, + Parse, + Runtime, + Syntax, + Unknown, +} diff --git a/core/src/errors/mod.rs b/core/src/errors/mod.rs index bb5c8ff8..defc3d8b 100644 --- a/core/src/errors/mod.rs +++ b/core/src/errors/mod.rs @@ -2,15 +2,15 @@ Appellation: errors Contrib: FL03 */ -pub use self::{error::*, utils::*}; +pub use self::{error::*, kinds::*}; pub(crate) mod error; +pub(crate) mod kinds; -pub(crate) mod utils { +pub trait KindOf { + type Kind; - pub fn random_err() -> String { - String::new() - } + fn kind(&self) -> Self::Kind; } #[cfg(test)] diff --git a/core/src/id/identity.rs b/core/src/id/identity.rs new file mode 100644 index 00000000..b2b65f7d --- /dev/null +++ b/core/src/id/identity.rs @@ -0,0 +1,10 @@ +/* + Appellation: identity + Contrib: FL03 +*/ + +pub struct Id { + id: String, + name: String, + timestamp: i32, +} diff --git a/core/src/id/mod.rs b/core/src/id/mod.rs new file mode 100644 index 00000000..dbe3ce71 --- /dev/null +++ b/core/src/id/mod.rs @@ -0,0 +1,25 @@ +/* + Appellation: id + Contrib: FL03 +*/ +//! # id +pub use self::{identity::*, utils::*}; + +pub(crate) mod identity; + +pub(crate) mod utils { + + pub fn rid(length: usize) -> String { + use rand::distributions::Alphanumeric; + use rand::{thread_rng, Rng}; + + thread_rng() + .sample_iter(&Alphanumeric) + .take(length) + .map(char::from) + .collect() + } +} + +#[cfg(test)] +mod tests {} diff --git a/core/src/lib.rs b/core/src/lib.rs index 93f3bc8e..2674282f 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -10,6 +10,7 @@ pub(crate) mod primitives; pub(crate) mod utils; pub mod errors; +pub mod id; pub mod masks; pub mod params; pub mod specs; @@ -29,6 +30,7 @@ pub mod prelude { pub use crate::utils::*; pub use crate::errors::*; + pub use crate::id::*; pub use crate::masks::*; pub use crate::specs::*; pub use crate::states::*; diff --git a/core/src/params/group.rs b/core/src/params/group.rs index b5e4138e..47239b47 100644 --- a/core/src/params/group.rs +++ b/core/src/params/group.rs @@ -8,6 +8,7 @@ use ndarray::linalg::Dot; use ndarray::prelude::{Array, Axis, Dimension, Ix2, NdFloat}; use ndarray::{IntoDimension, RemoveAxis}; use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use num::Float; use serde::{Deserialize, Serialize}; @@ -77,6 +78,7 @@ impl ParamGroup where D: Dimension + RemoveAxis, T: Float + SampleUniform, + StandardNormal: Distribution, { pub fn init(mut self, biased: bool) -> Self { if biased { diff --git a/core/src/params/param.rs b/core/src/params/param.rs index 673de647..36e91b74 100644 --- a/core/src/params/param.rs +++ b/core/src/params/param.rs @@ -8,6 +8,7 @@ use ndarray::linalg::Dot; use ndarray::prelude::{Array, Dimension, Ix2}; use ndarray::IntoDimension; use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use num::Float; use serde::{Deserialize, Serialize}; @@ -98,6 +99,7 @@ impl Parameter where D: Dimension, T: Float + SampleUniform, + StandardNormal: Distribution, { pub fn init_uniform(mut self, dk: T) -> Self { let dim = self.params.dim(); diff --git a/core/src/params/store.rs b/core/src/params/store.rs index d8bf4d3a..b337143b 100644 --- a/core/src/params/store.rs +++ b/core/src/params/store.rs @@ -28,9 +28,21 @@ where } } + pub fn get(&self, kind: &ParamKind) -> Option<&Parameter> { + self.store.get(kind) + } + + pub fn get_mut(&mut self, kind: &ParamKind) -> Option<&mut Parameter> { + self.store.get_mut(kind) + } + pub fn insert(&mut self, param: Parameter) { self.store.insert(param.kind().clone(), param); } + + pub fn remove(&mut self, kind: &ParamKind) -> Option> { + self.store.remove(kind) + } } impl Extend> for ParamStore @@ -58,6 +70,19 @@ where } } +impl<'a, T, D> IntoIterator for &'a mut ParamStore +where + D: Dimension, + T: Float, +{ + type Item = (&'a ParamKind, &'a mut Parameter); + type IntoIter = std::collections::hash_map::IterMut<'a, ParamKind, Parameter>; + + fn into_iter(self) -> Self::IntoIter { + self.store.iter_mut() + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/core/src/specs/arrays.rs b/core/src/specs/arrays.rs index 8058d3b2..3531bdec 100644 --- a/core/src/specs/arrays.rs +++ b/core/src/specs/arrays.rs @@ -2,13 +2,33 @@ Appellation: base Contrib: FL03 */ -use ndarray::prelude::{Array, Axis, Dimension, Ix2, NdFloat}; -use ndarray::IntoDimension; +use ndarray::prelude::{Array, Axis, Dimension, Ix1, Ix2, NdFloat}; +use ndarray::{IntoDimension, ScalarOperand, ShapeError}; // use ndarray::linalg::Dot; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Uniform}; +use distr::uniform::SampleUniform; +use distr::{Bernoulli, BernoulliError, Distribution, StandardNormal, Uniform}; +use ndarray_rand::rand_distr as distr; use ndarray_rand::RandomExt; -use num::Float; + +use num::{Float, Num}; + +pub trait Affine: Sized { + type Error; + + fn affine(&self, mul: T, add: T) -> Result; +} + +impl Affine for Array +where + T: Num + ScalarOperand, + D: Dimension, +{ + type Error = ShapeError; + + fn affine(&self, mul: T, add: T) -> Result { + Ok(self * mul + add) + } +} pub trait Arange { fn arange(start: T, stop: T, step: T) -> Self; @@ -24,49 +44,100 @@ where } } +impl Arange for Array +where + T: Float, +{ + fn arange(start: T, stop: T, step: T) -> Self { + let n = ((stop - start) / step).ceil().to_usize().unwrap(); + Array::from_shape_fn(n, |i| start + step * T::from(i).unwrap()) + } +} + impl Arange for Array where T: Float, { fn arange(start: T, stop: T, step: T) -> Self { let n = ((stop - start) / step).ceil().to_usize().unwrap(); - let mut a = Array::zeros((n, 1)); - for (i, v) in a.iter_mut().enumerate() { - *v = start + step * T::from(i).unwrap(); - } - a + Array::from_shape_fn((n, 1), |(i, ..)| start + step * T::from(i).unwrap()) } } -pub trait GenerateRandom +pub trait RandNum: SampleUniform +where + StandardNormal: Distribution, +{ +} + +impl RandNum for T +where + T: SampleUniform, + StandardNormal: Distribution, +{ +} + +pub trait GenerateRandom: Sized where D: Dimension, T: Float + SampleUniform, + StandardNormal: Distribution, { - fn bernoulli( - dim: impl IntoDimension, - p: Option, - ) -> Result, BernoulliError> { + fn rand(dim: impl IntoDimension, distr: IdS) -> Self + where + IdS: Distribution; + + fn bernoulli(dim: impl IntoDimension, p: Option) -> Result + where + Bernoulli: Distribution, + { let dist = Bernoulli::new(p.unwrap_or(0.5))?; - Ok(Array::random(dim.into_dimension(), dist)) + Ok(Self::rand(dim.into_dimension(), dist)) } - fn uniform(axis: usize, dim: impl IntoDimension) -> Array { + fn stdnorm(dim: impl IntoDimension) -> Self { + Self::rand(dim, StandardNormal) + } + + fn uniform(axis: usize, dim: impl IntoDimension) -> Self { let dim = dim.into_dimension(); - let dk = (T::one() / T::from(dim[axis]).unwrap()).sqrt(); + let dk = T::from(dim[axis]).unwrap().recip().sqrt(); Self::uniform_between(dk, dim) } - fn uniform_between(dk: T, dim: impl IntoDimension) -> Array { - Array::random(dim, Uniform::new(-dk, dk)) + fn uniform_between(dk: T, dim: impl IntoDimension) -> Self { + Self::rand(dim, Uniform::new(-dk, dk)) } } +pub trait GenerateExt: GenerateRandom +where + D: Dimension, + T: Float + SampleUniform, + StandardNormal: Distribution, +{ + type Output; + // fn bernoulli( + // dim: impl IntoDimension, + // p: Option, + // ) -> Result { + // let dist = Bernoulli::new(p.unwrap_or(0.5))?; + // Ok(Array::random(dim.into_dimension(), dist)) + // } +} + impl GenerateRandom for Array where T: Float + SampleUniform, D: Dimension, + StandardNormal: Distribution, { + fn rand(dim: impl IntoDimension, distr: IdS) -> Self + where + IdS: Distribution, + { + Self::random(dim.into_dimension(), distr) + } } pub trait IntoAxis { diff --git a/core/src/specs/mod.rs b/core/src/specs/mod.rs index f929a11a..3407cec3 100644 --- a/core/src/specs/mod.rs +++ b/core/src/specs/mod.rs @@ -9,8 +9,13 @@ pub(crate) mod base; pub(crate) mod init; pub(crate) mod math; +use num::traits::float::FloatCore; use num::{Complex, Num, Zero}; +pub trait CncFloat: FloatCore {} + +impl CncFloat for T where T: FloatCore {} + pub trait AsComplex: Num { fn as_complex(&self) -> Complex; @@ -47,6 +52,7 @@ where mod tests { use super::*; + use ndarray::prelude::*; #[test] fn test_as_complex() { @@ -54,4 +60,12 @@ mod tests { let y = x.as_complex(); assert_eq!(y, Complex::new(1.0, 0.0)); } + + #[test] + fn test_affine() { + let x = array![[0.0, 1.0], [2.0, 3.0]]; + + let y = x.affine(4.0, -2.0).unwrap(); + assert_eq!(y, array![[-2.0, 2.0], [6.0, 10.0]]); + } } diff --git a/core/src/utils.rs b/core/src/utils.rs index a44e8911..a37a6585 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -5,6 +5,8 @@ use ndarray::prelude::*; use ndarray::{concatenate, IntoDimension, RemoveAxis, ShapeError}; +use ndarray_rand::rand_distr::{Bernoulli, BernoulliError}; +use ndarray_rand::RandomExt; use num::cast::AsPrimitive; use num::{Float, Num, NumCast, Zero}; @@ -21,6 +23,17 @@ where res } +pub fn bernoulli( + dim: impl IntoDimension, + p: Option, +) -> Result, BernoulliError> +where + D: Dimension, +{ + let dist = Bernoulli::new(p.unwrap_or(0.5))?; + Ok(Array::random(dim.into_dimension(), dist)) +} + pub fn cauchy_dot(a: &Array, lambda: &Array, omega: &Array) -> T where D: Dimension, @@ -32,7 +45,7 @@ where pub fn compute_inverse(matrix: &Array2) -> Option> { let (rows, cols) = matrix.dim(); - if rows != cols { + if !matrix.is_square() { return None; // Matrix must be square for inversion } @@ -163,3 +176,27 @@ where } out } + +pub fn hstack(iter: impl IntoIterator>) -> Array2 +where + T: Clone + Num, +{ + let iter = Vec::from_iter(iter); + let mut res = Array2::::zeros((iter.first().unwrap().len(), iter.len())); + for (i, s) in iter.iter().enumerate() { + res.slice_mut(s![.., i]).assign(s); + } + res +} + +pub fn vstack(iter: impl IntoIterator>) -> Array2 +where + T: Clone + Num, +{ + let iter = Vec::from_iter(iter); + let mut res = Array2::::zeros((iter.len(), iter.first().unwrap().len())); + for (i, s) in iter.iter().enumerate() { + res.slice_mut(s![i, ..]).assign(s); + } + res +} diff --git a/data/Cargo.toml b/data/Cargo.toml index 3aaf713e..3fdce024 100644 --- a/data/Cargo.toml +++ b/data/Cargo.toml @@ -15,6 +15,7 @@ version.workspace = true default = [] blas = [ + "linfa/blas", "ndarray/blas", ] diff --git a/data/src/lib.rs b/data/src/lib.rs index 77dc561c..2e5ae7ee 100644 --- a/data/src/lib.rs +++ b/data/src/lib.rs @@ -6,8 +6,9 @@ //! #![feature(associated_type_defaults)] -pub use self::{primitives::*, specs::*, utils::*}; +pub use self::{misc::*, primitives::*, specs::*, utils::*}; +pub(crate) mod misc; pub(crate) mod primitives; pub(crate) mod specs; pub(crate) mod utils; diff --git a/data/src/misc/dtype.rs b/data/src/misc/dtype.rs new file mode 100644 index 00000000..1a176c44 --- /dev/null +++ b/data/src/misc/dtype.rs @@ -0,0 +1,11 @@ +/* + Appellation: dtype + Contrib: FL03 +*/ + +pub enum DType {} + +pub enum FloatingPoint { + F32, + F64, +} diff --git a/data/src/misc/mod.rs b/data/src/misc/mod.rs new file mode 100644 index 00000000..c0a8989f --- /dev/null +++ b/data/src/misc/mod.rs @@ -0,0 +1,8 @@ +/* + Appellation: misc + Contrib: FL03 +*/ +//! # Misc +pub use self::dtype::*; + +pub(crate) mod dtype; diff --git a/data/src/specs.rs b/data/src/specs.rs index ba4b3df3..b6d9d9d7 100644 --- a/data/src/specs.rs +++ b/data/src/specs.rs @@ -2,6 +2,7 @@ Appellation: specs Contrib: FL03 */ +use ndarray::prelude::{Array1, Array2}; pub trait Records { fn features(&self) -> usize; @@ -9,17 +10,34 @@ pub trait Records { fn samples(&self) -> usize; } -impl Records for S -where - S: AsRef<(usize, usize)>, -{ +impl Records for Array1 { fn features(&self) -> usize { - self.as_ref().1 + self.shape()[1] } fn samples(&self) -> usize { - self.as_ref().0 + self.shape()[0] + } +} + +impl Records for Array2 { + fn features(&self) -> usize { + self.shape()[1] + } + + fn samples(&self) -> usize { + self.shape()[0] } } pub trait NdArrayExt {} + +pub trait Store { + fn get(&self, key: &K) -> Option<&V>; + + fn get_mut(&mut self, key: &K) -> Option<&mut V>; + + fn insert(&mut self, key: K, value: V) -> Option; + + fn remove(&mut self, key: &K) -> Option; +} diff --git a/ml/linear/src/params/store.rs b/ml/linear/src/params/store.rs index 27a2034a..84127685 100644 --- a/ml/linear/src/params/store.rs +++ b/ml/linear/src/params/store.rs @@ -9,6 +9,7 @@ use crate::neural::prelude::{Biased, Features, Forward, Node, Weighted}; use ndarray::linalg::Dot; use ndarray::prelude::{Array, Array1, Array2, Axis, Dimension, Ix2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use num::Float; use serde::{Deserialize, Serialize}; use std::ops; @@ -83,6 +84,7 @@ where impl LayerParams where T: Float + SampleUniform, + StandardNormal: Distribution, { pub fn init(mut self, biased: bool) -> Self { if biased { diff --git a/ml/neural/src/layers/exp/layer.rs b/ml/neural/src/layers/exp/layer.rs index a6f20dc5..b2724850 100644 --- a/ml/neural/src/layers/exp/layer.rs +++ b/ml/neural/src/layers/exp/layer.rs @@ -3,13 +3,13 @@ Contrib: FL03 */ use super::LayerConfig; -use crate::func::activate::{Activate, Activator, Linear}; -use crate::layers::{LayerParams, LayerShape}; -use crate::prelude::{Features, Forward, Node, ParamGroup, Parameterized, Params}; +use crate::func::activate::{Activate, Activator}; +use crate::layers::LayerShape; +use crate::prelude::{Forward, ParamGroup, Parameterized, Params}; use ndarray::prelude::{Array2, Dimension, Ix1, Ix2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use num::Float; -use serde::{Deserialize, Serialize}; pub struct Layer where @@ -90,6 +90,7 @@ where impl Layer where T: Float + SampleUniform, + StandardNormal: Distribution, { pub fn init(mut self, biased: bool) -> Self { self.params = self.params.init(biased); diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index a4466e16..17c8996e 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -7,6 +7,7 @@ use crate::func::activate::{Activate, Gradient, Linear}; use crate::prelude::{Features, Forward, Node, Parameterized, Params, Perceptron}; use ndarray::prelude::{Array2, Ix1, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use ndarray_stats::DeviationExt; use num::{Float, Signed}; use serde::{Deserialize, Serialize}; @@ -156,6 +157,7 @@ impl Layer where A: Activate, T: Float + SampleUniform, + StandardNormal: Distribution, { pub fn init(mut self, biased: bool) -> Self { self.params = self.params.init(biased); diff --git a/ml/neural/src/layers/params.rs b/ml/neural/src/layers/params.rs index fe7f3449..d50917b3 100644 --- a/ml/neural/src/layers/params.rs +++ b/ml/neural/src/layers/params.rs @@ -8,6 +8,7 @@ use crate::prelude::{Biased, Features, Forward, Node, Weighted}; use ndarray::linalg::Dot; use ndarray::prelude::{Array, Array1, Array2, Axis, Dimension, Ix2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use num::Float; use serde::{Deserialize, Serialize}; use std::ops; @@ -82,6 +83,7 @@ where impl LayerParams where T: Float + SampleUniform, + StandardNormal: Distribution, { pub fn init(mut self, biased: bool) -> Self { if biased { diff --git a/ml/neural/src/layers/stack.rs b/ml/neural/src/layers/stack.rs index 0ea51b61..60eb1a41 100644 --- a/ml/neural/src/layers/stack.rs +++ b/ml/neural/src/layers/stack.rs @@ -4,6 +4,8 @@ */ use crate::layers::{Layer, LayerShape}; use crate::prelude::{Activate, Features, Linear, Parameterized}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use num::Float; use serde::{Deserialize, Serialize}; use std::ops; @@ -80,7 +82,8 @@ where impl Stack where A: Activate + Clone + Default, - T: Float + crate::core::prelude::SampleUniform, + T: Float + SampleUniform, + StandardNormal: Distribution, { pub fn init_layers(mut self, biased: bool) -> Self { self.children diff --git a/ml/neural/src/models/exp/mod.rs b/ml/neural/src/models/exp/mod.rs index 51aa3c3d..e60e5f74 100644 --- a/ml/neural/src/models/exp/mod.rs +++ b/ml/neural/src/models/exp/mod.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ //! # Experimental Models -pub use self::{modules::*, store::*, utils::*}; +pub use self::{modules::*, store::*}; pub(crate) mod modules; pub(crate) mod store; @@ -34,7 +34,5 @@ where } } -pub(crate) mod utils {} - #[cfg(test)] mod tests {} diff --git a/ml/neural/src/models/exp/modules.rs b/ml/neural/src/models/exp/modules.rs index c388bfa1..64cddbea 100644 --- a/ml/neural/src/models/exp/modules.rs +++ b/ml/neural/src/models/exp/modules.rs @@ -4,8 +4,8 @@ */ //! # Model //! -use crate::prelude::{Forward, Weighted}; -use ndarray::prelude::{Array2, NdFloat}; +use crate::prelude::Forward; +use ndarray::prelude::Array2; use num::Float; use std::collections::HashMap; diff --git a/ml/neural/src/models/exp/store.rs b/ml/neural/src/models/exp/store.rs index 758755f7..2367d85a 100644 --- a/ml/neural/src/models/exp/store.rs +++ b/ml/neural/src/models/exp/store.rs @@ -6,6 +6,7 @@ use crate::prelude::{LayerParams, LayerPosition, LayerShape}; use ndarray::prelude::{Dimension, Ix2}; use ndarray::IntoDimension; use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use num::Float; // use serde::{Deserialize, Serialize}; @@ -75,6 +76,7 @@ where impl ModelStore where T: Float + SampleUniform, + StandardNormal: Distribution, { pub fn init(mut self, biased: bool) -> Self { self.store.iter_mut().for_each(|(_, l)| { diff --git a/ml/neural/src/models/mod.rs b/ml/neural/src/models/mod.rs index e570cf49..d89fdfd0 100644 --- a/ml/neural/src/models/mod.rs +++ b/ml/neural/src/models/mod.rs @@ -4,7 +4,7 @@ */ //! # Model //! -pub use self::{config::*, model::*, modes::*, params::*, utils::*}; +pub use self::{config::*, model::*, modes::*, params::*}; pub(crate) mod config; pub(crate) mod model; @@ -13,7 +13,5 @@ pub(crate) mod params; pub mod exp; -pub(crate) mod utils {} - #[cfg(test)] mod tests {} diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs index 636e719a..edc49384 100644 --- a/ml/neural/src/models/model.rs +++ b/ml/neural/src/models/model.rs @@ -7,6 +7,7 @@ use crate::prelude::{Forward, Gradient, LayerParams, Weighted}; use ndarray::linalg::Dot; use ndarray::prelude::{Array, Array1, Array2, Dimension, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use ndarray_stats::DeviationExt; use num::{Float, Signed}; use std::ops; @@ -110,6 +111,7 @@ where impl Model where T: Float + SampleUniform, + StandardNormal: Distribution, { pub fn init(mut self, biased: bool) -> Self { self.params = self.params.init(biased); diff --git a/ml/neural/src/models/params.rs b/ml/neural/src/models/params.rs index bf051167..3476dc03 100644 --- a/ml/neural/src/models/params.rs +++ b/ml/neural/src/models/params.rs @@ -6,6 +6,7 @@ use crate::prelude::{Features, Forward, LayerParams, LayerShape}; use ndarray::prelude::{Array2, Dimension, Ix2, NdFloat}; use ndarray::IntoDimension; use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use num::Float; use serde::{Deserialize, Serialize}; @@ -93,6 +94,7 @@ where impl ModelParams where T: Float + SampleUniform, + StandardNormal: Distribution, { pub fn init(mut self, biased: bool) -> Self { self.children diff --git a/ml/neural/src/neurons/node.rs b/ml/neural/src/neurons/node.rs index ee9d451c..1f9dd4bc 100644 --- a/ml/neural/src/neurons/node.rs +++ b/ml/neural/src/neurons/node.rs @@ -9,6 +9,7 @@ use ndarray::linalg::Dot; use ndarray::prelude::{Array, Array0, Array1, Array2, Dimension, Ix1, NdFloat}; use ndarray::RemoveAxis; use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use num::{Float, FromPrimitive}; use std::ops; @@ -65,6 +66,7 @@ where impl Node where T: Float + SampleUniform, + StandardNormal: Distribution, { pub fn init(mut self, biased: bool) -> Self { if biased { diff --git a/ml/neural/src/neurons/perceptron.rs b/ml/neural/src/neurons/perceptron.rs index da2c5bab..f636988b 100644 --- a/ml/neural/src/neurons/perceptron.rs +++ b/ml/neural/src/neurons/perceptron.rs @@ -7,6 +7,7 @@ use crate::func::activate::{Activate, Linear}; use crate::prelude::{Forward, Parameterized, ParameterizedExt, Weighted}; use ndarray::prelude::{Array0, Array1, Array2, Ix1, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use num::Float; /// Artificial Neuron @@ -95,6 +96,7 @@ impl Perceptron where T: Float + SampleUniform, A: Activate, + StandardNormal: Distribution, { pub fn init(mut self, biased: bool) -> Self { if biased { diff --git a/ml/neural/src/params/bias.rs b/ml/neural/src/params/bias.rs deleted file mode 100644 index 1fa1ae67..00000000 --- a/ml/neural/src/params/bias.rs +++ /dev/null @@ -1,359 +0,0 @@ -/* - Appellation: bias - Contrib: FL03 -*/ -use crate::core::prelude::GenerateRandom; -use crate::generate_uniform_arr; -use ndarray::prelude::{Array, Array1, Dimension, Ix1, NdFloat}; -use ndarray::IntoDimension; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use num::Float; -use serde::{Deserialize, Serialize}; -use smart_default::SmartDefault; -use std::ops; -use strum::EnumIs; - -pub struct Belief -where - D: Dimension, - T: Float, -{ - pub bias: Array, - pub features: D, -} - -#[derive(Clone, Debug, Deserialize, EnumIs, PartialEq, Serialize, SmartDefault)] -pub enum Biases -where - D: Dimension, - T: Float, -{ - Biased(Array), - #[default] - Unbiased, -} - -impl Biases -where - D: Dimension, - T: Float, -{ - pub fn biased(bias: Array) -> Self { - Self::Biased(bias) - } -} - -impl Biases -where - D: Dimension, - T: Float + SampleUniform, -{ - pub fn init(self, dk: T, features: impl IntoDimension) -> Self { - Self::Biased(Array::uniform_between(dk, features)) - } - pub fn uniform(dk: T, features: impl IntoDimension) -> Self { - Self::Biased(Array::uniform_between(dk, features)) - } -} - -impl From> for Biases -where - D: Dimension, - T: Float, -{ - fn from(bias: Array) -> Self { - Self::Biased(bias) - } -} - -impl From>> for Biases -where - D: Dimension, - T: Float, -{ - fn from(bias: Option>) -> Self { - match bias { - Some(bias) => Self::Biased(bias), - None => Self::Unbiased, - } - } -} - -impl From> for Option> -where - D: Dimension, - T: Float, -{ - fn from(bias: Biases) -> Self { - match bias { - Biases::Biased(bias) => Some(bias), - Biases::Unbiased => None, - } - } -} - -impl From> for Array -where - D: Dimension, - T: Float, -{ - fn from(bias: Biases) -> Self { - match bias { - Biases::Biased(bias) => bias, - Biases::Unbiased => Array::zeros(D::zeros(D::NDIM.unwrap_or_default())), - } - } -} - -// impl<'a, T, D> From<&'a Biases> for ArrayView<'a, T, D> -// where -// D: Dimension + 'a, -// T: Float, -// { -// fn from(bias: &'a Biases) -> Self { -// match bias { -// Biases::Biased(bias) => bias.view(), -// Biases::Unbiased => ArrayView::empty(D::zeros(D::NDIM.unwrap_or_default())), -// } -// } -// } - -#[derive(Clone, Debug, Deserialize, EnumIs, PartialEq, Serialize, SmartDefault)] -pub enum Bias { - Biased(Array1), - #[default] - Unbiased, - Value(T), -} - -impl Bias -where - T: Float, -{ - pub fn update_at(&mut self, index: usize, value: T) { - match self { - Self::Biased(bias) => bias[index] = value, - Self::Unbiased => (), - Self::Value(bias) => *bias = value, - } - } -} - -impl Bias -where - T: NdFloat, -{ - pub fn forward(&self, data: &Array1) -> Array1 { - match self { - Self::Biased(bias) => data + bias, - Self::Unbiased => data.clone(), - Self::Value(value) => data + value.clone(), - } - } -} - -impl Bias -where - T: Float + SampleUniform, -{ - pub fn biased(size: usize) -> Self { - let bias = generate_uniform_arr(0, size); - Self::Biased(bias) - } -} - -impl From for Bias -where - T: Float, -{ - fn from(value: T) -> Self { - Self::Value(value) - } -} - -impl From> for Bias -where - T: Float, -{ - fn from(bias: Array1) -> Self { - Self::Biased(bias) - } -} - -impl From>> for Bias -where - T: Float, -{ - fn from(bias: Option>) -> Self { - match bias { - Some(bias) => Self::Biased(bias), - None => Self::Unbiased, - } - } -} - -impl From> for Option> -where - T: Float, -{ - fn from(bias: Bias) -> Self { - match bias { - Bias::Biased(bias) => Some(bias), - Bias::Unbiased => None, - Bias::Value(value) => Some(Array1::::from_elem(1, value)), - } - } -} - -impl ops::Add> for Bias -where - D: Dimension, - T: NdFloat, - Array: ops::Add, Output = Array>, -{ - type Output = Array; - - fn add(self, rhs: Array) -> Self::Output { - match self { - Self::Biased(bias) => rhs + bias, - Self::Unbiased => rhs, - Self::Value(value) => &rhs + value, - } - } -} - -impl ops::Add> for &Bias -where - D: Dimension, - T: NdFloat, - Array: ops::Add, Output = Array>, -{ - type Output = Array; - - fn add(self, rhs: Array) -> Self::Output { - match self.clone() { - Bias::Biased(bias) => rhs + bias, - Bias::Unbiased => rhs, - Bias::Value(value) => &rhs + value, - } - } -} - -impl ops::Add<&Array> for Bias -where - D: Dimension, - T: NdFloat, - Array: ops::Add, Output = Array>, -{ - type Output = Array; - - fn add(self, rhs: &Array) -> Self::Output { - match self { - Self::Biased(bias) => rhs.clone() + bias, - Self::Unbiased => rhs.clone(), - Self::Value(value) => rhs + value, - } - } -} - -impl ops::Add> for Array -where - D: Dimension, - T: NdFloat, - Array: ops::Add, Output = Array>, -{ - type Output = Array; - - fn add(self, bias: Bias) -> Self::Output { - match bias.clone() { - Bias::Biased(bias) => self.clone() + bias, - Bias::Unbiased => self.clone(), - Bias::Value(value) => &self + value, - } - } -} - -impl ops::Add<&Bias> for Array -where - D: Dimension, - T: NdFloat, - Array: ops::Add, Output = Array>, -{ - type Output = Array; - - fn add(self, bias: &Bias) -> Self::Output { - match bias.clone() { - Bias::Biased(bias) => self.clone() + bias, - Bias::Unbiased => self.clone(), - Bias::Value(value) => &self + value, - } - } -} - -impl ops::Sub> for Bias -where - D: Dimension, - T: NdFloat, - Array: ops::Sub, Output = Array>, -{ - type Output = Array; - - fn sub(self, rhs: Array) -> Self::Output { - match self { - Self::Biased(bias) => rhs - bias, - Self::Unbiased => rhs, - Self::Value(value) => &rhs - value, - } - } -} - -impl ops::Sub<&Array> for Bias -where - D: Dimension, - T: NdFloat, - Array: ops::Sub, Output = Array>, -{ - type Output = Array; - - fn sub(self, rhs: &Array) -> Self::Output { - match self { - Self::Biased(bias) => rhs.clone() - bias, - Self::Unbiased => rhs.clone(), - Self::Value(value) => rhs - value, - } - } -} - -impl ops::Sub> for Array -where - D: Dimension, - T: NdFloat, - Array: ops::Sub, Output = Array>, -{ - type Output = Array; - - fn sub(self, bias: Bias) -> Self::Output { - match bias.clone() { - Bias::Biased(bias) => self.clone() - bias, - Bias::Unbiased => self.clone(), - Bias::Value(value) => &self - value, - } - } -} - -impl ops::Sub<&Bias> for Array -where - D: Dimension, - T: NdFloat, - Array: ops::Sub, Output = Array>, -{ - type Output = Array; - - fn sub(self, bias: &Bias) -> Self::Output { - match bias.clone() { - Bias::Biased(bias) => self.clone() - bias, - Bias::Unbiased => self.clone(), - Bias::Value(value) => &self - value, - } - } -} diff --git a/ml/neural/src/params/group.rs b/ml/neural/src/params/group.rs index 6c306aed..b8f5b9e9 100644 --- a/ml/neural/src/params/group.rs +++ b/ml/neural/src/params/group.rs @@ -4,10 +4,11 @@ */ use super::{Biased, Weighted}; use crate::core::prelude::GenerateRandom; -use crate::prelude::{Forward, Node}; -use ndarray::prelude::{s, Array, Axis, Dimension, Ix1, Ix2}; +use crate::prelude::Forward; +use ndarray::prelude::{Array, Axis, Dimension, Ix1, Ix2}; use ndarray::{IntoDimension, RemoveAxis}; use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use num::Float; use serde::{Deserialize, Serialize}; @@ -75,6 +76,7 @@ impl ParamGroup where T: Float + SampleUniform, D: Dimension + RemoveAxis, + StandardNormal: Distribution, { pub fn init(mut self, biased: bool) -> Self { if biased { diff --git a/ml/neural/src/params/mod.rs b/ml/neural/src/params/mod.rs index b2b1b20d..78f0a632 100644 --- a/ml/neural/src/params/mod.rs +++ b/ml/neural/src/params/mod.rs @@ -6,13 +6,11 @@ //! //! ## Overview //! -pub use self::{bias::*, group::*, param::*, shapes::*, utils::*, weight::*}; +pub use self::{group::*, param::*, shapes::*}; -pub(crate) mod bias; pub(crate) mod group; pub(crate) mod param; pub(crate) mod shapes; -pub(crate) mod weight; use ndarray::linalg::Dot; use ndarray::prelude::{Array, Dimension, Ix2}; @@ -215,7 +213,5 @@ where // } // } -pub(crate) mod utils {} - #[cfg(test)] mod tests {} diff --git a/ml/neural/src/params/weight.rs b/ml/neural/src/params/weight.rs deleted file mode 100644 index f1d65959..00000000 --- a/ml/neural/src/params/weight.rs +++ /dev/null @@ -1,62 +0,0 @@ -/* - Appellation: weight - Contrib: FL03 -*/ -use crate::core::prelude::GenerateRandom; -use ndarray::prelude::Array2; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use num::Float; -use serde::{Deserialize, Serialize}; - -pub enum WeightShape {} - -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct Weight { - weights: Array2, -} - -impl Weight -where - T: Float, -{ - pub fn new(inputs: usize, outputs: Option) -> Self { - Self { - weights: Array2::zeros((outputs.unwrap_or(1), inputs)), - } - } - /// Returns the shape of the weights. (outputs, inputs) - pub fn shape(&self) -> (usize, usize) { - self.weights.dim() - } - - pub fn inputs(&self) -> usize { - self.shape().1 - } - - pub fn outputs(&self) -> usize { - self.shape().0 - } -} - -impl Weight -where - T: Float + SampleUniform, -{ - pub fn init(mut self) -> Self { - let dk = (T::one() / T::from(self.inputs()).unwrap()).sqrt(); - self.weights = Array2::uniform_between(dk, self.shape()); - self - } -} - -impl AsRef> for Weight { - fn as_ref(&self) -> &Array2 { - &self.weights - } -} - -impl AsMut> for Weight { - fn as_mut(&mut self) -> &mut Array2 { - &mut self.weights - } -} diff --git a/ml/optim/src/optimizer.rs b/ml/optim/src/optimizer.rs index 56c7f783..025b1b23 100644 --- a/ml/optim/src/optimizer.rs +++ b/ml/optim/src/optimizer.rs @@ -20,14 +20,3 @@ where fn step(&mut self, params: impl Params) -> T; } - -pub trait OptimizerExt: Optimizer -where - T: Float, -{ -} - -pub struct Opt { - epochs: usize, - gamma: T, -} diff --git a/ml/optim/src/params/mod.rs b/ml/optim/src/params/mod.rs index 62d370bd..2bbe6ef2 100644 --- a/ml/optim/src/params/mod.rs +++ b/ml/optim/src/params/mod.rs @@ -6,8 +6,6 @@ //! //! ## Overview //! -pub use self::utils::*; - use num::Float; pub trait Minimize { @@ -44,7 +42,5 @@ where fn nestrov(&self) -> bool; } -pub(crate) mod utils {} - #[cfg(test)] mod tests {} diff --git a/ml/s4/src/dplr/hippo.rs b/ml/s4/src/dplr/hippo.rs index 7497721b..d364c2ad 100644 --- a/ml/s4/src/dplr/hippo.rs +++ b/ml/s4/src/dplr/hippo.rs @@ -16,8 +16,9 @@ pub enum HiPPOs { b: Array2, }, DPLR { - a: Array2, + lambda: Array2, p: Array2, + q: Array2, b: Array2, c: Array2, }, diff --git a/ml/s4/src/dplr/mod.rs b/ml/s4/src/dplr/mod.rs index c6c92f90..cd06181e 100644 --- a/ml/s4/src/dplr/mod.rs +++ b/ml/s4/src/dplr/mod.rs @@ -20,10 +20,9 @@ pub(crate) mod utils { use ndarray::prelude::{Array1, Array2, Axis}; use ndarray::ScalarOperand; - use ndarray_linalg::eigh::Eigh; - use ndarray_linalg::{IntoTriangular, UPLO}; - use num::complex::ComplexFloat; - use num::Complex; + use ndarray_linalg::{Eigh, IntoTriangular, Lapack, UPLO}; + use num::complex::{Complex, ComplexFloat}; + use num::FromPrimitive; pub fn make_hippo(features: usize) -> Array2 where @@ -48,15 +47,11 @@ pub(crate) mod utils { (hippo, p, b) } - pub fn make_dplr_hippo( - features: usize, - ) -> ( - Array2>, - Array2>, - Array2>, - Array2>, - ) { - let (a, p, b) = make_nplr_hippo::>(features); + pub fn make_dplr_hippo(features: usize) -> (Array2, Array2, Array2, Array2) + where + T: AsComplex + ComplexFloat + Conjugate + FromPrimitive + Lapack + ScalarOperand, + { + let (a, p, b) = make_nplr_hippo(features); let p = p.insert_axis(Axis(1)); let b = b.insert_axis(Axis(1)); @@ -68,10 +63,12 @@ pub(crate) mod utils { let a = Array2::ones(s.dim()) * sd.mean().expect("Average of diagonal is NaN"); // TODO: replace with eigh - let (e, v) = &(&s * (-1.0).as_imag()).eigh(UPLO::Lower).expect(""); - let e = e.mapv(|x| x.as_complex()); + let (e, v) = &(&s * T::from(T::one().neg().as_imag()).unwrap()) + .eigh(UPLO::Lower) + .expect(""); + let e = e.mapv(|x| T::from(x).unwrap()); - let a = a + &e * 1.0.as_imag(); + let a = a + &e * T::from(T::one().as_imag()).unwrap(); let p = v.conj().t().dot(&p); let b = v.conj().t().dot(&b); (a, p, b, v.clone()) diff --git a/ml/s4/src/model/params.rs b/ml/s4/src/model/params.rs index 0aa8108d..098f2acc 100644 --- a/ml/s4/src/model/params.rs +++ b/ml/s4/src/model/params.rs @@ -2,9 +2,10 @@ Appellation: params Contrib: FL03 */ -use ndarray::prelude::{Array1, Array2, ArrayView1, NdFloat}; -use num::complex::{Complex, ComplexFloat}; -use num::Float; +use ndarray::prelude::{Array1, Array2, ArrayView1}; +use ndarray_linalg::error::LinalgError; +use ndarray_linalg::{vstack, Scalar}; +use num::complex::ComplexFloat; pub struct S4Store where @@ -67,15 +68,20 @@ where impl S4Store where - T: ComplexFloat + 'static, + T: ComplexFloat + Scalar + 'static, { - pub fn scan(&self, u: &Array2, x0: &Array1) -> Array2 { + pub fn scan(&self, u: &Array2, x0: &Array1) -> Result, LinalgError> { let step = |xs: &mut Array1, us: ArrayView1| { let x1 = self.a.dot(xs) + self.b.t().dot(&us); let y1 = self.c.dot(&x1.t()); Some(y1) }; - crate::stack_arrays(u.outer_iter().scan(x0.clone(), step)) + vstack( + u.outer_iter() + .scan(x0.clone(), step) + .collect::>() + .as_slice(), + ) } } diff --git a/ml/s4/src/ops/discretize.rs b/ml/s4/src/ops/discretize.rs index a963f67e..922b5d24 100644 --- a/ml/s4/src/ops/discretize.rs +++ b/ml/s4/src/ops/discretize.rs @@ -2,11 +2,12 @@ Appellation: discretize Contrib: FL03 */ -use crate::core::prelude::Inverse; +use crate::core::prelude::Conjugate; +use crate::prelude::powmat; -use ndarray::prelude::{Array2, NdFloat}; - -use num::{Float, ToPrimitive}; +use ndarray::{Array2, ScalarOperand}; +use ndarray_linalg::{Inverse, Lapack, Scalar}; +use num::Float; pub fn discretize( a: &Array2, @@ -15,12 +16,12 @@ pub fn discretize( step: T, ) -> anyhow::Result<(Array2, Array2, Array2)> where - T: NdFloat, + T: Lapack + Scalar + ScalarOperand, { let ss = step / T::from(2).unwrap(); // half step let eye = Array2::::eye(a.shape()[0]); - let be = (&eye - a * ss).inverse().expect("Could not invert matrix"); + let be = (&eye - a * ss).inv().expect("Could not invert matrix"); let ab = be.dot(&(&eye + a * ss)); let bb = (b * ss).dot(&b.t()); @@ -28,6 +29,56 @@ where Ok((ab, bb, c.clone())) } +pub fn discretize_dplr( + lambda: &Array2, + p: &Array2, + q: &Array2, + b: &Array2, + c: &Array2, + step: T, + l: usize, +) -> anyhow::Result<(Array2, Array2, Array2)> +where + T: Conjugate + Float + Lapack + Scalar + ScalarOperand, +{ + let (n, _m) = lambda.dim(); + + let eye = Array2::::eye(n); + let ss = T::from(2).unwrap() * step.recip(); + + let a = { + let tmp = Array2::from_diag(&lambda.diag()); + tmp - &p.dot(&q.conj().t()) + }; + + let a0 = &eye * ss + &a; + + let d = { + let tmp = lambda.mapv(|i| (ss - i).recip()); + Array2::from_diag(&tmp.diag()) + }; + + let qc = { + let tmp = q.conj(); + tmp.t().to_owned() + }; + let p2 = p.clone(); + + let a1 = { + let tmp = qc.dot(&d.dot(&p2)).mapv(|i| (T::one() + i).recip()); + &d - &d.dot(&p2) * &tmp * &qc.dot(&d) + }; + + let ab = a0.dot(&a1); + let bb = a1.dot(b) * T::from(2).unwrap(); + let cb = { + let tmp = (&eye - powmat(&ab, l)).inv()?.conj(); + c.dot(&tmp) + }; + + Ok((ab, bb, cb.conj())) +} + pub trait Discretize where T: Float, diff --git a/ml/s4/src/ops/scan.rs b/ml/s4/src/ops/scan.rs index 361bfa09..9454044a 100644 --- a/ml/s4/src/ops/scan.rs +++ b/ml/s4/src/ops/scan.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use crate::params::SSMStore; -use crate::prelude::stack_arrays; + use num::Float; pub struct Scan<'a, T = f64> diff --git a/ml/s4/src/params/kinds.rs b/ml/s4/src/params/kinds.rs index 5651dd95..490e71d8 100644 --- a/ml/s4/src/params/kinds.rs +++ b/ml/s4/src/params/kinds.rs @@ -63,3 +63,34 @@ impl From for SSMParams { } } } + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumCount, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum DPLRParams { + #[default] + Lambda = 0, + P = 1, + Q = 2, + B = 3, + C = 4, +} diff --git a/ml/s4/src/params/store.rs b/ml/s4/src/params/store.rs index 403cb940..25d21608 100644 --- a/ml/s4/src/params/store.rs +++ b/ml/s4/src/params/store.rs @@ -4,14 +4,12 @@ */ use super::SSMParams; use crate::core::prelude::GenerateRandom; -use crate::prelude::stack_arrays; -use ndarray::prelude::{Array1, Array2, ArrayView1, NdFloat}; +use ndarray::prelude::{Array1, Array2, ArrayView1}; use ndarray_linalg::error::LinalgError; use ndarray_linalg::{vstack, Scalar}; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use ndarray_rand::RandomExt; -use num::complex::ComplexFloat; use num::{Float, Num}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; diff --git a/ml/s4/src/ssm/model.rs b/ml/s4/src/ssm/model.rs index af65165d..094e6cbc 100644 --- a/ml/s4/src/ssm/model.rs +++ b/ml/s4/src/ssm/model.rs @@ -5,9 +5,10 @@ use super::SSMConfig; use crate::neural::Forward; use crate::params::{SSMParams::*, SSMStore}; -use crate::prelude::{discretize, k_convolve, scanner}; +use crate::prelude::{discretize, k_convolve}; use ndarray::prelude::{Array1, Array2, NdFloat}; use ndarray_conv::{Conv2DFftExt, PaddingMode, PaddingSize}; +use ndarray_linalg::{Lapack, Scalar}; use num::Float; use rustfft::FftNum; @@ -115,7 +116,7 @@ where impl SSM where - T: NdFloat, + T: Lapack + NdFloat + Scalar, { pub fn setup(mut self) -> Self { self.kernel = self.gen_filter(); @@ -127,10 +128,14 @@ where impl SSM where - T: NdFloat, + T: NdFloat + Lapack + Scalar, { - pub fn scan(&self, u: &Array2, x0: &Array1) -> Array2 { - scanner(&self.params[A], &self.params[B], &self.params[C], u, x0) + pub fn scan( + &self, + u: &Array2, + x0: &Array1, + ) -> Result, ndarray_linalg::error::LinalgError> { + self.params.scan(u, x0) } pub fn conv(&self, u: &Array2) -> anyhow::Result> @@ -163,16 +168,17 @@ where impl Forward> for SSM where - T: FftNum + NdFloat, + T: FftNum + Lapack + NdFloat + Scalar, { - type Output = Array2; + type Output = anyhow::Result>; - fn forward(&self, args: &Array2) -> Array2 { + fn forward(&self, args: &Array2) -> Self::Output { let res = if !self.config().decode() { - self.conv(args).expect("convolution failed") + self.conv(args)? } else { - self.scan(args, &self.cache) + self.scan(args, &self.cache)? }; - res + args * &self.params[D] + let pred = res + args * &self.params[D]; + Ok(pred) } } diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index 9af5551f..09ae480b 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -10,8 +10,8 @@ use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, StandardNormal, Uniform}; use ndarray_rand::RandomExt; use num::complex::{Complex, ComplexFloat}; -use num::traits::FloatConst; -use num::{Float, Num, Signed}; +use num::traits::float::{Float, FloatConst, FloatCore}; +use num::{Num, Signed}; use rustfft::{FftNum, FftPlanner}; pub fn stdnorm(shape: impl IntoDimension) -> Array @@ -148,18 +148,6 @@ where res } -pub fn stack_arrays(iter: impl IntoIterator>) -> Array2 -where - T: Clone + Num, -{ - let iter = Vec::from_iter(iter); - let mut res = Array2::::zeros((iter.len(), iter.first().unwrap().len())); - for (i, s) in iter.iter().enumerate() { - res.slice_mut(s![i, ..]).assign(s); - } - res -} - pub fn kernel_dplr( lambda: &Array2, p: &Array2, @@ -175,7 +163,7 @@ where ::Complex: ComplexFloat, { let omega_l = { - let f = | i: usize | { + let f = |i: usize| { T::from(i).unwrap() * T::from(Complex::new(T::one(), -T::PI() / T::from(l).unwrap())).unwrap() }; diff --git a/ml/transformers/src/attention/head.rs b/ml/transformers/src/attention/head.rs index 10218b77..11d3c154 100644 --- a/ml/transformers/src/attention/head.rs +++ b/ml/transformers/src/attention/head.rs @@ -7,6 +7,7 @@ use super::Weight; use crate::neural::func::activate::{Activate, Softmax}; use ndarray::prelude::{Array2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use num::Float; use serde::{Deserialize, Serialize}; use std::ops; @@ -52,6 +53,7 @@ where impl AttentionHead where T: Float + SampleUniform, + StandardNormal: Distribution, { pub fn new(dim: HeadShape) -> Self { Self { diff --git a/ml/transformers/src/attention/multi/attention.rs b/ml/transformers/src/attention/multi/attention.rs index 27ef8aaf..fe71084e 100644 --- a/ml/transformers/src/attention/multi/attention.rs +++ b/ml/transformers/src/attention/multi/attention.rs @@ -10,6 +10,7 @@ use crate::ops::Split; use ndarray::prelude::{Array2, NdFloat}; use ndarray::ShapeError; use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use num::Float; use serde::{Deserialize, Serialize}; @@ -40,6 +41,7 @@ where impl MultiHeadAttention where T: Float + SampleUniform, + StandardNormal: Distribution, { pub fn new(heads: usize, model: usize) -> Self { let features = MultiHeadParams::new(heads, model); diff --git a/ml/transformers/src/attention/weights.rs b/ml/transformers/src/attention/weights.rs index 302db46e..09b23a7b 100644 --- a/ml/transformers/src/attention/weights.rs +++ b/ml/transformers/src/attention/weights.rs @@ -31,6 +31,7 @@ use ndarray::linalg::Dot; use ndarray::prelude::{Array, Array2, Array3, Ix2}; use ndarray::IntoDimension; use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use num::Float; use serde::{Deserialize, Serialize}; use std::ops; @@ -98,6 +99,7 @@ where impl Weight where T: Float + SampleUniform, + StandardNormal: Distribution, { pub fn uniform(dim: impl IntoDimension) -> Self { let dim = dim.into_dimension(); From 9548645ea44657835bde9beba9f69ee9b90f5f74 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 31 Dec 2023 13:23:30 -0600 Subject: [PATCH 100/118] update Signed-off-by: FL03 --- Cargo.toml | 1 + concision/Cargo.toml | 2 + core/src/id/ids/atomic.rs | 52 +++++++ core/src/id/ids/mod.rs | 8 + core/src/id/mod.rs | 27 +++- core/src/params/iter.rs | 7 + core/src/params/kinds.rs | 9 ++ core/src/params/mod.rs | 2 + core/src/params/param.rs | 2 + core/src/primitives.rs | 2 +- core/src/specs/arrays.rs | 8 +- core/src/specs/math.rs | 24 ++- core/src/specs/mod.rs | 4 + data/Cargo.toml | 3 +- data/src/datasets/mod.rs | 5 +- data/src/flows/flow.rs | 19 +-- data/src/flows/mod.rs | 5 +- data/src/lib.rs | 14 +- data/src/misc/dtype.rs | 67 ++++++++- data/src/shape/dimension.rs | 6 + data/src/shape/mod.rs | 11 ++ data/src/specs.rs | 2 +- data/src/store/mod.rs | 13 ++ data/src/store/storage.rs | 6 + data/src/tensors/tensor.rs | 3 + ml/linear/src/model/module.rs | 49 ++---- ml/linear/src/params/store.rs | 145 ++++++++++-------- ml/neural/Cargo.toml | 1 + ml/neural/src/layers/exp/layer.rs | 180 ---------------------- ml/neural/src/layers/exp/mod.rs | 3 +- ml/neural/src/layers/layer.rs | 43 +++--- ml/neural/src/layers/mod.rs | 9 +- ml/neural/src/layers/params.rs | 126 ++++++++++------ ml/neural/src/layers/seq/mod.rs | 11 ++ ml/neural/src/layers/seq/sequential.rs | 6 + ml/neural/src/layers/stack.rs | 2 +- ml/neural/src/models/exp/mod.rs | 38 ++--- ml/neural/src/models/exp/modules.rs | 27 ++-- ml/neural/src/models/model.rs | 2 +- ml/neural/src/neurons/node.rs | 200 +++++++++++++------------ ml/neural/src/neurons/perceptron.rs | 123 +++++---------- ml/neural/src/nn/ffn/mlp.rs | 2 +- ml/neural/src/nn/ffn/mod.rs | 4 +- ml/neural/src/nn/ffn/model.rs | 2 +- ml/neural/src/ops/norm.rs | 5 +- ml/neural/src/params/mod.rs | 170 +-------------------- ml/neural/src/specs.rs | 44 ++---- ml/optim/src/grad/gradient.rs | 2 +- ml/optim/src/grad/mod.rs | 51 ++++--- ml/optim/src/grad/sgd.rs | 2 +- ml/s4/src/dplr/hippo.rs | 58 ------- ml/s4/src/dplr/mod.rs | 76 ---------- ml/s4/src/hippo/dplr.rs | 35 +++++ ml/s4/src/hippo/hippo.rs | 146 ++++++++++++++++++ ml/s4/src/{dplr => hippo}/kinds.rs | 0 ml/s4/src/hippo/mod.rs | 127 ++++++++++++++++ ml/s4/src/hippo/nplr.rs | 42 ++++++ ml/s4/src/lib.rs | 2 +- ml/s4/src/model/model.rs | 11 +- ml/s4/src/primitives.rs | 17 +++ ml/s4/src/specs.rs | 48 ++++++ 61 files changed, 1126 insertions(+), 985 deletions(-) create mode 100644 core/src/id/ids/atomic.rs create mode 100644 core/src/id/ids/mod.rs create mode 100644 data/src/shape/dimension.rs create mode 100644 data/src/shape/mod.rs create mode 100644 data/src/store/mod.rs create mode 100644 data/src/store/storage.rs delete mode 100644 ml/neural/src/layers/exp/layer.rs create mode 100644 ml/neural/src/layers/seq/mod.rs create mode 100644 ml/neural/src/layers/seq/sequential.rs delete mode 100644 ml/s4/src/dplr/hippo.rs delete mode 100644 ml/s4/src/dplr/mod.rs create mode 100644 ml/s4/src/hippo/dplr.rs create mode 100644 ml/s4/src/hippo/hippo.rs rename ml/s4/src/{dplr => hippo}/kinds.rs (100%) create mode 100644 ml/s4/src/hippo/mod.rs create mode 100644 ml/s4/src/hippo/nplr.rs diff --git a/Cargo.toml b/Cargo.toml index 78364f23..13eae1e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ version = "0.1.12" # TODO - Update the cargo package version anyhow = "1" approx = "0.5" +itertools = { features = [], version = "0.12" } lazy_static = "1" # ndarray = { features = ["serde-1"], version = "0.15" } # ndarray-linalg = { features = [], version = "0.16" } diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 52a72a08..40f4bbe2 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -111,6 +111,8 @@ openblas-static = [ "concision-s4/openblas-static", ] +serde = [] + [lib] bench = true crate-type = ["rlib"] diff --git a/core/src/id/ids/atomic.rs b/core/src/id/ids/atomic.rs new file mode 100644 index 00000000..c32aafeb --- /dev/null +++ b/core/src/id/ids/atomic.rs @@ -0,0 +1,52 @@ +/* + Appellation: atomic + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct AtomicId(usize); + +impl AtomicId { + pub fn new() -> Self { + use std::sync::atomic; + static COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::new(1); + Self(COUNTER.fetch_add(1, atomic::Ordering::Relaxed)) + } +} + +impl AsRef for AtomicId { + fn as_ref(&self) -> &usize { + &self.0 + } +} + +impl AsMut for AtomicId { + fn as_mut(&mut self) -> &mut usize { + &mut self.0 + } +} + +impl Default for AtomicId { + fn default() -> Self { + Self::new() + } +} + +impl std::fmt::Display for AtomicId { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl From for AtomicId { + fn from(id: usize) -> Self { + Self(id) + } +} + +impl From for usize { + fn from(id: AtomicId) -> Self { + id.0 + } +} diff --git a/core/src/id/ids/mod.rs b/core/src/id/ids/mod.rs new file mode 100644 index 00000000..31e6d967 --- /dev/null +++ b/core/src/id/ids/mod.rs @@ -0,0 +1,8 @@ +/* + Appellation: ids + Contrib: FL03 +*/ +//! # ids +pub use self::atomic::*; + +pub(crate) mod atomic; diff --git a/core/src/id/mod.rs b/core/src/id/mod.rs index dbe3ce71..592c3677 100644 --- a/core/src/id/mod.rs +++ b/core/src/id/mod.rs @@ -3,11 +3,18 @@ Contrib: FL03 */ //! # id -pub use self::{identity::*, utils::*}; +pub use self::{identity::*, ids::*, utils::*}; pub(crate) mod identity; +pub(crate) mod ids; pub(crate) mod utils { + // https://users.rust-lang.org/t/idiomatic-rust-way-to-generate-unique-id/33805 + pub fn atomic_id() -> usize { + use std::sync::atomic; + static COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::new(0); + COUNTER.fetch_add(1, atomic::Ordering::Relaxed) + } pub fn rid(length: usize) -> String { use rand::distributions::Alphanumeric; @@ -22,4 +29,20 @@ pub(crate) mod utils { } #[cfg(test)] -mod tests {} +mod tests { + + use super::*; + + #[test] + fn test_atomic_id() { + let id = atomic_id(); + assert_eq!(id, 0); + assert_ne!(id, atomic_id()); + } + + #[test] + fn test_rid() { + let id = rid(10); + assert_eq!(id.len(), 10); + } +} diff --git a/core/src/params/iter.rs b/core/src/params/iter.rs index 359553ef..7af19abb 100644 --- a/core/src/params/iter.rs +++ b/core/src/params/iter.rs @@ -3,6 +3,13 @@ Contrib: FL03 */ +pub struct Entry { + key: K, + value: V, +} + +pub struct IntoIter; + pub struct Iter; pub struct IterMut; diff --git a/core/src/params/kinds.rs b/core/src/params/kinds.rs index 660fa41b..6c8a7f84 100644 --- a/core/src/params/kinds.rs +++ b/core/src/params/kinds.rs @@ -9,6 +9,15 @@ pub trait ParamType: ToString { fn kind(&self) -> String; } +impl ParamType for T +where + T: ToString, +{ + fn kind(&self) -> String { + self.to_string() + } +} + #[derive( Clone, Debug, diff --git a/core/src/params/mod.rs b/core/src/params/mod.rs index 2d4f3a9d..d2c2d211 100644 --- a/core/src/params/mod.rs +++ b/core/src/params/mod.rs @@ -19,6 +19,8 @@ use num::Float; use std::collections::HashMap; pub trait Param { + type Dim: Dimension; + fn kind(&self) -> &ParamKind; fn name(&self) -> &str; diff --git a/core/src/params/param.rs b/core/src/params/param.rs index 36e91b74..f10c79c1 100644 --- a/core/src/params/param.rs +++ b/core/src/params/param.rs @@ -113,6 +113,8 @@ where T: Float, D: Dimension, { + type Dim = D; + fn kind(&self) -> &ParamKind { &self.kind } diff --git a/core/src/primitives.rs b/core/src/primitives.rs index 920c63cc..7c12ff9e 100644 --- a/core/src/primitives.rs +++ b/core/src/primitives.rs @@ -18,7 +18,7 @@ mod statics {} /// Collection of types used throughout the system mod types { /// - pub type BoxError = Box; + pub type BoxError = Box; /// pub type BoxResult = std::result::Result; diff --git a/core/src/specs/arrays.rs b/core/src/specs/arrays.rs index 3531bdec..a201609e 100644 --- a/core/src/specs/arrays.rs +++ b/core/src/specs/arrays.rs @@ -11,6 +11,7 @@ use ndarray_rand::rand_distr as distr; use ndarray_rand::RandomExt; use num::{Float, Num}; +use std::ops; pub trait Affine: Sized { type Error; @@ -18,15 +19,16 @@ pub trait Affine: Sized { fn affine(&self, mul: T, add: T) -> Result; } -impl Affine for Array +impl Affine for Array where T: Num + ScalarOperand, D: Dimension, + Array: ops::Mul> + ops::Add>, { type Error = ShapeError; - fn affine(&self, mul: T, add: T) -> Result { - Ok(self * mul + add) + fn affine(&self, mul: S, add: S) -> Result { + Ok(self.clone() * mul + add) } } diff --git a/core/src/specs/math.rs b/core/src/specs/math.rs index 2ab59c89..e56eac62 100644 --- a/core/src/specs/math.rs +++ b/core/src/specs/math.rs @@ -50,24 +50,22 @@ pub trait FloatExt: FromPrimitive + NdFloat + Signed + SampleUniform {} impl FloatExt for T where T: FromPrimitive + NdFloat + Signed + SampleUniform {} -pub trait Arithmetic: - ops::Add - + ops::Div - + ops::Mul - + ops::Sub -{ +pub trait Arithmetic: ops::Add + ops::Div + ops::Mul + ops::Sub { + type Output; } -impl Arithmetic for A where +impl Arithmetic for A +where A: ops::Add + ops::Div + ops::Mul - + ops::Sub + + ops::Sub, { + type Output = T; } pub trait MatrixOps: - Arithmetic, Array> + Sized + Arithmetic, Output = Array> + Sized where A: Dimension, B: Dimension, @@ -79,8 +77,8 @@ where A: Dimension, B: Dimension, D: Dimension, - T: Float, - Self: Arithmetic, Array>, + T: Arithmetic, + Self: Arithmetic, Output = Array>, { } @@ -89,8 +87,8 @@ where A: Dimension, B: Dimension, D: Dimension, - T: Float, - Self: Arithmetic, Array>, + T: Arithmetic, + Self: Arithmetic, Output = Array>, { } diff --git a/core/src/specs/mod.rs b/core/src/specs/mod.rs index 3407cec3..390b602e 100644 --- a/core/src/specs/mod.rs +++ b/core/src/specs/mod.rs @@ -35,6 +35,10 @@ where } } +pub trait Named { + fn name(&self) -> &str; +} + pub trait RoundTo { fn round_to(&self, places: usize) -> Self; } diff --git a/data/Cargo.toml b/data/Cargo.toml index 3fdce024..c9f426b1 100644 --- a/data/Cargo.toml +++ b/data/Cargo.toml @@ -15,7 +15,6 @@ version.workspace = true default = [] blas = [ - "linfa/blas", "ndarray/blas", ] @@ -54,8 +53,8 @@ test = true [build-dependencies] [dependencies] +concision-core = { path = "../core", version = "0.1.12" } anyhow.workspace = true -linfa = { features = ["serde"], version = "0.7" } ndarray = { features = ["serde-1"], version = "0.15" } num.workspace = true serde.workspace = true diff --git a/data/src/datasets/mod.rs b/data/src/datasets/mod.rs index 00597a07..95041bce 100644 --- a/data/src/datasets/mod.rs +++ b/data/src/datasets/mod.rs @@ -3,9 +3,10 @@ Contrib: FL03 */ //! # Dataset -pub use self::{dataset::*, group::*, utils::*}; +pub use self::{dataset::*, group::*}; pub(crate) mod dataset; pub(crate) mod group; -pub(crate) mod utils {} +#[cfg(test)] +mod tests {} diff --git a/data/src/flows/flow.rs b/data/src/flows/flow.rs index 07f5229b..a64d001e 100644 --- a/data/src/flows/flow.rs +++ b/data/src/flows/flow.rs @@ -2,22 +2,7 @@ Appellation: flow Contrib: FL03 */ -use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] -#[serde(rename_all = "lowercase")] -pub struct Flow { - data: Vec, -} - -impl Flow { - pub fn new() -> Self { - Self { data: Vec::new() } - } -} - -impl std::fmt::Display for Flow { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{}", serde_json::to_string(self).unwrap()) - } +pub trait Flow { + fn flow(&self, input: T) -> T; } diff --git a/data/src/flows/mod.rs b/data/src/flows/mod.rs index 08de4074..4b044153 100644 --- a/data/src/flows/mod.rs +++ b/data/src/flows/mod.rs @@ -3,9 +3,10 @@ Contrib: FL03 */ //! # Flows -pub use self::{direction::*, flow::*, utils::*}; +pub use self::{direction::*, flow::*}; pub(crate) mod direction; pub(crate) mod flow; -pub(crate) mod utils {} +#[cfg(test)] +mod tests {} diff --git a/data/src/lib.rs b/data/src/lib.rs index 2e5ae7ee..85a723da 100644 --- a/data/src/lib.rs +++ b/data/src/lib.rs @@ -16,17 +16,23 @@ pub(crate) mod utils; pub mod datasets; pub mod df; pub mod flows; +pub mod shape; +pub mod store; pub mod tensors; +pub(crate) use concision_core as core; + pub mod prelude { // pub use linfa::dataset::{Dataset, DatasetBase, DatasetView}; + pub use crate::primitives::*; + pub use crate::specs::*; + pub use crate::utils::*; + pub use crate::datasets::*; pub use crate::df::*; pub use crate::flows::*; + pub use crate::shape::*; + pub use crate::store::*; pub use crate::tensors::*; - - pub use crate::primitives::*; - pub use crate::specs::*; - pub use crate::utils::*; } diff --git a/data/src/misc/dtype.rs b/data/src/misc/dtype.rs index 1a176c44..15c3e4ac 100644 --- a/data/src/misc/dtype.rs +++ b/data/src/misc/dtype.rs @@ -3,9 +3,74 @@ Contrib: FL03 */ -pub enum DType {} +pub trait DataType { + fn dtype(&self) -> DType; +} + +impl DataType for T +where + T: Clone + Into, +{ + fn dtype(&self) -> DType { + self.clone().into() + } +} + +pub enum DType { + FloatingPoint(FloatingPoint), + Integer(Integer), + Unsigned(Unsigned), +} + +impl From for DType { + fn from(_: f32) -> Self { + DType::FloatingPoint(FloatingPoint::F32) + } +} + +impl From for DType { + fn from(_: f64) -> Self { + DType::FloatingPoint(FloatingPoint::F64) + } +} pub enum FloatingPoint { F32, F64, } + +impl From for FloatingPoint { + fn from(_: f32) -> Self { + FloatingPoint::F32 + } +} + +impl From for FloatingPoint { + fn from(_: f64) -> Self { + FloatingPoint::F64 + } +} + +impl From for DType { + fn from(dtype: FloatingPoint) -> Self { + DType::FloatingPoint(dtype) + } +} + +pub enum Integer { + I8, + I16, + I32, + I64, + I128, + ISIZE, +} + +pub enum Unsigned { + U8, + U16, + U32, + U64, + U128, + USIZE, +} diff --git a/data/src/shape/dimension.rs b/data/src/shape/dimension.rs new file mode 100644 index 00000000..f7c016bd --- /dev/null +++ b/data/src/shape/dimension.rs @@ -0,0 +1,6 @@ +/* + Appellation: dimension + Contrib: FL03 +*/ + +pub trait Dimension {} diff --git a/data/src/shape/mod.rs b/data/src/shape/mod.rs new file mode 100644 index 00000000..d250c40b --- /dev/null +++ b/data/src/shape/mod.rs @@ -0,0 +1,11 @@ +/* + Appellation: shapes + Contrib: FL03 +*/ +//! # Shapes +pub use self::dimension::*; + +pub(crate) mod dimension; + +#[cfg(test)] +mod tests {} diff --git a/data/src/specs.rs b/data/src/specs.rs index b6d9d9d7..b9598307 100644 --- a/data/src/specs.rs +++ b/data/src/specs.rs @@ -12,7 +12,7 @@ pub trait Records { impl Records for Array1 { fn features(&self) -> usize { - self.shape()[1] + 1 } fn samples(&self) -> usize { diff --git a/data/src/store/mod.rs b/data/src/store/mod.rs new file mode 100644 index 00000000..dd17be4f --- /dev/null +++ b/data/src/store/mod.rs @@ -0,0 +1,13 @@ +/* + Appellation: store + Contrib: FL03 +*/ +//! # Store +pub use self::storage::*; + +pub(crate) mod storage; + +pub trait Store {} + +#[cfg(test)] +mod tests {} diff --git a/data/src/store/storage.rs b/data/src/store/storage.rs new file mode 100644 index 00000000..026e49f8 --- /dev/null +++ b/data/src/store/storage.rs @@ -0,0 +1,6 @@ +/* + Appellation: storage + Contrib: FL03 +*/ + +pub struct Storage {} diff --git a/data/src/tensors/tensor.rs b/data/src/tensors/tensor.rs index af399da4..d8097141 100644 --- a/data/src/tensors/tensor.rs +++ b/data/src/tensors/tensor.rs @@ -2,6 +2,7 @@ Appellation: tensor Contrib: FL03 */ +use crate::core::id::AtomicId; use ndarray::prelude::{Array, Dimension, Ix2}; use ndarray::IntoDimension; use num::Float; @@ -14,6 +15,7 @@ where D: Dimension, T: Float, { + id: AtomicId, data: Array, } @@ -24,6 +26,7 @@ where { pub fn new(shape: impl IntoDimension) -> Self { Self { + id: AtomicId::new(), data: Array::zeros(shape), } } diff --git a/ml/linear/src/model/module.rs b/ml/linear/src/model/module.rs index 4fafa7cc..30aded23 100644 --- a/ml/linear/src/model/module.rs +++ b/ml/linear/src/model/module.rs @@ -41,22 +41,22 @@ where } } -impl Module for LinearModel -where - T: NdFloat, -{ - fn name(&self) -> &str { - "LinearModel" - } - - fn parameters(&self) -> &ModuleParams { - &self.params - } - - fn parameters_mut(&mut self) -> &mut ModuleParams { - &mut self.params - } -} +// impl Module> for LinearModel> +// where +// T: NdFloat, +// { +// fn name(&self) -> &str { +// "LinearModel" +// } + +// fn parameters(&self) -> &ModuleParams { +// &self.params +// } + +// fn parameters_mut(&mut self) -> &mut ModuleParams { +// &mut self.params +// } +// } impl Biased for LinearModel where @@ -96,23 +96,6 @@ where } } -impl crate::neural::prelude::Weighted for LinearModel -where - T: NdFloat, -{ - fn weights(&self) -> &Array2 { - &self.params["weight"] - } - - fn weights_mut(&mut self) -> &mut Array2 { - self.params.get_mut("weight").unwrap() - } - - fn set_weights(&mut self, weights: Array2) { - self.params.insert("weight".to_string(), weights); - } -} - impl Forward> for LinearModel where T: NdFloat, diff --git a/ml/linear/src/params/store.rs b/ml/linear/src/params/store.rs index 84127685..23efc53e 100644 --- a/ml/linear/src/params/store.rs +++ b/ml/linear/src/params/store.rs @@ -4,10 +4,9 @@ */ use super::LayerShape; use crate::core::prelude::GenerateRandom; -// use crate::core::params::{Biased, Weighted}; -use crate::neural::prelude::{Biased, Features, Forward, Node, Weighted}; +use crate::neural::prelude::{Features, Forward, Node}; use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Array1, Array2, Axis, Dimension, Ix2, NdFloat}; +use ndarray::prelude::{Array, Array1, Array2, Axis, Dimension, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use num::Float; @@ -15,24 +14,33 @@ use serde::{Deserialize, Serialize}; use std::ops; #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub struct LayerParams { - bias: Array1, +pub struct LinearParams { + bias: Option>, pub features: LayerShape, weights: Array2, } -impl LayerParams +impl LinearParams where T: Float, { - pub fn new(features: LayerShape) -> Self { + pub fn new(biased: bool, features: LayerShape) -> Self { + let bias = if biased { + Some(Array1::zeros(features.outputs())) + } else { + None + }; Self { - bias: Array1::zeros(features.outputs()), + bias, features, weights: Array2::zeros(features.out_by_in()), } } + pub fn biased(features: LayerShape) -> Self { + Self::new(true, features) + } + pub fn features(&self) -> &LayerShape { &self.features } @@ -41,17 +49,30 @@ where &mut self.features } + pub fn is_biased(&self) -> bool { + self.bias.is_some() + } + pub fn set_node(&mut self, idx: usize, node: Node) { - self.bias_mut() - .index_axis_mut(Axis(0), idx) - .assign(&node.bias()); + if let Some(bias) = node.bias() { + if !self.is_biased() { + let mut tmp = Array1::zeros(self.features().outputs()); + tmp.index_axis_mut(Axis(0), idx).assign(bias); + self.bias = Some(tmp); + } + self.bias + .as_mut() + .unwrap() + .index_axis_mut(Axis(0), idx) + .assign(bias); + } self.weights_mut() .index_axis_mut(Axis(0), idx) .assign(&node.weights()); } - pub fn with_bias(mut self, bias: Array1) -> Self { + pub fn with_bias(mut self, bias: Option>) -> Self { self.bias = bias; self } @@ -60,9 +81,33 @@ where self.weights = weights; self } + + pub fn bias(&self) -> Option<&Array1> { + self.bias.as_ref() + } + + pub fn bias_mut(&mut self) -> Option<&mut Array1> { + self.bias.as_mut() + } + + pub fn set_bias(&mut self, bias: Option>) { + self.bias = bias; + } + + pub fn set_weights(&mut self, weights: Array2) { + self.weights = weights; + } + + pub fn weights(&self) -> &Array2 { + &self.weights + } + + pub fn weights_mut(&mut self) -> &mut Array2 { + &mut self.weights + } } -impl LayerParams +impl LinearParams where T: Float + 'static, { @@ -71,17 +116,19 @@ where } } -impl LayerParams +impl LinearParams where T: NdFloat, { pub fn reset(&mut self) { - self.bias *= T::zero(); + if let Some(bias) = self.bias_mut() { + *bias = Array1::zeros(bias.dim()); + } self.weights *= T::zero(); } } -impl LayerParams +impl LinearParams where T: Float + SampleUniform, StandardNormal: Distribution, @@ -95,7 +142,7 @@ where pub fn init_bias(mut self) -> Self { let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); - self.bias = Array1::uniform_between(dk, self.features().outputs()); + self.bias = Some(Array1::uniform_between(dk, self.features().outputs())); self } @@ -106,41 +153,7 @@ where } } -impl Biased for LayerParams -where - T: Float, -{ - fn bias(&self) -> &Array1 { - &self.bias - } - - fn bias_mut(&mut self) -> &mut Array1 { - &mut self.bias - } - - fn set_bias(&mut self, bias: Array1) { - self.bias = bias; - } -} - -impl Weighted for LayerParams -where - T: Float, -{ - fn set_weights(&mut self, weights: Array2) { - self.weights = weights; - } - - fn weights(&self) -> &Array2 { - &self.weights - } - - fn weights_mut(&mut self) -> &mut Array2 { - &mut self.weights - } -} - -impl Features for LayerParams +impl Features for LinearParams where T: Float, { @@ -153,7 +166,7 @@ where } } -impl Forward> for LayerParams +impl Forward> for LinearParams where D: Dimension, T: NdFloat, @@ -162,11 +175,15 @@ where type Output = Array; fn forward(&self, input: &Array) -> Self::Output { - input.dot(&self.weights().t().to_owned()) + self.bias().clone() + let w = self.weights().t().to_owned(); + if let Some(bias) = self.bias() { + return input.dot(&w) + bias.clone(); + } + input.dot(&w) } } -impl IntoIterator for LayerParams +impl IntoIterator for LinearParams where T: Float, { @@ -174,16 +191,24 @@ where type IntoIter = std::vec::IntoIter; fn into_iter(self) -> Self::IntoIter { + if let Some(bias) = self.bias() { + return self + .weights() + .axis_iter(Axis(0)) + .zip(bias.axis_iter(Axis(0))) + .map(|(w, b)| (w.to_owned(), b.to_owned()).into()) + .collect::>() + .into_iter(); + } self.weights() .axis_iter(Axis(0)) - .zip(self.bias().axis_iter(Axis(0))) - .map(|(w, b)| (w.to_owned(), b.to_owned()).into()) + .map(|w| (w.to_owned(), None).into()) .collect::>() .into_iter() } } -impl FromIterator> for LayerParams +impl FromIterator> for LinearParams where T: Float, { @@ -191,8 +216,8 @@ where let nodes = nodes.into_iter().collect::>(); let mut iter = nodes.iter(); let node = iter.next().unwrap(); - let shape = LayerShape::new(*node.features(), nodes.len()); - let mut params = LayerParams::new(shape); + let shape = LayerShape::new(node.features(), nodes.len()); + let mut params = Self::new(true, shape); params.set_node(0, node.clone()); for (i, node) in iter.into_iter().enumerate() { params.set_node(i + 1, node.clone()); diff --git a/ml/neural/Cargo.toml b/ml/neural/Cargo.toml index d70fd817..14022756 100644 --- a/ml/neural/Cargo.toml +++ b/ml/neural/Cargo.toml @@ -32,6 +32,7 @@ test = true concision-core = { path = "../../core", version = "0.1.12" } anyhow.workspace = true +itertools.workspace = true ndarray = { features = ["serde-1"], version = "0.15" } ndarray-rand.workspace = true ndarray-stats.workspace = true diff --git a/ml/neural/src/layers/exp/layer.rs b/ml/neural/src/layers/exp/layer.rs deleted file mode 100644 index b2724850..00000000 --- a/ml/neural/src/layers/exp/layer.rs +++ /dev/null @@ -1,180 +0,0 @@ -/* - Appellation: model - Contrib: FL03 -*/ -use super::LayerConfig; -use crate::func::activate::{Activate, Activator}; -use crate::layers::LayerShape; -use crate::prelude::{Forward, ParamGroup, Parameterized, Params}; -use ndarray::prelude::{Array2, Dimension, Ix1, Ix2, NdFloat}; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::{Distribution, StandardNormal}; -use num::Float; - -pub struct Layer -where - D: Dimension, - T: Float, -{ - activator: Activator, - config: LayerConfig, - params: ParamGroup, -} - -impl Layer -where - T: Float, -{ - pub fn new(activator: impl Activate + 'static, config: LayerConfig) -> Self { - let params = ParamGroup::new(*config.features()); - Self { - activator: Activator::new(Box::new(activator)), - config, - params, - } - } - - pub fn activator(&self) -> &Activator { - &self.activator - } - - pub fn config(&self) -> &LayerConfig { - &self.config - } - - pub fn config_mut(&mut self) -> &mut LayerConfig { - &mut self.config - } - - // pub fn set_node(&mut self, idx: usize, node: &Node) { - // self.params - // .weights_mut() - // .slice_mut(s![idx, ..]) - // .assign(&node.weights()); - // } - - // pub fn validate_layer(&self, other: &Self, next: bool) -> bool { - // if next { - // return self.features().inputs() == other.features().outputs(); - // } - // self.features().outputs() == other.features().inputs() - // } -} - -impl Layer -where - T: Float + 'static, -{ - pub fn apply_gradient(&mut self, gamma: T, gradient: F) - where - F: Fn(&Array2) -> Array2, - { - let grad = gradient(&self.params.weights()); - self.params.weights_mut().scaled_add(-gamma, &grad); - } - - pub fn update_with_gradient(&mut self, gamma: T, grad: &Array2) { - self.params.weights_mut().scaled_add(-gamma, grad); - } -} - -impl Layer -where - T: NdFloat, -{ - pub fn linear(&self, args: &Array2) -> Array2 { - args.dot(&self.params.weights().t()) + self.params.bias() - } -} - -impl Layer -where - T: Float + SampleUniform, - StandardNormal: Distribution, -{ - pub fn init(mut self, biased: bool) -> Self { - self.params = self.params.init(biased); - self - } -} - -impl Forward> for Layer -where - T: NdFloat, -{ - type Output = Array2; - - fn forward(&self, args: &Array2) -> Self::Output { - self.activator.activate(&self.linear(args)) - } -} - -impl Parameterized for Layer -where - D: Dimension, - T: Float, -{ - type Features = D; - type Params = ParamGroup; - - fn features(&self) -> &D { - self.params().features() - } - - fn features_mut(&mut self) -> &mut D { - self.params_mut().features_mut() - } - - fn params(&self) -> &Self::Params { - &self.params - } - - fn params_mut(&mut self) -> &mut Self::Params { - &mut self.params - } -} - -// impl PartialOrd for Layer -// where -// A: Activate + PartialEq, -// T: Float, -// { -// fn partial_cmp(&self, other: &Self) -> Option { -// self.position.partial_cmp(&other.position) -// } -// } - -impl From for Layer -where - T: Float + 'static, -{ - fn from(features: LayerShape) -> Self { - Self::new(Activator::linear(), features.into()) - } -} - -// impl IntoIterator for Layer -// where -// T: Float, -// { -// type Item = Node; -// type IntoIter = std::vec::IntoIter; - -// fn into_iter(self) -> Self::IntoIter { -// self.params.into_iter() -// } -// } - -// impl FromIterator> for Layer -// where -// T: Float, -// { -// fn from_iter>>(nodes: I) -> Self { -// let params = LayerParams::from_iter(nodes); -// Self { -// activator: Activator::linear(), -// config: LayerConfig::from(*params.features()), -// params, -// } -// } -// } diff --git a/ml/neural/src/layers/exp/mod.rs b/ml/neural/src/layers/exp/mod.rs index 89669e8b..444368fc 100644 --- a/ml/neural/src/layers/exp/mod.rs +++ b/ml/neural/src/layers/exp/mod.rs @@ -3,10 +3,9 @@ Contrib: FL03 */ //! # Experimental Layers -pub use self::{config::*, layer::*, sublayer::*, wrapper::*}; +pub use self::{config::*, sublayer::*, wrapper::*}; pub(crate) mod config; -pub(crate) mod layer; pub(crate) mod sublayer; pub(crate) mod wrapper; diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 17c8996e..02edb0b6 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -4,7 +4,7 @@ */ use super::{LayerParams, LayerShape}; use crate::func::activate::{Activate, Gradient, Linear}; -use crate::prelude::{Features, Forward, Node, Parameterized, Params, Perceptron}; +use crate::prelude::{Features, Forward, Node, Perceptron}; use ndarray::prelude::{Array2, Ix1, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, StandardNormal}; @@ -58,10 +58,26 @@ where &self.activator } + pub fn features(&self) -> &LayerShape { + &self.features + } + + pub fn features_mut(&mut self) -> &mut LayerShape { + &mut self.features + } + pub fn name(&self) -> &str { &self.name } + pub fn params(&self) -> &LayerParams { + &self.params + } + + pub fn params_mut(&mut self) -> &mut LayerParams { + &mut self.params + } + pub fn set_name(&mut self, name: impl ToString) { self.name = name.to_string(); } @@ -205,31 +221,6 @@ where } } -impl Parameterized for Layer -where - A: Activate, - T: Float, -{ - type Features = LayerShape; - type Params = LayerParams; - - fn features(&self) -> &LayerShape { - &self.features - } - - fn features_mut(&mut self) -> &mut LayerShape { - &mut self.features - } - - fn params(&self) -> &LayerParams { - &self.params - } - - fn params_mut(&mut self) -> &mut LayerParams { - &mut self.params - } -} - // impl PartialOrd for Layer // where // A: Activate + PartialEq, diff --git a/ml/neural/src/layers/mod.rs b/ml/neural/src/layers/mod.rs index cf28b291..5a43db19 100644 --- a/ml/neural/src/layers/mod.rs +++ b/ml/neural/src/layers/mod.rs @@ -11,6 +11,7 @@ pub(crate) mod params; pub(crate) mod stack; pub mod exp; +pub mod seq; use crate::prelude::{Activate, ActivateDyn, Forward, Node}; use ndarray::prelude::{Array2, Ix2}; @@ -74,8 +75,7 @@ pub(crate) mod utils { mod tests { use super::*; use crate::core::prelude::linarr; - use crate::func::activate::Softmax; - use crate::prelude::{Biased, Forward, Node, Parameterized}; + use crate::prelude::{Forward, Node, Softmax}; use ndarray::prelude::Ix2; #[test] @@ -106,8 +106,9 @@ mod tests { let layer = Layer::::from(features).init(true); for node in layer.into_iter() { - assert_eq!(node.features(), &inputs); - assert_eq!(node.bias().dim(), ()); + assert!(node.is_biased()); + assert_eq!(node.features(), inputs); + assert_eq!(node.bias().as_ref().unwrap().dim(), ()); } } } diff --git a/ml/neural/src/layers/params.rs b/ml/neural/src/layers/params.rs index d50917b3..dab9a47b 100644 --- a/ml/neural/src/layers/params.rs +++ b/ml/neural/src/layers/params.rs @@ -4,7 +4,7 @@ */ use super::LayerShape; use crate::core::prelude::GenerateRandom; -use crate::prelude::{Biased, Features, Forward, Node, Weighted}; +use crate::prelude::{Features, Forward, Node}; use ndarray::linalg::Dot; use ndarray::prelude::{Array, Array1, Array2, Axis, Dimension, Ix2, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; @@ -15,7 +15,7 @@ use std::ops; #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct LayerParams { - bias: Array1, + bias: Option>, pub features: LayerShape, weights: Array2, } @@ -25,13 +25,34 @@ where T: Float, { pub fn new(features: LayerShape) -> Self { + Self::create(false, features) + } + + pub fn biased(features: LayerShape) -> Self { + Self::create(true, features) + } + + pub fn create(biased: bool, features: LayerShape) -> Self { + let bias = if biased { + Some(Array1::zeros(features.outputs())) + } else { + None + }; Self { - bias: Array1::zeros(features.outputs()), + bias, features, weights: Array2::zeros(features.out_by_in()), } } + pub fn bias(&self) -> &Option> { + &self.bias + } + + pub fn bias_mut(&mut self) -> &mut Option> { + &mut self.bias + } + pub fn features(&self) -> &LayerShape { &self.features } @@ -40,17 +61,46 @@ where &mut self.features } + pub fn is_biased(&self) -> bool { + self.bias.is_some() + } + + pub fn set_bias(&mut self, bias: Option>) { + self.bias = bias; + } + pub fn set_node(&mut self, idx: usize, node: Node) { - self.bias_mut() - .index_axis_mut(Axis(0), idx) - .assign(&node.bias()); + if let Some(bias) = node.bias() { + if !self.is_biased() { + let mut tmp = Array1::zeros(self.features().outputs()); + tmp.index_axis_mut(Axis(0), idx).assign(bias); + self.bias = Some(tmp); + } + self.bias + .as_mut() + .unwrap() + .index_axis_mut(Axis(0), idx) + .assign(bias); + } self.weights_mut() .index_axis_mut(Axis(0), idx) .assign(&node.weights()); } - pub fn with_bias(mut self, bias: Array1) -> Self { + pub fn set_weights(&mut self, weights: Array2) { + self.weights = weights; + } + + pub fn weights(&self) -> &Array2 { + &self.weights + } + + pub fn weights_mut(&mut self) -> &mut Array2 { + &mut self.weights + } + + pub fn with_bias(mut self, bias: Option>) -> Self { self.bias = bias; self } @@ -75,7 +125,9 @@ where T: NdFloat, { pub fn reset(&mut self) { - self.bias *= T::zero(); + if let Some(bias) = self.bias() { + self.bias = Some(Array1::zeros(bias.dim())); + } self.weights *= T::zero(); } } @@ -94,7 +146,7 @@ where pub fn init_bias(mut self) -> Self { let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); - self.bias = Array1::uniform_between(dk, self.features().outputs()); + self.bias = Some(Array1::uniform_between(dk, self.features().outputs())); self } @@ -105,40 +157,6 @@ where } } -impl Biased for LayerParams -where - T: Float, -{ - fn bias(&self) -> &Array1 { - &self.bias - } - - fn bias_mut(&mut self) -> &mut Array1 { - &mut self.bias - } - - fn set_bias(&mut self, bias: Array1) { - self.bias = bias; - } -} - -impl Weighted for LayerParams -where - T: Float, -{ - fn set_weights(&mut self, weights: Array2) { - self.weights = weights; - } - - fn weights(&self) -> &Array2 { - &self.weights - } - - fn weights_mut(&mut self) -> &mut Array2 { - &mut self.weights - } -} - impl Features for LayerParams where T: Float, @@ -161,7 +179,11 @@ where type Output = Array; fn forward(&self, input: &Array) -> Self::Output { - input.dot(&self.weights().t().to_owned()) + self.bias().clone() + let w = self.weights().t().to_owned(); + if let Some(bias) = self.bias() { + return input.dot(&w) + bias.clone(); + } + input.dot(&w) } } @@ -173,10 +195,18 @@ where type IntoIter = std::vec::IntoIter; fn into_iter(self) -> Self::IntoIter { + if let Some(bias) = self.bias() { + return self + .weights() + .axis_iter(Axis(0)) + .zip(bias.axis_iter(Axis(0))) + .map(|(w, b)| (w.to_owned(), b.to_owned()).into()) + .collect::>() + .into_iter(); + } self.weights() .axis_iter(Axis(0)) - .zip(self.bias().axis_iter(Axis(0))) - .map(|(w, b)| (w.to_owned(), b.to_owned()).into()) + .map(|w| (w.to_owned(), None).into()) .collect::>() .into_iter() } @@ -190,8 +220,8 @@ where let nodes = nodes.into_iter().collect::>(); let mut iter = nodes.iter(); let node = iter.next().unwrap(); - let shape = LayerShape::new(*node.features(), nodes.len()); - let mut params = LayerParams::new(shape); + let shape = LayerShape::new(node.features(), nodes.len()); + let mut params = LayerParams::create(true, shape); params.set_node(0, node.clone()); for (i, node) in iter.into_iter().enumerate() { params.set_node(i + 1, node.clone()); diff --git a/ml/neural/src/layers/seq/mod.rs b/ml/neural/src/layers/seq/mod.rs new file mode 100644 index 00000000..74a592d6 --- /dev/null +++ b/ml/neural/src/layers/seq/mod.rs @@ -0,0 +1,11 @@ +/* + Appellation: seq + Contrib: FL03 +*/ + +pub use self::sequential::*; + +pub(crate) mod sequential; + +#[cfg(test)] +mod tests {} diff --git a/ml/neural/src/layers/seq/sequential.rs b/ml/neural/src/layers/seq/sequential.rs new file mode 100644 index 00000000..1a50b186 --- /dev/null +++ b/ml/neural/src/layers/seq/sequential.rs @@ -0,0 +1,6 @@ +/* + Appellation: sequential + Contrib: FL03 +*/ + +pub struct Sequential {} diff --git a/ml/neural/src/layers/stack.rs b/ml/neural/src/layers/stack.rs index 60eb1a41..6fbaedef 100644 --- a/ml/neural/src/layers/stack.rs +++ b/ml/neural/src/layers/stack.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use crate::layers::{Layer, LayerShape}; -use crate::prelude::{Activate, Features, Linear, Parameterized}; +use crate::prelude::{Activate, Features, Linear}; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use num::Float; diff --git a/ml/neural/src/models/exp/mod.rs b/ml/neural/src/models/exp/mod.rs index e60e5f74..c73329b1 100644 --- a/ml/neural/src/models/exp/mod.rs +++ b/ml/neural/src/models/exp/mod.rs @@ -8,31 +8,31 @@ pub use self::{modules::*, store::*}; pub(crate) mod modules; pub(crate) mod store; -use crate::prelude::Forward; -use ndarray::prelude::Array2; -use num::Float; +// use crate::prelude::Predict; +// use ndarray::prelude::Array2; +// use num::Float; -pub trait Model: Forward> -where - T: Float, -{ - type Config; +// pub trait Model: Predict> +// where +// T: Float, +// { +// type Config; - fn name(&self) -> &str; +// fn name(&self) -> &str; - fn modules(&self) -> &Vec>>; +// fn modules(&self) -> &Vec>>; - fn modules_mut(&mut self) -> &mut Vec>>; +// fn modules_mut(&mut self) -> &mut Vec>>; - fn register_module(&mut self, module: Box>) -> &mut Self { - self.modules_mut().push(module); - self - } +// fn register_module(&mut self, module: Box>) -> &mut Self { +// self.modules_mut().push(module); +// self +// } - fn get_module(&self, name: &str) -> Option<&Box>> { - self.modules().iter().find(|m| m.name() == name) - } -} +// fn get_module(&self, name: &str) -> Option<&Box>> { +// self.modules().iter().find(|m| m.name() == name) +// } +// } #[cfg(test)] mod tests {} diff --git a/ml/neural/src/models/exp/modules.rs b/ml/neural/src/models/exp/modules.rs index 64cddbea..21071fc1 100644 --- a/ml/neural/src/models/exp/modules.rs +++ b/ml/neural/src/models/exp/modules.rs @@ -4,32 +4,29 @@ */ //! # Model //! -use crate::prelude::Forward; +use crate::prelude::Predict; use ndarray::prelude::Array2; -use num::Float; use std::collections::HashMap; pub type ModuleParams = HashMap>; -pub struct M(Box>>); +// pub struct M(Box>); -pub trait Module: Forward> -where - T: Float, -{ +pub trait Store { + fn get(&self, name: &str) -> Option<&V>; + fn get_mut(&mut self, name: &str) -> Option<&mut V>; + fn insert(&mut self, name: K, value: V) -> Option; + fn remove(&mut self, name: &str) -> Option; +} + +pub trait Module: Predict> { fn get_param(&self, name: &str) -> Option<&Array2> { self.parameters().get(name) } fn name(&self) -> &str; - fn parameters(&self) -> &ModuleParams; - - fn parameters_mut(&mut self) -> &mut ModuleParams; -} + fn parameters(&self) -> &HashMap>; -pub trait ModuleExt: Module -where - T: Float, -{ + fn parameters_mut(&mut self) -> &mut HashMap>; } diff --git a/ml/neural/src/models/model.rs b/ml/neural/src/models/model.rs index edc49384..025e11e6 100644 --- a/ml/neural/src/models/model.rs +++ b/ml/neural/src/models/model.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use super::{ModelConfig, ModelParams}; -use crate::prelude::{Forward, Gradient, LayerParams, Weighted}; +use crate::prelude::{Forward, Gradient, LayerParams}; use ndarray::linalg::Dot; use ndarray::prelude::{Array, Array1, Array2, Dimension, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; diff --git a/ml/neural/src/neurons/node.rs b/ml/neural/src/neurons/node.rs index 1f9dd4bc..26885bb7 100644 --- a/ml/neural/src/neurons/node.rs +++ b/ml/neural/src/neurons/node.rs @@ -4,21 +4,18 @@ */ use crate::core::prelude::GenerateRandom; -use crate::prelude::{Biased, Forward, Weighted}; +use crate::prelude::Forward; use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Array0, Array1, Array2, Dimension, Ix1, NdFloat}; -use ndarray::RemoveAxis; +use ndarray::prelude::{Array, Array0, Array1, Array2, Dimension, NdFloat}; +use ndarray::{RemoveAxis, ScalarOperand}; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, StandardNormal}; -use num::{Float, FromPrimitive}; +use num::Float; use std::ops; #[derive(Clone, Debug, PartialEq)] -pub struct Node -where - T: Float, -{ - bias: Array0, +pub struct Node { + bias: Option>, features: usize, weights: Array1, } @@ -27,27 +24,76 @@ impl Node where T: Float, { - pub fn new(features: usize) -> Self { + pub fn create(biased: bool, features: usize) -> Self { + let bias = if biased { + Some(Array0::zeros(())) + } else { + None + }; Self { - bias: Array0::zeros(()), + bias, features, weights: Array1::zeros(features), } } - pub fn features(&self) -> &usize { - &self.features + pub fn biased(features: usize) -> Self { + Self::create(true, features) + } + + pub fn new(features: usize) -> Self { + Self::create(false, features) + } + + pub fn bias(&self) -> Option<&Array0> { + self.bias.as_ref() + } + + pub fn bias_mut(&mut self) -> Option<&mut Array0> { + self.bias.as_mut() + } + + pub fn features(&self) -> usize { + self.features } - pub fn features_mut(&mut self) -> &mut usize { - &mut self.features + pub fn is_biased(&self) -> bool { + self.bias.is_some() + } + + pub fn linear(&self, data: &Array2) -> Array1 + where + T: 'static, + { + let w = self.weights().t().to_owned(); + if let Some(bias) = self.bias() { + data.dot(&w) + bias + } else { + data.dot(&w) + } + } + + pub fn set_bias(&mut self, bias: Option>) { + self.bias = bias; } pub fn set_features(&mut self, features: usize) { self.features = features; } - pub fn with_bias(mut self, bias: Array0) -> Self { + pub fn set_weights(&mut self, weights: Array1) { + self.weights = weights; + } + + pub fn weights(&self) -> &Array1 { + &self.weights + } + + pub fn weights_mut(&mut self) -> &mut Array1 { + &mut self.weights + } + + pub fn with_bias(mut self, bias: Option>) -> Self { self.bias = bias; self } @@ -77,7 +123,7 @@ where pub fn init_bias(mut self) -> Self { let dk = (T::one() / T::from(self.features).unwrap()).sqrt(); - self.bias = Array0::uniform_between(dk, ()); + self.bias = Some(Array0::uniform_between(dk, ())); self } @@ -91,8 +137,7 @@ where impl Node where - T: FromPrimitive + NdFloat, - Self: Weighted, + T: NdFloat, { pub fn apply_gradient(&mut self, gamma: T, gradient: G) where @@ -106,89 +151,25 @@ where where A: Fn(&Array1) -> Array1, { - activator(&self.linear(data)) - } -} -impl Node -where - T: FromPrimitive + NdFloat, - Self: Biased + Weighted, -{ - pub fn linear(&self, data: &Array2) -> Array1 { - data.dot(&self.weights().t()) + self.bias() + activator(&self.forward(data)) } } impl Forward> for Node where - Self: Biased + Weighted, D: Dimension + RemoveAxis, - T: FromPrimitive + NdFloat, + T: NdFloat, Array: Dot, Output = Array>, Array: ops::Add, Output = Array>, { type Output = Array; fn forward(&self, data: &Array) -> Self::Output { - data.dot(&self.weights().t().to_owned()) + self.bias().clone() - } -} - -// impl Forward> for Node -// where -// Self: Biased + Weighted, -// T: FromPrimitive + NdFloat, -// { -// type Output = T; - -// fn forward(&self, data: &Array1) -> Self::Output { -// data.dot(&self.weights().t()) + self.bias().first().unwrap().clone() -// } -// } - -// impl Forward> for Node -// where -// Self: Biased + Weighted, -// T: FromPrimitive + NdFloat, -// { -// type Output = Array1; - -// fn forward(&self, data: &Array2) -> Self::Output { -// data.dot(&self.weights().t()) + self.bias().clone() -// } -// } - -impl Biased for Node -where - T: Float, -{ - fn bias(&self) -> &Array0 { - &self.bias - } - - fn bias_mut(&mut self) -> &mut Array0 { - &mut self.bias - } - - fn set_bias(&mut self, bias: Array0) { - self.bias = bias; - } -} - -impl Weighted for Node -where - T: Float, -{ - fn set_weights(&mut self, weights: Array1) { - self.weights = weights; - } - - fn weights(&self) -> &Array1 { - &self.weights - } - - fn weights_mut(&mut self) -> &mut Array1 { - &mut self.weights + let w = self.weights().t().to_owned(); + if let Some(bias) = self.bias() { + return data.dot(&w) + bias.clone(); + } + data.dot(&w) } } @@ -202,7 +183,7 @@ where { let weights = Array1::::from_iter(iter); Self { - bias: Array0::zeros(()), + bias: None, features: weights.len(), weights, } @@ -215,7 +196,7 @@ where { fn from((weights, bias): (Array1, Array0)) -> Self { Self { - bias, + bias: Some(bias), features: weights.len(), weights, } @@ -228,14 +209,45 @@ where { fn from((weights, bias): (Array1, T)) -> Self { Self { - bias: Array0::ones(()) * bias, + bias: Some(Array0::ones(()) * bias), + features: weights.len(), + weights, + } + } +} + +impl From<(Array1, Option)> for Node +where + T: Float + ScalarOperand, +{ + fn from((weights, bias): (Array1, Option)) -> Self { + let bias = if let Some(b) = bias { + Some(Array0::ones(()) * b) + } else { + None + }; + Self { + bias, + features: weights.len(), + weights, + } + } +} + +impl From<(Array1, Option>)> for Node +where + T: Float, +{ + fn from((weights, bias): (Array1, Option>)) -> Self { + Self { + bias, features: weights.len(), weights, } } } -impl From> for (Array1, Array0) +impl From> for (Array1, Option>) where T: Float, { diff --git a/ml/neural/src/neurons/perceptron.rs b/ml/neural/src/neurons/perceptron.rs index f636988b..916c1bb3 100644 --- a/ml/neural/src/neurons/perceptron.rs +++ b/ml/neural/src/neurons/perceptron.rs @@ -3,8 +3,7 @@ Contrib: FL03 */ use super::Node; -use crate::func::activate::{Activate, Linear}; -use crate::prelude::{Forward, Parameterized, ParameterizedExt, Weighted}; +use crate::prelude::{Activate, Forward, Linear}; use ndarray::prelude::{Array0, Array1, Array2, Ix1, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, StandardNormal}; @@ -26,6 +25,16 @@ where A: Activate, T: Float, { + pub fn new(features: usize) -> Self + where + A: Default, + { + Self { + activation: A::default(), + node: Node::create(false, features), + } + } + pub fn node(&self) -> &Node { &self.node } @@ -34,11 +43,23 @@ where &mut self.node } + pub fn features(&self) -> usize { + self.node.features() + } + + pub fn params(&self) -> &Node { + &self.node + } + + pub fn params_mut(&mut self) -> &mut Node { + &mut self.node + } + pub fn rho(&self) -> &A { &self.activation } - pub fn with_bias(mut self, bias: Array0) -> Self { + pub fn with_bias(mut self, bias: Option>) -> Self { self.node = self.node.with_bias(bias); self } @@ -59,18 +80,17 @@ where self.node = self.node.with_weights(weights); self } -} -impl Perceptron -where - T: NdFloat, - A: Activate + Default, -{ - pub fn new(features: usize) -> Self { - Self { - activation: A::default(), - node: Node::new(features), - } + pub fn weights(&self) -> &Array1 { + self.node.weights() + } + + pub fn weights_mut(&mut self) -> &mut Array1 { + self.node.weights_mut() + } + + pub fn set_weights(&mut self, weights: Array1) { + self.node.set_weights(weights); } } @@ -116,77 +136,6 @@ where } } -// impl Biased for Neuron -// where -// T: Float, -// A: Activate, -// { -// fn bias(&self) -> &Array0 { -// self.node.bias() -// } - -// fn bias_mut(&mut self) -> &mut Array0 { -// self.node.bias_mut() -// } - -// fn set_bias(&mut self, bias: Array0) { -// self.node.set_bias(bias); -// } -// } - -impl Weighted for Perceptron -where - T: Float, - A: Activate, -{ - fn weights(&self) -> &Array1 { - self.node.weights() - } - - fn weights_mut(&mut self) -> &mut Array1 { - self.node.weights_mut() - } - - fn set_weights(&mut self, weights: Array1) { - self.node.set_weights(weights); - } -} - -impl Parameterized for Perceptron -where - A: Activate, - T: Float, -{ - type Features = usize; - - type Params = Node; - - fn features(&self) -> &Self::Features { - self.node.features() - } - - fn features_mut(&mut self) -> &mut Self::Features { - self.node.features_mut() - } - - fn params(&self) -> &Self::Params { - &self.node - } - - fn params_mut(&mut self) -> &mut Self::Params { - &mut self.node - } -} - -// impl Forward> for Neuron { -// type Output = f64; - -// fn forward(&self, args: &Array1) -> Self::Output { -// self.rho().activate(args.dot(&self.weights().t().to_owned()) + self.bias) -// } - -// } - impl Forward> for Perceptron where T: NdFloat, @@ -195,7 +144,7 @@ where type Output = Array1; fn forward(&self, args: &Array2) -> Self::Output { - let linstep = args.dot(&self.node().weights().t()) + self.bias(); + let linstep = self.params().forward(args); self.rho().activate(&linstep) } } @@ -252,7 +201,7 @@ where } } -impl From> for (Array1, Array0) +impl From> for (Array1, Option>) where T: Float, A: Activate, diff --git a/ml/neural/src/nn/ffn/mlp.rs b/ml/neural/src/nn/ffn/mlp.rs index ca6f8d07..dfbd41bb 100644 --- a/ml/neural/src/nn/ffn/mlp.rs +++ b/ml/neural/src/nn/ffn/mlp.rs @@ -7,7 +7,7 @@ use crate::func::activate::{Activate, Linear, ReLU, Softmax}; use crate::layers::{Layer, LayerShape, Stack}; -use crate::prelude::{Features, Forward, Parameterized}; +use crate::prelude::{Features, Forward}; use ndarray::prelude::{Array2, Ix2, NdFloat}; use ndarray::IntoDimension; diff --git a/ml/neural/src/nn/ffn/mod.rs b/ml/neural/src/nn/ffn/mod.rs index f0a6d506..018b086c 100644 --- a/ml/neural/src/nn/ffn/mod.rs +++ b/ml/neural/src/nn/ffn/mod.rs @@ -4,13 +4,11 @@ */ //! # Feed Forward Neural Network //! -pub use self::{mlp::*, model::*, utils::*}; +pub use self::{mlp::*, model::*}; pub(crate) mod mlp; pub(crate) mod model; -pub(crate) mod utils {} - #[cfg(tets)] mod tests { use super::*; diff --git a/ml/neural/src/nn/ffn/model.rs b/ml/neural/src/nn/ffn/model.rs index 6daec964..51467898 100644 --- a/ml/neural/src/nn/ffn/model.rs +++ b/ml/neural/src/nn/ffn/model.rs @@ -4,7 +4,7 @@ */ use crate::func::activate::Activator; -use crate::prelude::{Features, Forward, Layer, Parameterized}; +use crate::prelude::{Features, Forward, Layer}; use ndarray::prelude::{Array2, NdFloat}; use num::Float; diff --git a/ml/neural/src/ops/norm.rs b/ml/neural/src/ops/norm.rs index 436346ef..53f44b23 100644 --- a/ml/neural/src/ops/norm.rs +++ b/ml/neural/src/ops/norm.rs @@ -2,19 +2,18 @@ Appellation: norm Contrib: FL03 */ -use crate::core::MatrixOps; use crate::prelude::Forward; use ndarray::prelude::{Array, Axis, Dimension, Ix2, NdFloat}; use ndarray::{IntoDimension, RemoveAxis}; use num::{Float, FromPrimitive}; use serde::{Deserialize, Serialize}; -use std::ops::{Add, Mul}; +use std::ops::{Add, Div, Mul}; pub fn norm(x: &Array, axis: usize) -> Array where D: Dimension + RemoveAxis, T: FromPrimitive + NdFloat, - Array: MatrixOps, + Array: Div, Output = Array>, { let axis = Axis(axis); let epsilon = T::from(1e-6).unwrap(); diff --git a/ml/neural/src/params/mod.rs b/ml/neural/src/params/mod.rs index 78f0a632..7191e882 100644 --- a/ml/neural/src/params/mod.rs +++ b/ml/neural/src/params/mod.rs @@ -17,8 +17,6 @@ use ndarray::prelude::{Array, Dimension, Ix2}; use ndarray::IntoDimension; use num::Float; -pub type BoxedParams = Box>; - pub trait Biased where D: Dimension, @@ -45,173 +43,7 @@ where fn set_weights(&mut self, weights: Array); } -pub trait Params -where - D: Dimension, - T: Float, -{ - /// Returns an owned reference to the bias of the layer. - fn bias(&self) -> &Array; - /// Returns a mutable reference to the bias of the layer. - fn bias_mut(&mut self) -> &mut Array; - /// Returns an owned reference to the weights of the layer. - fn weights(&self) -> &Array; - /// Returns a mutable reference to the weights of the layer. - fn weights_mut(&mut self) -> &mut Array; - /// Sets the bias of the layer. - fn set_bias(&mut self, bias: Array); - /// Sets the weights of the layer. - fn set_weights(&mut self, weights: Array); -} - -pub trait ParamsExt: Biased + Weighted -where - Array: Dot, Output = Array>, - D: Dimension, - T: Float, -{ - fn linear(&self, args: &Array) -> Array { - args.dot(self.weights()) + self.bias() - } -} - -pub trait Parameterized -where - D: Dimension, - T: Float, -{ - type Features: IntoDimension; - type Params; - - fn features(&self) -> &Self::Features; - - fn features_mut(&mut self) -> &mut Self::Features; - - fn params(&self) -> &Self::Params; - - fn params_mut(&mut self) -> &mut Self::Params; -} - -pub trait ParameterizedExt: Parameterized -where - D: Dimension, - T: Float, - >::Params: Params + 'static, -{ - fn bias(&self) -> &Array { - Params::bias(self.params()) - } - - fn bias_mut(&mut self) -> &mut Array { - Params::bias_mut(self.params_mut()) - } - - fn weights(&self) -> &Array { - Params::weights(self.params()) - } - - fn weights_mut(&mut self) -> &mut Array { - Params::weights_mut(self.params_mut()) - } - - fn set_bias(&mut self, bias: Array) { - Params::set_bias(self.params_mut(), bias) - } - - fn set_weights(&mut self, weights: Array) { - Params::set_weights(self.params_mut(), weights) - } -} - -impl ParameterizedExt for P -where - D: Dimension, - P: Parameterized, - T: Float, -

>::Params: Params + 'static, -{ -} - -// impl Params for S -// where -// S: Parameterized, -// D: Dimension, -// P: Biased, -// T: Float, -// ::Smaller: Dimension, -// { -// fn bias(&self) -> &Array { -// self.params().bias() -// } - -// fn bias_mut(&mut self) -> &mut Array { -// self.params_mut().bias_mut() -// } - -// fn weights(&self) -> &Array { -// self.params().weights() -// } - -// fn weights_mut(&mut self) -> &mut Array { -// self.params_mut().weights_mut() -// } - -// fn set_bias(&mut self, bias: Array) { -// self.params_mut().set_bias(bias) -// } - -// fn set_weights(&mut self, weights: Array) { -// self.params_mut().set_weights(weights) -// } -// } - -impl Params for P -where - D: Dimension, - T: Float, - Self: Biased + Weighted + Sized, -{ - fn bias(&self) -> &Array { - Biased::bias(self) - } - - fn bias_mut(&mut self) -> &mut Array { - Biased::bias_mut(self) - } - - fn weights(&self) -> &Array { - Weighted::weights(self) - } - - fn weights_mut(&mut self) -> &mut Array { - Weighted::weights_mut(self) - } - - fn set_bias(&mut self, bias: Array) { - Biased::set_bias(self, bias) - } - - fn set_weights(&mut self, weights: Array) { - Weighted::set_weights(self, weights) - } -} - -// impl Biased for P -// where -// D: Dimension, -// P: Parameterized, -// T: Float, -// ::Smaller: Dimension, -//

>::Params: 'static, -// { -// fn bias(&self) -> &Array { -// self.params().bias() -// } - -// fn bias_mut(&mut self) -> &mut Array { -// self.params_mut().bias_mut() -// } -// } +pub trait Params {} #[cfg(test)] mod tests {} diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index e48fdfd5..4374cff2 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -4,7 +4,7 @@ */ use crate::core::BoxResult; use crate::func::loss::Loss; -use ndarray::prelude::{Array, Array1, Axis, Dimension, Ix2}; +use ndarray::prelude::{Array, Axis, Dimension, Ix2}; use num::Float; pub trait Backward: Forward { @@ -36,40 +36,32 @@ where // { // } -pub trait Compile -where - D: Dimension, - T: Float, -{ +pub trait Batched { + type Output; + + fn batch(&self, batch_size: usize) -> Self::Output; +} + +pub trait Compile { type Opt; - fn compile(&mut self, loss: impl Loss>, optimizer: Self::Opt) -> BoxResult<()>; + fn compile(&mut self, loss: impl Loss, optimizer: Self::Opt) -> BoxResult<()>; } -pub trait Predict -where - D: Dimension, - T: Float, -{ +pub trait Predict { type Output; - fn predict(&self, input: &Array) -> BoxResult; - - fn predict_batch(&self, input: &[Array]) -> BoxResult> { - let res = input.iter().map(|x| self.predict(x).expect("")).collect(); - Ok(res) - } + fn predict(&self, input: &T) -> BoxResult; } -impl Predict for Box> +impl Predict for S where - D: Dimension, - T: Float, + S: Forward, { type Output = O; - fn predict(&self, input: &Array) -> BoxResult { - self.as_ref().predict(input) + fn predict(&self, input: &T) -> BoxResult { + Ok(self.forward(input)) } } @@ -97,10 +89,6 @@ where } } -pub trait Module: Forward, Output = Array> -where - D: Dimension, - T: Float, -{ +pub trait Module: Compile + Predict { type Config; } diff --git a/ml/optim/src/grad/gradient.rs b/ml/optim/src/grad/gradient.rs index 967ba7e0..61cc8230 100644 --- a/ml/optim/src/grad/gradient.rs +++ b/ml/optim/src/grad/gradient.rs @@ -4,7 +4,7 @@ */ use crate::neural::func::activate::Sigmoid; use crate::neural::models::ModelParams; -use crate::neural::prelude::{Forward, Gradient, Weighted}; +use crate::neural::prelude::{Forward, Gradient}; use ndarray::prelude::{Array2, Axis, NdFloat}; use ndarray_stats::DeviationExt; use num::{Float, Signed}; diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index 671365b2..2732d288 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -28,13 +28,15 @@ pub struct DescentParams { } pub(crate) mod utils { + use crate::core::prelude::BoxResult; use crate::neural::func::activate::Gradient; - use crate::neural::params::{Biased, Weighted}; - use crate::neural::prelude::{Forward, ForwardIter, Parameterized, Params}; + use crate::neural::models::exp::Module; + use crate::neural::prelude::{Forward, ForwardIter}; use ndarray::linalg::Dot; use ndarray::prelude::{Array, Array2, Dimension, NdFloat}; use ndarray_stats::DeviationExt; use num::{FromPrimitive, Signed}; + use std::ops::Sub; pub fn gradient_descent( gamma: T, @@ -46,7 +48,7 @@ pub(crate) mod utils { where D: Dimension, M: Clone + ForwardIter, I, Output = Array>, - I: Forward, Output = Array> + Biased + Weighted, + I: Forward, Output = Array>, T: FromPrimitive + NdFloat + Signed, Array2: Dot, Output = Array>, { @@ -54,22 +56,22 @@ pub(crate) mod utils { Ok(loss) } - pub fn gradient( + pub fn gradient<'a, T, D, A>( gamma: T, model: &mut A, data: &Array2, targets: &Array, grad: impl Gradient, - ) -> f64 + ) -> BoxResult where - A: Forward, Output = Array> + Parameterized, - D: Dimension, + A: Module>, + D: Dimension + 'a, T: FromPrimitive + NdFloat + Signed, - >::Params: Params + 'static, Array2: Dot, Output = Array>, + &'a Array2: Sub<&'a Array, Output = Array>, { let (_samples, _inputs) = data.dim(); - let pred = model.forward(data); + let pred = model.predict(data)?; let ns = T::from(data.len()).unwrap(); @@ -83,13 +85,14 @@ pub(crate) mod utils { // let db = dz.sum_axis(Axis(0)) / ns; // // Apply the gradients to the model's learnable parameters // model.params_mut().bias_mut().scaled_add(-gamma, &db.t()); - - model.params_mut().weights_mut().scaled_add(-gamma, &dw.t()); + for p in model.parameters_mut().values_mut() { + p.scaled_add(-gamma, &dw.t()); + } let loss = targets - .mean_sq_err(&model.forward(data)) + .mean_sq_err(&model.predict(data)?) .expect("Error when calculating the MSE of the model"); - loss + Ok(loss) } } @@ -100,7 +103,7 @@ mod tests { use crate::core::prelude::linarr; use crate::neural::func::activate::{Linear, Sigmoid}; use crate::neural::models::ModelParams; - use crate::neural::prelude::{Features, Layer, LayerShape}; + use crate::neural::prelude::{Features, Forward, Layer, LayerShape}; use ndarray::prelude::{Array1, Ix2}; #[test] @@ -139,17 +142,19 @@ mod tests { let features = LayerShape::new(inputs, outputs); // Generate some example data - let x = linarr((samples, features.inputs())).unwrap(); - let y = linarr((samples, features.outputs())).unwrap(); + let x = linarr::((samples, features.inputs())).unwrap(); + let y = linarr::((samples, features.outputs())).unwrap(); let mut model = Layer::::from(features).init(true); - let mut losses = Array1::zeros(epochs); - for e in 0..epochs { - let cost = gradient(gamma, &mut model, &x, &y, Sigmoid); - losses[e] = cost; - } - assert_eq!(losses.len(), epochs); - assert!(losses.first() > losses.last()); + let pred = model.forward(&x); + + // let mut losses = Array1::zeros(epochs); + // for e in 0..epochs { + // let cost = gradient(gamma, &mut model, &x, &y, Sigmoid).unwrap(); + // losses[e] = cost; + // } + // assert_eq!(losses.len(), epochs); + // assert!(losses.first() > losses.last()); } } diff --git a/ml/optim/src/grad/sgd.rs b/ml/optim/src/grad/sgd.rs index 12473391..b2f1d0ed 100644 --- a/ml/optim/src/grad/sgd.rs +++ b/ml/optim/src/grad/sgd.rs @@ -6,7 +6,7 @@ //! //! -use crate::neural::prelude::{Activate, Features, Forward, Layer, Parameterized, Weighted}; +use crate::neural::prelude::{Activate, Features, Forward, Layer}; // use crate::prelude::ObjectiveFn; use ndarray::prelude::{s, Array1, Array2, Axis, Ix2, NdFloat}; use ndarray_stats::DeviationExt; diff --git a/ml/s4/src/dplr/hippo.rs b/ml/s4/src/dplr/hippo.rs deleted file mode 100644 index d364c2ad..00000000 --- a/ml/s4/src/dplr/hippo.rs +++ /dev/null @@ -1,58 +0,0 @@ -/* - Appellation: hippo - Contrib: FL03 -*/ -use super::utils::*; -use ndarray::prelude::Array2; -use ndarray::ScalarOperand; -use num::complex::ComplexFloat; -use num::Float; - -pub enum HiPPOs { - HiPPO(Array2), - NPLR { - a: Array2, - p: Array2, - b: Array2, - }, - DPLR { - lambda: Array2, - p: Array2, - q: Array2, - b: Array2, - c: Array2, - }, -} - -pub struct HiPPO(Array2); - -impl HiPPO -where - T: Float, -{ - pub fn new(hippo: Array2) -> Self { - Self(hippo) - } - - pub fn hippo(&self) -> &Array2 { - &self.0 - } - - pub fn hippo_mut(&mut self) -> &mut Array2 { - &mut self.0 - } -} - -impl HiPPO -where - T: ComplexFloat + ScalarOperand, -{ - pub fn square(features: usize) -> Self { - Self(make_hippo(features)) - } - - pub fn nplr(features: usize) -> Self { - let (hippo, p, b) = make_nplr_hippo(features); - Self(hippo) - } -} diff --git a/ml/s4/src/dplr/mod.rs b/ml/s4/src/dplr/mod.rs deleted file mode 100644 index cd06181e..00000000 --- a/ml/s4/src/dplr/mod.rs +++ /dev/null @@ -1,76 +0,0 @@ -/* - Appellation: dplr - Contrib: FL03 -*/ -//! # Diagonal Plus Low Rank (DPLR) -//! -//! -pub use self::{kinds::*, utils::*}; - -pub(crate) mod kinds; - -pub mod hippo; - -pub struct LowRank { - pub mode: Mode, -} - -pub(crate) mod utils { - use crate::core::prelude::{rangespace, AsComplex, Conjugate}; - - use ndarray::prelude::{Array1, Array2, Axis}; - use ndarray::ScalarOperand; - use ndarray_linalg::{Eigh, IntoTriangular, Lapack, UPLO}; - use num::complex::{Complex, ComplexFloat}; - use num::FromPrimitive; - - pub fn make_hippo(features: usize) -> Array2 - where - T: ComplexFloat + ScalarOperand, - { - let base = rangespace((features, 1)); - let p = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); - let mut a = &p * &p.t(); - a = &a.into_triangular(UPLO::Lower) - &base.diag(); - -a - } - - pub fn make_nplr_hippo(features: usize) -> (Array2, Array1, Array1) - where - T: ComplexFloat + ScalarOperand, - { - let hippo = make_hippo(features); - - let base = rangespace((features,)); - let p = (&base + T::one() / T::from(2).unwrap()).mapv(T::sqrt); - let b = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); - (hippo, p, b) - } - - pub fn make_dplr_hippo(features: usize) -> (Array2, Array2, Array2, Array2) - where - T: AsComplex + ComplexFloat + Conjugate + FromPrimitive + Lapack + ScalarOperand, - { - let (a, p, b) = make_nplr_hippo(features); - let p = p.insert_axis(Axis(1)); - let b = b.insert_axis(Axis(1)); - - // - let s = &a + p.dot(&p.t()); - // - let sd = s.diag(); - - let a = Array2::ones(s.dim()) * sd.mean().expect("Average of diagonal is NaN"); - - // TODO: replace with eigh - let (e, v) = &(&s * T::from(T::one().neg().as_imag()).unwrap()) - .eigh(UPLO::Lower) - .expect(""); - let e = e.mapv(|x| T::from(x).unwrap()); - - let a = a + &e * T::from(T::one().as_imag()).unwrap(); - let p = v.conj().t().dot(&p); - let b = v.conj().t().dot(&b); - (a, p, b, v.clone()) - } -} diff --git a/ml/s4/src/hippo/dplr.rs b/ml/s4/src/hippo/dplr.rs new file mode 100644 index 00000000..227b7e94 --- /dev/null +++ b/ml/s4/src/hippo/dplr.rs @@ -0,0 +1,35 @@ +/* + Appellation: dplr + Contrib: FL03 +*/ +//! # Diagonal Plus Low Rank (DPLR) +//! +//! +use super::utils::*; +use crate::prelude::S4Float; +use ndarray::prelude::{Array1, Array2}; +use ndarray::{LinalgScalar, ScalarOperand}; +use num::complex::ComplexFloat; +use num::{Complex, Num}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +pub struct DPLR +where + T: Clone + Num, +{ + pub lambda: Array2>, + pub p: Array1, + pub b: Array1, + pub v: Array2, +} + +impl DPLR +where + T: S4Float, + Complex<::Real>: LinalgScalar + ScalarOperand, +{ + pub fn new(features: usize) -> Self { + make_dplr_hippo(features) + } +} diff --git a/ml/s4/src/hippo/hippo.rs b/ml/s4/src/hippo/hippo.rs new file mode 100644 index 00000000..7e22e1cd --- /dev/null +++ b/ml/s4/src/hippo/hippo.rs @@ -0,0 +1,146 @@ +/* + Appellation: hippo + Contrib: FL03 +*/ +use super::dplr::DPLR; +use super::nplr::NPLR; +use crate::core::prelude::{rangespace, Conjugate}; +use crate::prelude::S4Float; +use ndarray::prelude::{Array2, Axis}; +use ndarray::{LinalgScalar, ScalarOperand}; +use ndarray_linalg::{Eigh, IntoTriangular, UPLO}; +use num::complex::{Complex, ComplexFloat}; +use num::traits::Num; +// use num::traits::{Float, FloatConst}; +use serde::{Deserialize, Serialize}; + +pub enum HiPPOs +where + T: Clone + Num, +{ + HiPPO(HiPPO), + DPLR(DPLR), + NPLR(NPLR), +} + +impl HiPPOs +where + T: ComplexFloat + ScalarOperand, +{ + pub fn new(features: usize) -> Self { + Self::HiPPO(HiPPO::new(features)) + } + + pub fn nplr(features: usize) -> Self { + Self::NPLR(NPLR::new(features)) + } +} + +impl HiPPOs +where + T: S4Float, + Complex<::Real>: LinalgScalar + ScalarOperand, +{ + pub fn dplr(features: usize) -> Self { + Self::DPLR(DPLR::new(features)) + } +} + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +pub struct HiPPO { + features: usize, + data: Array2, +} + +impl HiPPO { + pub fn features(&self) -> usize { + self.features + } +} + +impl HiPPO +where + T: ComplexFloat + ScalarOperand, +{ + pub fn new(features: usize) -> Self { + let base = rangespace((features, 1)); + let p = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); + let mut a = &p * &p.t(); + a = &a.into_triangular(UPLO::Lower) - &base.diag(); + + Self { features, data: -a } + } + + pub fn nplr(&self) -> NPLR { + let base = rangespace(self.features()); + let p = (&base + T::one() / T::from(2).unwrap()).mapv(T::sqrt); + let b = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); + NPLR { + a: self.as_ref().clone(), + p, + b, + } + } +} + +impl HiPPO +where + T: S4Float, +{ + pub fn dplr(&self) -> DPLR { + let (a, p, b) = super::make_nplr_hippo(self.features).into(); + + // + let s = &a + + p.clone() + .insert_axis(Axis(1)) + .dot(&p.clone().insert_axis(Axis(0))); + // + let sd = s.diag(); + + let a = Array2::ones(s.dim()) * sd.mean().expect("Average of diagonal is NaN"); + + // TODO: replace with eigh + let (e, v) = &(&s * T::from(T::one().neg().as_imag()).unwrap()) + .eigh(UPLO::Lower) + .expect(""); + let e = e.mapv(|x| T::from(x).unwrap()); + + let a = a + &e * T::from(T::one().as_imag()).unwrap(); + let p = v.conj().t().dot(&p); + let b = v.conj().t().dot(&b); + DPLR { + lambda: a, + p, + b, + v: v.clone(), + } + } +} + +impl AsRef> for HiPPO { + fn as_ref(&self) -> &Array2 { + &self.data + } +} + +impl AsMut> for HiPPO { + fn as_mut(&mut self) -> &mut Array2 { + &mut self.data + } +} + +impl From> for HiPPO { + fn from(a: Array2) -> Self { + Self { + features: a.dim().0, + data: a, + } + } +} + +impl From> for Array2 { + fn from(hippo: HiPPO) -> Self { + hippo.data + } +} diff --git a/ml/s4/src/dplr/kinds.rs b/ml/s4/src/hippo/kinds.rs similarity index 100% rename from ml/s4/src/dplr/kinds.rs rename to ml/s4/src/hippo/kinds.rs diff --git a/ml/s4/src/hippo/mod.rs b/ml/s4/src/hippo/mod.rs new file mode 100644 index 00000000..7fbc8506 --- /dev/null +++ b/ml/s4/src/hippo/mod.rs @@ -0,0 +1,127 @@ +/* + Appellation: hippo + Contrib: FL03 +*/ +//! # HiPPO +//! +//! +pub use self::{hippo::*, kinds::*, utils::*}; + +pub(crate) mod hippo; +pub(crate) mod kinds; + +pub mod dplr; +pub mod nplr; + +pub struct LowRank { + pub mode: Mode, +} + +pub(crate) mod utils { + use super::dplr::DPLR; + use super::nplr::NPLR; + use crate::core::prelude::{rangespace, Conjugate}; + use crate::prelude::S4Float; + use ndarray::prelude::{Array2, Axis}; + use ndarray::{LinalgScalar, ScalarOperand}; + use ndarray_linalg::{Eigh, IntoTriangular, UPLO}; + use num::complex::{Complex, ComplexFloat}; + // use num::traits::{Float, FloatConst}; + use std::ops::Neg; + + pub(crate) fn make_hippo(features: usize) -> Array2 + where + T: ComplexFloat + ScalarOperand, + { + let base = rangespace(features).insert_axis(Axis(1)); + let p = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); + let mut a = &p * &p.t(); + a = &a.into_triangular(UPLO::Lower) - &base.diag(); + -a + } + + pub(crate) fn make_nplr_hippo(features: usize) -> NPLR + where + T: ComplexFloat + ScalarOperand, + { + let hippo = make_hippo(features); + + let base = rangespace((features,)); + let p = (&base + T::one() / T::from(2).unwrap()).mapv(T::sqrt); + let b = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); + NPLR { a: hippo, p, b } + } + + pub(crate) fn make_dplr_hippo(features: usize) -> DPLR + where + T: S4Float, + Complex<::Real>: LinalgScalar + ScalarOperand, + { + let (a, p, b) = make_nplr_hippo(features).into(); + + // + let s = &a + + p.clone() + .insert_axis(Axis(1)) + .dot(&p.clone().insert_axis(Axis(0))); + // + let sd = s.diag(); + + let a = Array2::ones(s.dim()) * sd.mean().expect("Average of diagonal is NaN"); + + // TODO: replace with eigh + let (e, v) = &(&s * T::from((Complex::<::Real>::i().neg())).unwrap()) + .eigh(UPLO::Lower) + .expect(""); + let e = e.mapv(|x| T::from(x).unwrap()); + + let a = a + &e * T::from(T::one().as_imag()).unwrap(); + let p = v.conj().t().dot(&p); + let b = v.conj().t().dot(&b); + DPLR { + lambda: a, + p, + b, + v: v.clone(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use dplr::DPLR; + use nplr::NPLR; + use num::Complex; + + #[test] + fn test_hippo() { + let features = 10; + + let a = make_hippo::(features); + let b = HiPPO::::new(features); + assert_eq!(&a, b.as_ref()); + } + + #[test] + fn test_dplr() { + let features = 10; + + let a = make_dplr_hippo::>(features); + let b = HiPPO::>::new(features).dplr(); + let c = DPLR::>::new(features); + assert_eq!(&a, &b); + assert_eq!(&a, &c); + } + + #[test] + fn test_nplr() { + let features = 10; + + let a = make_nplr_hippo::(features); + let b = HiPPO::::new(features).nplr(); + let c = NPLR::::new(features); + assert_eq!(&a, &b); + assert_eq!(&a, &c); + } +} diff --git a/ml/s4/src/hippo/nplr.rs b/ml/s4/src/hippo/nplr.rs new file mode 100644 index 00000000..6beb48bf --- /dev/null +++ b/ml/s4/src/hippo/nplr.rs @@ -0,0 +1,42 @@ +/* + Appellation: nplr + Contrib: FL03 +*/ +//! # Normal Plus Low Rank (NPLR) +//! +//! +use super::utils::*; + +use ndarray::prelude::{Array1, Array2}; +use ndarray::ScalarOperand; +use num::complex::ComplexFloat; +// use num::traits::{Float, FloatConst}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +pub struct NPLR { + pub a: Array2, + pub p: Array1, + pub b: Array1, +} + +impl NPLR +where + T: ComplexFloat + ScalarOperand, +{ + pub fn new(features: usize) -> Self { + make_nplr_hippo(features) + } +} + +impl From> for (Array2, Array1, Array1) { + fn from(nplr: NPLR) -> Self { + (nplr.a, nplr.p, nplr.b) + } +} + +impl From<(Array2, Array1, Array1)> for NPLR { + fn from((a, p, b): (Array2, Array1, Array1)) -> Self { + Self { a, p, b } + } +} diff --git a/ml/s4/src/lib.rs b/ml/s4/src/lib.rs index 349dd913..09faeadc 100644 --- a/ml/s4/src/lib.rs +++ b/ml/s4/src/lib.rs @@ -13,7 +13,7 @@ pub(crate) mod specs; pub(crate) mod utils; pub mod cmp; -pub mod dplr; +pub mod hippo; pub mod ops; pub mod params; pub mod ssm; diff --git a/ml/s4/src/model/model.rs b/ml/s4/src/model/model.rs index 2c043593..e3b9b483 100644 --- a/ml/s4/src/model/model.rs +++ b/ml/s4/src/model/model.rs @@ -8,12 +8,19 @@ use crate::prelude::SSMStore; use ndarray::prelude::{Array1, Array2, NdFloat}; use ndarray_conv::{Conv2DFftExt, PaddingMode, PaddingSize}; use ndarray_linalg::Scalar; +use num::complex::{Complex, ComplexFloat}; use num::Float; -// use num::complex::{Complex, ComplexFloat}; use rustfft::FftNum; - +// use std::collections::HashMap; use crate::prelude::SSMParams::*; +pub struct S4State +where + T: ComplexFloat, +{ + cache: Array1, +} + pub struct S4 where T: Float, diff --git a/ml/s4/src/primitives.rs b/ml/s4/src/primitives.rs index 7db8f95d..09ef539b 100644 --- a/ml/s4/src/primitives.rs +++ b/ml/s4/src/primitives.rs @@ -3,6 +3,23 @@ Contrib: FL03 */ pub use self::constants::*; +use crate::core::specs::{Arithmetic, AsComplex, Conjugate}; +use ndarray::ScalarOperand; +use ndarray_linalg::Lapack; +use num::complex::ComplexFloat; +use num::traits::FromPrimitive; +// use num::traits::{Float, FromPrimitive, FloatConst}; +use std::ops; + +pub trait S4Float: + Arithmetic + AsComplex + Conjugate + ComplexFloat + FromPrimitive + Lapack + ScalarOperand +{ +} + +impl S4Float for T where + T: AsComplex + Conjugate + ComplexFloat + FromPrimitive + Lapack + ScalarOperand +{ +} mod constants { /// The default model size for S4 models diff --git a/ml/s4/src/specs.rs b/ml/s4/src/specs.rs index 0c990b8d..8935e73a 100644 --- a/ml/s4/src/specs.rs +++ b/ml/s4/src/specs.rs @@ -2,6 +2,10 @@ Appellation: specs Contrib: FL03 */ +use crate::core::prelude::AsComplex; +use ndarray::prelude::{Array, Dimension}; +use num::complex::ComplexFloat; +use rustfft::{Fft, FftNum, FftPlanner}; pub trait Scan { type Output; @@ -14,3 +18,47 @@ pub trait StateSpace { fn config(&self) -> &Self::Config; } + +pub trait NdFft { + type Output; + + fn fft(&self, args: &Self) -> Self::Output; + + fn ifft(&self, args: &Self) -> Self::Output; +} + +impl NdFft for Array +where + D: Dimension, + T: AsComplex + ComplexFloat + FftNum, +{ + type Output = Self; + + fn fft(&self, args: &Self) -> Self::Output { + let dim = args.dim(); + let mut out = Self::ones(args.dim()); + let mut buffer = vec![T::zero().as_complex(); args.len()]; + let mut planner = FftPlanner::new(); + let fft = planner.plan_fft_forward(args.len()); + fft.process(buffer.as_mut_slice()); + let buffer = buffer + .into_iter() + .map(|i| T::from(i).unwrap()) + .collect::>(); + Self::from_shape_vec(args.dim(), buffer).expect("") + } + + fn ifft(&self, args: &Self) -> Self::Output { + let dim = args.dim(); + let mut out = Self::ones(args.dim()); + let mut buffer = vec![T::zero().as_complex(); args.len()]; + let mut planner = FftPlanner::new(); + let fft = planner.plan_fft_inverse(args.len()); + fft.process(buffer.as_mut_slice()); + let buffer = buffer + .into_iter() + .map(|i| T::from(i).unwrap()) + .collect::>(); + Self::from_shape_vec(args.dim(), buffer).expect("") + } +} From 383d69f096a11f3325aaf44a251b7a5036699a09 Mon Sep 17 00:00:00 2001 From: FL03 Date: Tue, 2 Jan 2024 15:30:30 -0600 Subject: [PATCH 101/118] update Signed-off-by: FL03 --- core/src/ops/kinds.rs | 8 + core/src/ops/mod.rs | 17 ++ core/src/specs/arrays.rs | 34 +++ core/src/specs/math.rs | 42 ++-- core/src/specs/mod.rs | 40 ++-- core/src/specs/numerical.rs | 101 +++++++++ data/src/lib.rs | 8 +- data/src/mat/matrix.rs | 8 + data/src/mat/mod.rs | 15 ++ data/src/misc/dtype.rs | 150 +++++++++++++ data/src/specs/elements.rs | 4 + data/src/{specs.rs => specs/mod.rs} | 21 +- data/src/store/mod.rs | 10 +- data/src/store/storage.rs | 6 + data/src/tensors/mod.rs | 2 + ml/linear/src/model/layer.rs | 287 +++++++++++++++++++++++++ ml/linear/src/model/mod.rs | 46 +++- ml/linear/src/params/store.rs | 144 +++++++------ ml/neural/src/layers/seq/sequential.rs | 168 ++++++++++++++- ml/neural/src/neurons/node.rs | 36 ++-- ml/neural/src/nn/ffn/mlp.rs | 1 + ml/neural/src/nn/mod.rs | 5 +- ml/neural/src/nn/sequential.rs | 12 -- ml/neural/src/ops/mod.rs | 6 +- ml/neural/src/ops/norm.rs | 32 +-- ml/s4/src/hippo/dplr.rs | 76 ++++++- ml/s4/src/hippo/hippo.rs | 124 +++++------ ml/s4/src/hippo/mod.rs | 155 +++++++++---- ml/s4/src/hippo/nplr.rs | 19 +- ml/s4/src/primitives.rs | 33 ++- ml/s4/src/specs.rs | 4 +- 31 files changed, 1322 insertions(+), 292 deletions(-) create mode 100644 core/src/ops/kinds.rs create mode 100644 core/src/ops/mod.rs create mode 100644 core/src/specs/numerical.rs create mode 100644 data/src/mat/matrix.rs create mode 100644 data/src/mat/mod.rs create mode 100644 data/src/specs/elements.rs rename data/src/{specs.rs => specs/mod.rs} (58%) create mode 100644 ml/linear/src/model/layer.rs delete mode 100644 ml/neural/src/nn/sequential.rs diff --git a/core/src/ops/kinds.rs b/core/src/ops/kinds.rs new file mode 100644 index 00000000..92d439f8 --- /dev/null +++ b/core/src/ops/kinds.rs @@ -0,0 +1,8 @@ +/* + Appellation: kinds + Contrib: FL03 +*/ + +pub enum Ops { + +} \ No newline at end of file diff --git a/core/src/ops/mod.rs b/core/src/ops/mod.rs new file mode 100644 index 00000000..b8bc97c9 --- /dev/null +++ b/core/src/ops/mod.rs @@ -0,0 +1,17 @@ +/* + Appellation: ops + Contrib: FL03 +*/ +//! # Operations +pub use self::kinds::*; + +pub(crate) mod kinds; + +pub trait Operation { + type Output; + + fn eval(&self, args: &T) -> Self::Output; +} + +#[cfg(test)] +mod tests {} diff --git a/core/src/specs/arrays.rs b/core/src/specs/arrays.rs index a201609e..8bc4154b 100644 --- a/core/src/specs/arrays.rs +++ b/core/src/specs/arrays.rs @@ -172,3 +172,37 @@ where } // pub trait Stack + +pub trait Genspace { + fn arange(start: T, stop: T, step: T) -> Self; + + fn linspace(start: T, stop: T, n: usize) -> Self; + + fn logspace(start: T, stop: T, n: usize) -> Self; + + fn geomspace(start: T, stop: T, n: usize) -> Self; + + fn ones(n: usize) -> Self; + + fn zeros(n: usize) -> Self; +} + +pub trait ArrayLike { + fn ones_like(&self) -> Self; + + fn zeros_like(&self) -> Self; +} + +impl ArrayLike for Array +where + T: Clone + Num, + D: Dimension, +{ + fn ones_like(&self) -> Self { + Array::ones(self.dim()) + } + + fn zeros_like(&self) -> Self { + Array::zeros(self.dim()) + } +} diff --git a/core/src/specs/math.rs b/core/src/specs/math.rs index e56eac62..8c763ea7 100644 --- a/core/src/specs/math.rs +++ b/core/src/specs/math.rs @@ -2,15 +2,11 @@ Appellation: math Contrib: FL03 */ -use ndarray::prelude::{Array, Dimension, Ix2, NdFloat}; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use num::{Complex, Float, FromPrimitive, Num, One, Signed, Zero}; +use ndarray::prelude::{Array, Dimension, Ix2}; +use num::complex::Complex; +use num::{Float, Num, Signed}; use std::ops; -pub trait Binary: One + Zero {} - -impl Binary for T where T: One + Zero {} - pub trait Conjugate { fn conj(&self) -> Self; } @@ -29,13 +25,22 @@ impl Conjugate for f64 { impl Conjugate for Complex where - T: Copy + Num + Signed, + T: Clone + Num + Signed, { fn conj(&self) -> Self { - Complex::::new(self.re, -self.im) + Complex::::new(self.re.clone(), -self.im.clone()) } } +// impl Conjugate for T +// where +// T: ComplexFloat, +// { +// fn conj(&self) -> Self { +// ComplexFloat::conj(self) +// } +// } + impl Conjugate for Array where D: Dimension, @@ -46,11 +51,10 @@ where } } -pub trait FloatExt: FromPrimitive + NdFloat + Signed + SampleUniform {} - -impl FloatExt for T where T: FromPrimitive + NdFloat + Signed + SampleUniform {} - -pub trait Arithmetic: ops::Add + ops::Div + ops::Mul + ops::Sub { +pub trait Arithmetic +where + Self: ops::Add + ops::Div + ops::Mul + ops::Sub, +{ type Output; } @@ -116,3 +120,13 @@ where Complex::::sqrt(self) } } + +impl SquareRoot for Array +where + D: Dimension, + T: Float, +{ + fn sqrt(self) -> Self { + self.mapv(|x| x.sqrt()) + } +} diff --git a/core/src/specs/mod.rs b/core/src/specs/mod.rs index 390b602e..fe3bd7da 100644 --- a/core/src/specs/mod.rs +++ b/core/src/specs/mod.rs @@ -2,36 +2,40 @@ Appellation: specs Contrib: FL03 */ -pub use self::{arrays::*, base::*, init::*, math::*}; +pub use self::{arrays::*, base::*, init::*, math::*, numerical::*}; pub(crate) mod arrays; pub(crate) mod base; pub(crate) mod init; pub(crate) mod math; +pub(crate) mod numerical; -use num::traits::float::FloatCore; -use num::{Complex, Num, Zero}; +use num::complex::Complex; +use num::traits::{Float, Num}; -pub trait CncFloat: FloatCore {} +pub trait AsComplex: Sized { + fn as_complex(self, real: bool) -> Complex; -impl CncFloat for T where T: FloatCore {} - -pub trait AsComplex: Num { - fn as_complex(&self) -> Complex; + fn as_re(self) -> Complex { + self.as_complex(true) + } - fn as_imag(&self) -> Complex; + fn as_im(self) -> Complex { + self.as_complex(false) + } } impl AsComplex for T where - T: Copy + Num + Zero, + T: Num, { - fn as_complex(&self) -> Complex { - Complex::new(*self, T::zero()) - } - - fn as_imag(&self) -> Complex { - Complex::new(T::zero(), *self) + fn as_complex(self, real: bool) -> Complex { + let (re, im): (Self, Self) = if real { + (self, Self::zero()) + } else { + (Self::zero(), self) + }; + Complex::new(re, im) } } @@ -45,7 +49,7 @@ pub trait RoundTo { impl RoundTo for T where - T: num::Float, + T: Float, { fn round_to(&self, places: usize) -> Self { crate::round_to(*self, places) @@ -61,7 +65,7 @@ mod tests { #[test] fn test_as_complex() { let x = 1.0; - let y = x.as_complex(); + let y = x.as_re(); assert_eq!(y, Complex::new(1.0, 0.0)); } diff --git a/core/src/specs/numerical.rs b/core/src/specs/numerical.rs new file mode 100644 index 00000000..bf726adb --- /dev/null +++ b/core/src/specs/numerical.rs @@ -0,0 +1,101 @@ +/* + Appellation: num + Contrib: FL03 +*/ +use std::ops::{self, Add, Div, Mul, Sub}; + +pub trait Algebraic +where + Self: Add + Div + Mul + Sub + Sized, +{ + type Output; +} + +pub trait AlgebraicExt +where + Self: Algebraic + + ops::AddAssign + + ops::DivAssign + + ops::MulAssign + + ops::SubAssign, +{ +} + +impl Algebraic for A +where + A: Add + Div + Mul + Sub, +{ + type Output = C; +} + +impl AlgebraicExt for A where + A: Algebraic + + ops::AddAssign + + ops::DivAssign + + ops::MulAssign + + ops::SubAssign +{ +} + +pub trait ComplexNum: + Algebraic + Algebraic +{ + type DType; + + fn imag(self) -> Self::DType; + + fn real(self) -> Self::DType; +} + +impl ComplexNum for num::Complex +where + T: Clone + num::Num, +{ + type DType = T; + + fn imag(self) -> Self::DType { + self.im + } + + fn real(self) -> Self::DType { + self.re + } +} + +pub trait Number {} + +impl Number for i8 {} + +impl Number for i16 {} + +impl Number for i32 {} + +impl Number for i64 {} + +impl Number for i128 {} + +impl Number for isize {} + +impl Number for u8 {} + +impl Number for u16 {} + +impl Number for u32 {} + +impl Number for u64 {} + +impl Number for u128 {} + +impl Number for usize {} + +impl Number for f32 {} + +impl Number for f64 {} + +impl Number for S where S: ComplexNum {} + +pub trait Numerical +where + T: Number, +{ +} diff --git a/data/src/lib.rs b/data/src/lib.rs index 85a723da..80c0aa92 100644 --- a/data/src/lib.rs +++ b/data/src/lib.rs @@ -6,17 +6,18 @@ //! #![feature(associated_type_defaults)] -pub use self::{misc::*, primitives::*, specs::*, utils::*}; +pub use self::{misc::*, primitives::*, utils::*}; pub(crate) mod misc; pub(crate) mod primitives; -pub(crate) mod specs; pub(crate) mod utils; pub mod datasets; pub mod df; pub mod flows; +pub mod mat; pub mod shape; +pub mod specs; pub mod store; pub mod tensors; @@ -26,13 +27,14 @@ pub mod prelude { // pub use linfa::dataset::{Dataset, DatasetBase, DatasetView}; pub use crate::primitives::*; - pub use crate::specs::*; + pub use crate::utils::*; pub use crate::datasets::*; pub use crate::df::*; pub use crate::flows::*; pub use crate::shape::*; + pub use crate::specs::*; pub use crate::store::*; pub use crate::tensors::*; } diff --git a/data/src/mat/matrix.rs b/data/src/mat/matrix.rs new file mode 100644 index 00000000..dbf2e4d7 --- /dev/null +++ b/data/src/mat/matrix.rs @@ -0,0 +1,8 @@ +/* + Appellation: matrix + Contrib: FL03 +*/ + +pub struct Matrix { + store: Vec>, +} diff --git a/data/src/mat/mod.rs b/data/src/mat/mod.rs new file mode 100644 index 00000000..ed4a755b --- /dev/null +++ b/data/src/mat/mod.rs @@ -0,0 +1,15 @@ +/* + Appellation: mat + Contrib: FL03 +*/ +//! # Matrix +//! +//! A matrix is a two-dimensional array of elements. +pub use self::matrix::*; + +pub(crate) mod matrix; + +pub trait Mat {} + +#[cfg(test)] +mod tests {} diff --git a/data/src/misc/dtype.rs b/data/src/misc/dtype.rs index 15c3e4ac..49fefcff 100644 --- a/data/src/misc/dtype.rs +++ b/data/src/misc/dtype.rs @@ -34,6 +34,78 @@ impl From for DType { } } +impl From for DType { + fn from(_: i8) -> Self { + DType::Integer(Integer::I8) + } +} + +impl From for DType { + fn from(_: i16) -> Self { + DType::Integer(Integer::I16) + } +} + +impl From for DType { + fn from(_: i32) -> Self { + DType::Integer(Integer::I32) + } +} + +impl From for DType { + fn from(_: i64) -> Self { + DType::Integer(Integer::I64) + } +} + +impl From for DType { + fn from(_: i128) -> Self { + DType::Integer(Integer::I128) + } +} + +impl From for DType { + fn from(_: isize) -> Self { + DType::Integer(Integer::ISIZE) + } +} + +impl From for DType { + fn from(_: u8) -> Self { + DType::Unsigned(Unsigned::U8) + } +} + +impl From for DType { + fn from(_: u16) -> Self { + DType::Unsigned(Unsigned::U16) + } +} + +impl From for DType { + fn from(_: u32) -> Self { + DType::Unsigned(Unsigned::U32) + } +} + +impl From for DType { + fn from(_: u64) -> Self { + DType::Unsigned(Unsigned::U64) + } +} + +impl From for DType { + fn from(_: u128) -> Self { + DType::Unsigned(Unsigned::U128) + } +} + +impl From for DType { + fn from(_: usize) -> Self { + DType::Unsigned(Unsigned::USIZE) + } +} + pub enum FloatingPoint { F32, F64, @@ -66,6 +138,48 @@ pub enum Integer { ISIZE, } +impl From for Integer { + fn from(_: i8) -> Self { + Integer::I8 + } +} + +impl From for Integer { + fn from(_: i16) -> Self { + Integer::I16 + } +} + +impl From for Integer { + fn from(_: i32) -> Self { + Integer::I32 + } +} + +impl From for Integer { + fn from(_: i64) -> Self { + Integer::I64 + } +} + +impl From for Integer { + fn from(_: i128) -> Self { + Integer::I128 + } +} + +impl From for Integer { + fn from(_: isize) -> Self { + Integer::ISIZE + } +} + +impl From for DType { + fn from(dtype: Integer) -> Self { + DType::Integer(dtype) + } +} + pub enum Unsigned { U8, U16, @@ -74,3 +188,39 @@ pub enum Unsigned { U128, USIZE, } + +impl From for Unsigned { + fn from(_: u8) -> Self { + Unsigned::U8 + } +} + +impl From for Unsigned { + fn from(_: u16) -> Self { + Unsigned::U16 + } +} + +impl From for Unsigned { + fn from(_: u32) -> Self { + Unsigned::U32 + } +} + +impl From for Unsigned { + fn from(_: u64) -> Self { + Unsigned::U64 + } +} + +impl From for Unsigned { + fn from(_: u128) -> Self { + Unsigned::U128 + } +} + +impl From for Unsigned { + fn from(_: usize) -> Self { + Unsigned::USIZE + } +} diff --git a/data/src/specs/elements.rs b/data/src/specs/elements.rs new file mode 100644 index 00000000..f0bfb9f2 --- /dev/null +++ b/data/src/specs/elements.rs @@ -0,0 +1,4 @@ +/* + Appellation: elements + Contrib: FL03 +*/ diff --git a/data/src/specs.rs b/data/src/specs/mod.rs similarity index 58% rename from data/src/specs.rs rename to data/src/specs/mod.rs index b9598307..602237e6 100644 --- a/data/src/specs.rs +++ b/data/src/specs/mod.rs @@ -1,7 +1,11 @@ /* - Appellation: specs - Contrib: FL03 + Appellation: specs + Contrib: FL03 */ +pub use self::elements::*; + +pub(crate) mod elements; + use ndarray::prelude::{Array1, Array2}; pub trait Records { @@ -30,14 +34,5 @@ impl Records for Array2 { } } -pub trait NdArrayExt {} - -pub trait Store { - fn get(&self, key: &K) -> Option<&V>; - - fn get_mut(&mut self, key: &K) -> Option<&mut V>; - - fn insert(&mut self, key: K, value: V) -> Option; - - fn remove(&mut self, key: &K) -> Option; -} +#[cfg(test)] +mod tests {} diff --git a/data/src/store/mod.rs b/data/src/store/mod.rs index dd17be4f..a14c00b8 100644 --- a/data/src/store/mod.rs +++ b/data/src/store/mod.rs @@ -7,7 +7,15 @@ pub use self::storage::*; pub(crate) mod storage; -pub trait Store {} +pub trait Store { + fn get(&self, key: &K) -> Option<&V>; + + fn get_mut(&mut self, key: &K) -> Option<&mut V>; + + fn insert(&mut self, key: K, value: V) -> Option; + + fn remove(&mut self, key: &K) -> Option; +} #[cfg(test)] mod tests {} diff --git a/data/src/store/storage.rs b/data/src/store/storage.rs index 026e49f8..596e5335 100644 --- a/data/src/store/storage.rs +++ b/data/src/store/storage.rs @@ -4,3 +4,9 @@ */ pub struct Storage {} + +pub enum Rank { + Zero(T), + One(Vec), + N(Vec), +} diff --git a/data/src/tensors/mod.rs b/data/src/tensors/mod.rs index b5116995..e24e0393 100644 --- a/data/src/tensors/mod.rs +++ b/data/src/tensors/mod.rs @@ -3,6 +3,8 @@ Contrib: FL03 */ //! # Tensors +//! +//! A tensor is a generalization of vectors and matrices to potentially higher dimensions. pub use self::tensor::*; pub(crate) mod tensor; diff --git a/ml/linear/src/model/layer.rs b/ml/linear/src/model/layer.rs new file mode 100644 index 00000000..867ce99f --- /dev/null +++ b/ml/linear/src/model/layer.rs @@ -0,0 +1,287 @@ +/* + Appellation: model + Contrib: FL03 +*/ +use crate::neural::func::activate::{Activate, Gradient}; +use crate::neural::prelude::{Features, Forward, Node, Perceptron}; +use crate::params::{LayerShape, LinearParams as LayerParams}; +use ndarray::prelude::{Array2, Ix1, NdFloat}; +use ndarray::ShapeError; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; +use ndarray_stats::DeviationExt; +use num::{Float, Signed}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct Linear>> +where + A: Activate, + T: Float, +{ + activator: A, + features: LayerShape, + name: String, + params: LayerParams, +} + +impl Linear +where + A: Activate, + T: Float, +{ + pub fn new(activator: A, biased: bool, features: LayerShape, name: impl ToString) -> Self { + Self { + activator, + features, + name: name.to_string(), + params: LayerParams::zeros(biased, features), + } + } + + pub fn from_features(inputs: usize, outputs: usize) -> Self + where + A: Default, + { + let features = LayerShape::new(inputs, outputs); + Self { + activator: A::default(), + features, + name: String::new(), + params: LayerParams::zeros(false, features), + } + } + + pub fn activator(&self) -> &A { + &self.activator + } + + pub fn as_dyn(&self) -> Linear>> + where + A: Clone + 'static, + { + Linear { + activator: Box::new(self.activator.clone()), + features: self.features.clone(), + name: self.name.clone(), + params: self.params.clone(), + } + } + + pub fn features(&self) -> &LayerShape { + &self.features + } + + pub fn features_mut(&mut self) -> &mut LayerShape { + &mut self.features + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn params(&self) -> &LayerParams { + &self.params + } + + pub fn params_mut(&mut self) -> &mut LayerParams { + &mut self.params + } + + pub fn set_name(&mut self, name: impl ToString) { + self.name = name.to_string(); + } + + pub fn set_node(&mut self, idx: usize, neuron: &Perceptron) + where + A: Activate, + { + self.params.set_node(idx, neuron.node().clone()); + } + + pub fn reshape(&mut self, inputs: usize, outputs: usize) -> Result<(), ShapeError> { + self.features = LayerShape::new(inputs, outputs); + self.params.reshape(self.features) + } + + pub fn validate_layer(&self, other: &Self, next: bool) -> bool { + if next { + return self.features().inputs() == other.features().outputs(); + } + self.features().outputs() == other.features().inputs() + } + + pub fn with_name(mut self, name: impl ToString) -> Self { + self.name = name.to_string(); + self + } +} + +impl Linear +where + A: Activate, + T: Float + 'static, +{ + pub fn apply_gradient(&mut self, gamma: T, gradient: F) + where + F: Fn(&Array2) -> Array2, + { + let grad = gradient(&self.params.weights()); + self.params.weights_mut().scaled_add(-gamma, &grad); + } + + pub fn update_with_gradient(&mut self, gamma: T, grad: &Array2) { + self.params.weights_mut().scaled_add(-gamma, grad); + } +} + +impl Linear +where + A: Activate, + T: NdFloat, +{ + pub fn linear(&self, args: &Array2) -> Array2 { + self.params().forward(args) + } +} + +impl Linear +where + A: Activate + Gradient, + T: NdFloat + Signed, +{ + pub fn grad(&mut self, gamma: T, args: &Array2, targets: &Array2) -> T { + let ns = T::from(args.shape()[0]).unwrap(); + let pred = self.forward(args); + + let scale = T::from(2).unwrap() * ns; + + let errors = &pred - targets; + let dz = errors * self.activator.gradient(&pred); + let dw = args.t().dot(&dz) / scale; + + self.params_mut().weights_mut().scaled_add(-gamma, &dw.t()); + + let loss = targets + .mean_sq_err(&pred) + .expect("Failed to calculate loss"); + T::from(loss).unwrap() + } +} + +impl Linear +where + A: Activate, + T: Float + SampleUniform, + StandardNormal: Distribution, +{ + pub fn init(mut self, biased: bool) -> Self { + self.params = self.params.init(biased); + self + } +} + +impl Features for Linear +where + A: Activate, + T: Float, +{ + fn inputs(&self) -> usize { + self.features.inputs() + } + + fn outputs(&self) -> usize { + self.features.outputs() + } +} + +// impl Forward> for Layer +// where +// A: Activate, +// D: Dimension, +// T: NdFloat, +// Array: Dot, Output = Array>, +// { +// type Output = Array2; + +// fn forward(&self, args: &Array2) -> Self::Output { +// self.activator.activate(&self.linear(args)) +// } +// } + +impl Forward> for Linear +where + A: Activate, + T: NdFloat, +{ + type Output = Array2; + + fn forward(&self, args: &Array2) -> Self::Output { + self.activator.activate(&self.linear(args)) + } +} + +// impl PartialOrd for Layer +// where +// A: Activate + PartialEq, +// T: Float, +// { +// fn partial_cmp(&self, other: &Self) -> Option { +// self.position.partial_cmp(&other.position) +// } +// } + +// impl From for Layer +// where +// A: Activate + Default, +// S: IntoDimension +// T: Float, +// { +// fn from(features: LayerShape) -> Self { +// Self::new(features, LayerPosition::input()) +// } +// } + +impl From for Linear +where + A: Activate + Default, + T: Float, +{ + fn from(features: LayerShape) -> Self { + Self { + activator: A::default(), + features, + name: String::new(), + params: LayerParams::zeros(false, features), + } + } +} + +impl IntoIterator for Linear +where + A: Activate + Default, + T: Float, +{ + type Item = Node; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.params.into_iter() + } +} + +impl FromIterator> for Linear +where + A: Activate + Default, + T: Float, +{ + fn from_iter>>(nodes: I) -> Self { + let params = LayerParams::from_iter(nodes); + Self { + activator: A::default(), + features: *params.features(), + name: String::new(), + params, + } + } +} diff --git a/ml/linear/src/model/mod.rs b/ml/linear/src/model/mod.rs index a8eee288..c86b4d21 100644 --- a/ml/linear/src/model/mod.rs +++ b/ml/linear/src/model/mod.rs @@ -4,7 +4,51 @@ */ //! # Linear Model //! -pub use self::{config::*, module::*}; +pub use self::{config::*, layer::*, module::*}; pub(crate) mod config; +pub(crate) mod layer; pub(crate) mod module; + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::prelude::linarr; + use crate::neural::prelude::{Forward, Node, Softmax}; + use crate::params::LayerShape; + use ndarray::prelude::Ix2; + + #[test] + fn test_linear() { + let (samples, inputs, outputs) = (20, 5, 3); + let features = LayerShape::new(inputs, outputs); + + let args = linarr::((samples, inputs)).unwrap(); + + let layer = Linear::::from(features).init(true); + + let pred = layer.forward(&args); + + assert_eq!(pred.dim(), (samples, outputs)); + + let nodes = (0..outputs) + .map(|_| Node::::new(inputs).init(true)) + .collect::>(); + let layer = Linear::::from_iter(nodes); + assert_eq!(layer.features(), &features); + } + + #[test] + fn test_linear_iter() { + let (_samples, inputs, outputs) = (20, 5, 3); + let features = LayerShape::new(inputs, outputs); + + let layer = Linear::::from(features).init(true); + + for node in layer.into_iter() { + assert!(node.is_biased()); + assert_eq!(node.features(), inputs); + assert_eq!(node.bias().as_ref().unwrap().dim(), ()); + } + } +} diff --git a/ml/linear/src/params/store.rs b/ml/linear/src/params/store.rs index 23efc53e..0b08755c 100644 --- a/ml/linear/src/params/store.rs +++ b/ml/linear/src/params/store.rs @@ -7,9 +7,10 @@ use crate::core::prelude::GenerateRandom; use crate::neural::prelude::{Features, Forward, Node}; use ndarray::linalg::Dot; use ndarray::prelude::{Array, Array1, Array2, Axis, Dimension, NdFloat}; +use ndarray::{LinalgScalar, ShapeError}; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, StandardNormal}; -use num::Float; +use num::{Float, Num, Signed}; use serde::{Deserialize, Serialize}; use std::ops; @@ -20,25 +21,13 @@ pub struct LinearParams { weights: Array2, } -impl LinearParams -where - T: Float, -{ - pub fn new(biased: bool, features: LayerShape) -> Self { - let bias = if biased { - Some(Array1::zeros(features.outputs())) - } else { - None - }; - Self { - bias, - features, - weights: Array2::zeros(features.out_by_in()), - } +impl LinearParams { + pub fn bias(&self) -> Option<&Array1> { + self.bias.as_ref() } - pub fn biased(features: LayerShape) -> Self { - Self::new(true, features) + pub fn bias_mut(&mut self) -> Option<&mut Array1> { + self.bias.as_mut() } pub fn features(&self) -> &LayerShape { @@ -53,23 +42,32 @@ where self.bias.is_some() } - pub fn set_node(&mut self, idx: usize, node: Node) { - if let Some(bias) = node.bias() { - if !self.is_biased() { - let mut tmp = Array1::zeros(self.features().outputs()); - tmp.index_axis_mut(Axis(0), idx).assign(bias); - self.bias = Some(tmp); - } - self.bias - .as_mut() - .unwrap() - .index_axis_mut(Axis(0), idx) - .assign(bias); + pub fn reshape(&mut self, features: LayerShape) -> Result<(), ShapeError> + where + T: Clone, + { + self.features = features; + self.weights = self.weights().clone().into_shape(features.out_by_in())?; + if let Some(bias) = self.bias_mut() { + *bias = bias.clone().into_shape(features.outputs())?; } + Ok(()) + } - self.weights_mut() - .index_axis_mut(Axis(0), idx) - .assign(&node.weights()); + pub fn set_bias(&mut self, bias: Option>) { + self.bias = bias; + } + + pub fn set_weights(&mut self, weights: Array2) { + self.weights = weights; + } + + pub fn weights(&self) -> &Array2 { + &self.weights + } + + pub fn weights_mut(&mut self) -> &mut Array2 { + &mut self.weights } pub fn with_bias(mut self, bias: Option>) -> Self { @@ -81,53 +79,74 @@ where self.weights = weights; self } +} - pub fn bias(&self) -> Option<&Array1> { - self.bias.as_ref() +impl LinearParams +where + T: Clone + Num, +{ + pub fn new(bias: Option>, weights: Array2) -> Self { + let features = LayerShape::new(weights.ncols(), weights.nrows()); + Self { + bias, + features, + weights, + } } - pub fn bias_mut(&mut self) -> Option<&mut Array1> { - self.bias.as_mut() + pub fn zeros(biased: bool, features: LayerShape) -> Self { + let bias = if biased { + Some(Array1::zeros(features.outputs())) + } else { + None + }; + Self { + bias, + features, + weights: Array2::zeros(features.out_by_in()), + } } - pub fn set_bias(&mut self, bias: Option>) { - self.bias = bias; + pub fn biased(features: LayerShape) -> Self { + Self::zeros(true, features) } - pub fn set_weights(&mut self, weights: Array2) { - self.weights = weights; + pub fn reset(&mut self) { + if let Some(bias) = self.bias_mut() { + *bias = Array1::zeros(bias.dim()); + } + self.weights = Array2::zeros(self.weights.dim()); } - pub fn weights(&self) -> &Array2 { - &self.weights - } + pub fn set_node(&mut self, idx: usize, node: Node) { + if let Some(bias) = node.bias() { + if !self.is_biased() { + let mut tmp = Array1::zeros(self.features().outputs()); + tmp.index_axis_mut(Axis(0), idx).assign(bias); + self.bias = Some(tmp); + } + self.bias + .as_mut() + .unwrap() + .index_axis_mut(Axis(0), idx) + .assign(bias); + } - pub fn weights_mut(&mut self) -> &mut Array2 { - &mut self.weights + self.weights_mut() + .index_axis_mut(Axis(0), idx) + .assign(&node.weights()); } } impl LinearParams where - T: Float + 'static, + T: LinalgScalar + Signed, { pub fn update_with_gradient(&mut self, gamma: T, gradient: &Array2) { self.weights_mut().scaled_add(-gamma, gradient); } } -impl LinearParams -where - T: NdFloat, -{ - pub fn reset(&mut self) { - if let Some(bias) = self.bias_mut() { - *bias = Array1::zeros(bias.dim()); - } - self.weights *= T::zero(); - } -} - impl LinearParams where T: Float + SampleUniform, @@ -153,10 +172,7 @@ where } } -impl Features for LinearParams -where - T: Float, -{ +impl Features for LinearParams { fn inputs(&self) -> usize { self.features.inputs() } @@ -217,7 +233,7 @@ where let mut iter = nodes.iter(); let node = iter.next().unwrap(); let shape = LayerShape::new(node.features(), nodes.len()); - let mut params = Self::new(true, shape); + let mut params = Self::zeros(true, shape); params.set_node(0, node.clone()); for (i, node) in iter.into_iter().enumerate() { params.set_node(i + 1, node.clone()); diff --git a/ml/neural/src/layers/seq/sequential.rs b/ml/neural/src/layers/seq/sequential.rs index 1a50b186..e876a330 100644 --- a/ml/neural/src/layers/seq/sequential.rs +++ b/ml/neural/src/layers/seq/sequential.rs @@ -2,5 +2,171 @@ Appellation: sequential Contrib: FL03 */ +use crate::prelude::Forward; +use serde::{Deserialize, Serialize}; -pub struct Sequential {} +pub struct Sequential { + layers: Vec>>, +} + +impl Sequential { + pub fn new() -> Self { + Self { layers: Vec::new() } + } + + pub fn include(mut self, layer: L) -> Self + where + L: Forward + 'static, + { + self.layers.push(Box::new(layer)); + self + } + + pub fn push(&mut self, layer: L) + where + L: Forward + 'static, + { + self.layers.push(Box::new(layer)); + } +} + +impl AsRef<[Box>]> for Sequential { + fn as_ref(&self) -> &[Box>] { + &self.layers + } +} + +impl AsMut<[Box>]> for Sequential { + fn as_mut(&mut self) -> &mut [Box>] { + &mut self.layers + } +} + +impl Extend>> for Sequential { + fn extend>>>(&mut self, iter: I) { + self.layers.extend(iter); + } +} + +impl Forward for Sequential +where + T: Clone, +{ + type Output = T; + + fn forward(&self, input: &T) -> Self::Output { + let mut output = input.clone(); + for layer in &self.layers { + output = layer.forward(&output); + } + output + } +} + +impl FromIterator>> for Sequential { + fn from_iter>>>(iter: I) -> Self { + Self { + layers: Vec::from_iter(iter), + } + } +} + +impl IntoIterator for Sequential { + type Item = Box>; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.layers.into_iter() + } +} + +impl Clone for Sequential +where + Box>: Clone, +{ + fn clone(&self) -> Self { + Self { + layers: self.layers.clone(), + } + } +} + +impl Default for Sequential { + fn default() -> Self { + Self::new() + } +} + +impl std::fmt::Debug for Sequential +where + Box>: std::fmt::Debug, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Sequential") + .field("layers", &self.layers) + .finish() + } +} + +impl PartialEq for Sequential +where + Box>: PartialEq, +{ + fn eq(&self, other: &Self) -> bool { + self.layers == other.layers + } +} + +impl Eq for Sequential where Box>: Eq {} + +impl std::hash::Hash for Sequential +where + Box>: std::hash::Hash, +{ + fn hash(&self, state: &mut H) { + self.layers.hash(state); + } +} + +impl PartialOrd for Sequential +where + Box>: PartialOrd, +{ + fn partial_cmp(&self, other: &Self) -> Option { + self.layers.partial_cmp(&other.layers) + } +} + +impl Ord for Sequential +where + Box>: Ord, +{ + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.layers.cmp(&other.layers) + } +} + +impl<'a, T> Deserialize<'a> for Sequential +where + Box>: Deserialize<'a>, +{ + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'a>, + { + let layers = Vec::>>::deserialize(deserializer)?; + Ok(Self { layers }) + } +} + +impl Serialize for Sequential +where + Box>: Serialize, +{ + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + self.layers.serialize(serializer) + } +} diff --git a/ml/neural/src/neurons/node.rs b/ml/neural/src/neurons/node.rs index 26885bb7..c8d0be59 100644 --- a/ml/neural/src/neurons/node.rs +++ b/ml/neural/src/neurons/node.rs @@ -10,7 +10,7 @@ use ndarray::prelude::{Array, Array0, Array1, Array2, Dimension, NdFloat}; use ndarray::{RemoveAxis, ScalarOperand}; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, StandardNormal}; -use num::Float; +use num::{Float, Num}; use std::ops; #[derive(Clone, Debug, PartialEq)] @@ -22,7 +22,7 @@ pub struct Node { impl Node where - T: Float, + T: Clone + Num, { pub fn create(biased: bool, features: usize) -> Self { let bias = if biased { @@ -44,7 +44,11 @@ where pub fn new(features: usize) -> Self { Self::create(false, features) } - +} +impl Node +where + T: Num, +{ pub fn bias(&self) -> Option<&Array0> { self.bias.as_ref() } @@ -61,18 +65,6 @@ where self.bias.is_some() } - pub fn linear(&self, data: &Array2) -> Array1 - where - T: 'static, - { - let w = self.weights().t().to_owned(); - if let Some(bias) = self.bias() { - data.dot(&w) + bias - } else { - data.dot(&w) - } - } - pub fn set_bias(&mut self, bias: Option>) { self.bias = bias; } @@ -109,6 +101,20 @@ where } } +impl Node +where + T: Num + ScalarOperand + 'static, + Array2: Dot, Output = Array1>, +{ + pub fn linear(&self, data: &Array2) -> Array1 { + let w = self.weights().t().to_owned(); + if let Some(bias) = self.bias() { + data.dot(&w) + bias + } else { + data.dot(&w) + } + } +} impl Node where T: Float + SampleUniform, diff --git a/ml/neural/src/nn/ffn/mlp.rs b/ml/neural/src/nn/ffn/mlp.rs index dfbd41bb..bf1d4d45 100644 --- a/ml/neural/src/nn/ffn/mlp.rs +++ b/ml/neural/src/nn/ffn/mlp.rs @@ -7,6 +7,7 @@ use crate::func::activate::{Activate, Linear, ReLU, Softmax}; use crate::layers::{Layer, LayerShape, Stack}; + use crate::prelude::{Features, Forward}; use ndarray::prelude::{Array2, Ix2, NdFloat}; diff --git a/ml/neural/src/nn/mod.rs b/ml/neural/src/nn/mod.rs index b1366c57..912ea228 100644 --- a/ml/neural/src/nn/mod.rs +++ b/ml/neural/src/nn/mod.rs @@ -3,15 +3,14 @@ Contrib: FL03 */ //! # Neural Network -pub use self::{kinds::*, position::*, sequential::*, utils::*}; +pub use self::{kinds::*, position::*}; pub(crate) mod kinds; pub(crate) mod position; -pub(crate) mod sequential; pub mod cnn; pub mod ffn; pub mod gnn; pub mod rnn; -pub(crate) mod utils {} +pub trait NeuralNetwork {} diff --git a/ml/neural/src/nn/sequential.rs b/ml/neural/src/nn/sequential.rs deleted file mode 100644 index b3884a9e..00000000 --- a/ml/neural/src/nn/sequential.rs +++ /dev/null @@ -1,12 +0,0 @@ -/* - Appellation: sequential - Contrib: FL03 -*/ - -pub struct Sequential {} - -impl Sequential { - pub fn new() -> Self { - Self {} - } -} diff --git a/ml/neural/src/ops/mod.rs b/ml/neural/src/ops/mod.rs index e8b3d4cc..ad9f5fee 100644 --- a/ml/neural/src/ops/mod.rs +++ b/ml/neural/src/ops/mod.rs @@ -2,18 +2,16 @@ Appellation: ops Contrib: FL03 */ -pub use self::{dropout::*, norm::*, utils::*}; +pub use self::{dropout::*, norm::*}; pub(crate) mod dropout; pub(crate) mod norm; -pub(crate) mod utils {} - #[cfg(test)] mod tests { use super::*; + use crate::core::prelude::RoundTo; use crate::prelude::Forward; - use concision_core::prelude::RoundTo; use ndarray::prelude::{array, Array, Ix2}; #[test] diff --git a/ml/neural/src/ops/norm.rs b/ml/neural/src/ops/norm.rs index 53f44b23..167017be 100644 --- a/ml/neural/src/ops/norm.rs +++ b/ml/neural/src/ops/norm.rs @@ -5,15 +5,26 @@ use crate::prelude::Forward; use ndarray::prelude::{Array, Axis, Dimension, Ix2, NdFloat}; use ndarray::{IntoDimension, RemoveAxis}; -use num::{Float, FromPrimitive}; +use num::{FromPrimitive, Num}; use serde::{Deserialize, Serialize}; -use std::ops::{Add, Div, Mul}; -pub fn norm(x: &Array, axis: usize) -> Array +pub fn norm(x: &Array) -> Array where - D: Dimension + RemoveAxis, + D: Dimension, + T: FromPrimitive + NdFloat, +{ + let epsilon = T::from(1e-6).unwrap(); + // Calculate the mean and standard deviation of the activations along the feature axis. + let mean = x.mean().expect("mean_axis failed"); + + let std = x.std(T::one()); + (x.clone() - mean) / (std + epsilon) +} + +pub fn norma(x: &Array, axis: usize) -> Array +where + D: RemoveAxis, T: FromPrimitive + NdFloat, - Array: Div, Output = Array>, { let axis = Axis(axis); let epsilon = T::from(1e-6).unwrap(); @@ -32,8 +43,6 @@ pub fn norm_and_scale( where D: Dimension, T: FromPrimitive + NdFloat, - Array: - Add, Output = Array> + Mul, Output = Array>, { let epsilon = T::from(1e-6).unwrap(); // Calculate the mean and standard deviation of the activations along the feature axis. @@ -48,8 +57,8 @@ where #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] pub struct LayerNorm where - T: Float, D: Dimension, + T: Num, { alpha: Array, beta: Array, @@ -57,8 +66,8 @@ where impl LayerNorm where - T: Float, - D: Dimension + RemoveAxis, + D: Dimension, + T: Clone + Num, { pub fn new(dim: impl IntoDimension) -> Self { let dim = dim.into_dimension(); @@ -89,7 +98,6 @@ impl Forward> for LayerNorm where D: Dimension, T: FromPrimitive + NdFloat, - Array: Add, Output = Array> + Mul, Output = Array>, { type Output = Array; @@ -101,6 +109,6 @@ where let norm = (data - mean) / (data.std(T::one()) + epsilon); // Scale and shift the normalized activations with learnable parameters alpha and beta. - norm * self.alpha.clone() + self.beta.clone() + norm * self.alpha() + self.beta() } } diff --git a/ml/s4/src/hippo/dplr.rs b/ml/s4/src/hippo/dplr.rs index 227b7e94..86daf7d6 100644 --- a/ml/s4/src/hippo/dplr.rs +++ b/ml/s4/src/hippo/dplr.rs @@ -5,20 +5,24 @@ //! # Diagonal Plus Low Rank (DPLR) //! //! -use super::utils::*; +use super::nplr::NPLR; +use crate::core::prelude::{Conjugate, SquareRoot}; use crate::prelude::S4Float; -use ndarray::prelude::{Array1, Array2}; +use ndarray::prelude::{Array1, Array2, Axis}; use ndarray::{LinalgScalar, ScalarOperand}; -use num::complex::ComplexFloat; -use num::{Complex, Num}; +use ndarray_linalg::{Eigh, IntoTriangular, Lapack, Scalar, UPLO}; +use num::complex::{Complex, ComplexFloat}; +use num::{FromPrimitive, Num, One, Signed}; use serde::{Deserialize, Serialize}; +use std::ops::{Add, Mul, Neg}; -#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +// #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[derive(Clone, Debug, PartialEq)] pub struct DPLR where - T: Clone + Num, + T: Clone + Num + Scalar, { - pub lambda: Array2>, + pub lambda: Array2::Real>>, pub p: Array1, pub b: Array1, pub v: Array2, @@ -26,10 +30,60 @@ where impl DPLR where - T: S4Float, - Complex<::Real>: LinalgScalar + ScalarOperand, + T: Conjugate + Lapack + Scalar + ScalarOperand + Signed + SquareRoot, + T: Add::Real>, Output = Complex<::Real>> + + Mul::Real>, Output = Complex<::Real>>, + Complex<::Real>: Mul::Real>>, + Complex<::Real>: + Mul::Real>, Output = Complex<::Real>>, + Complex: Add::Real>, Output = Complex> + + Mul::Real>, Output = Complex>, { - pub fn new(features: usize) -> Self { - make_dplr_hippo(features) + pub fn create(features: usize) -> Self { + let (a, p, b) = NPLR::::new(features).into(); + + // + let s = { + let p2 = p.clone().insert_axis(Axis(1)); + &a + p2.dot(&p2.t()) + }; + // + let sd = s.diag().mean().expect("Average of diagonal is NaN"); + + let a = Array2::ones(s.dim()) * sd; + + // TODO: Fix this + // let (ee, vv) = { + // let si = + // }; + // let (e, v) = s.mapv(|i: T| T::from(Complex::new(i.re(), i.im()) * Complex::i().neg()).unwrap()) + // .eigh(UPLO::Lower) + // .expect(""); + let (e, v) = s.conj().eigh(UPLO::Lower).expect(""); + + // let a = a + &e * Complex::new(<::Real>::one(), <::Real>::one()); + let a = a + e.mapv(|i: ::Real| { + Complex::new(i.re(), i.im()) + * Complex::new(::Real::one(), ::Real::one().neg()) + }); + let p = v.conj().t().dot(&p); + let b = v.conj().t().dot(&b); + Self { + lambda: a, + p, + b, + v: v.clone(), + } } } + +// impl DPLR +// where +// T: S4Float, +// ::Real: FromPrimitive, +// Complex<::Real>: LinalgScalar + ScalarOperand, +// { +// pub fn new(features: usize) -> Self { +// make_dplr_hippo(features) +// } +// } diff --git a/ml/s4/src/hippo/hippo.rs b/ml/s4/src/hippo/hippo.rs index 7e22e1cd..c4c6aac9 100644 --- a/ml/s4/src/hippo/hippo.rs +++ b/ml/s4/src/hippo/hippo.rs @@ -2,30 +2,26 @@ Appellation: hippo Contrib: FL03 */ -use super::dplr::DPLR; +// use super::dplr::DPLR; use super::nplr::NPLR; -use crate::core::prelude::{rangespace, Conjugate}; -use crate::prelude::S4Float; -use ndarray::prelude::{Array2, Axis}; -use ndarray::{LinalgScalar, ScalarOperand}; -use ndarray_linalg::{Eigh, IntoTriangular, UPLO}; -use num::complex::{Complex, ComplexFloat}; -use num::traits::Num; +use super::utils::genspace; +use crate::core::prelude::SquareRoot; +use ndarray::prelude::Array2; +use ndarray::ScalarOperand; +use ndarray_linalg::{IntoTriangular, UPLO}; +use num::traits::{Num, NumCast, Signed}; // use num::traits::{Float, FloatConst}; use serde::{Deserialize, Serialize}; -pub enum HiPPOs -where - T: Clone + Num, -{ +pub enum HiPPOs { HiPPO(HiPPO), - DPLR(DPLR), + // DPLR(DPLR), NPLR(NPLR), } impl HiPPOs where - T: ComplexFloat + ScalarOperand, + T: Num + NumCast + ScalarOperand + Signed + SquareRoot, { pub fn new(features: usize) -> Self { Self::HiPPO(HiPPO::new(features)) @@ -36,15 +32,15 @@ where } } -impl HiPPOs -where - T: S4Float, - Complex<::Real>: LinalgScalar + ScalarOperand, -{ - pub fn dplr(features: usize) -> Self { - Self::DPLR(DPLR::new(features)) - } -} +// impl HiPPOs +// where +// T: S4Float, +// Complex<::Real>: LinalgScalar + ScalarOperand, +// { +// pub fn dplr(features: usize) -> Self { +// Self::DPLR(DPLR::new(features)) +// } +// } #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] pub struct HiPPO { @@ -60,19 +56,17 @@ impl HiPPO { impl HiPPO where - T: ComplexFloat + ScalarOperand, + T: Num + NumCast + ScalarOperand + Signed + SquareRoot, { pub fn new(features: usize) -> Self { - let base = rangespace((features, 1)); - let p = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); - let mut a = &p * &p.t(); - a = &a.into_triangular(UPLO::Lower) - &base.diag(); - - Self { features, data: -a } + Self { + features, + data: super::hippo(features), + } } pub fn nplr(&self) -> NPLR { - let base = rangespace(self.features()); + let base = genspace(self.features()); let p = (&base + T::one() / T::from(2).unwrap()).mapv(T::sqrt); let b = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); NPLR { @@ -83,40 +77,40 @@ where } } -impl HiPPO -where - T: S4Float, -{ - pub fn dplr(&self) -> DPLR { - let (a, p, b) = super::make_nplr_hippo(self.features).into(); - - // - let s = &a - + p.clone() - .insert_axis(Axis(1)) - .dot(&p.clone().insert_axis(Axis(0))); - // - let sd = s.diag(); - - let a = Array2::ones(s.dim()) * sd.mean().expect("Average of diagonal is NaN"); - - // TODO: replace with eigh - let (e, v) = &(&s * T::from(T::one().neg().as_imag()).unwrap()) - .eigh(UPLO::Lower) - .expect(""); - let e = e.mapv(|x| T::from(x).unwrap()); - - let a = a + &e * T::from(T::one().as_imag()).unwrap(); - let p = v.conj().t().dot(&p); - let b = v.conj().t().dot(&b); - DPLR { - lambda: a, - p, - b, - v: v.clone(), - } - } -} +// impl HiPPO +// where +// T: S4Float, +// { +// pub fn dplr(&self) -> DPLR { +// let (a, p, b) = super::make_nplr_hippo(self.features).into(); + +// // +// let s = &a +// + p.clone() +// .insert_axis(Axis(1)) +// .dot(&p.clone().insert_axis(Axis(0))); +// // +// let sd = s.diag(); + +// let a = Array2::ones(s.dim()) * sd.mean().expect("Average of diagonal is NaN"); + +// // TODO: replace with eigh +// let (e, v) = &(&s * T::from(T::one().neg().as_im()).unwrap()) +// .eigh(UPLO::Lower) +// .expect(""); +// let e = e.mapv(|x| T::from(x).unwrap()); + +// let a = a + &e * T::from(T::one().as_im()).unwrap(); +// let p = v.conj().t().dot(&p); +// let b = v.conj().t().dot(&b); +// DPLR { +// lambda: a, +// p, +// b, +// v: v.clone(), +// } +// } +// } impl AsRef> for HiPPO { fn as_ref(&self) -> &Array2 { diff --git a/ml/s4/src/hippo/mod.rs b/ml/s4/src/hippo/mod.rs index 7fbc8506..4ecc89d6 100644 --- a/ml/s4/src/hippo/mod.rs +++ b/ml/s4/src/hippo/mod.rs @@ -20,20 +20,35 @@ pub struct LowRank { pub(crate) mod utils { use super::dplr::DPLR; use super::nplr::NPLR; - use crate::core::prelude::{rangespace, Conjugate}; + use crate::core::prelude::{rangespace, Conjugate, SquareRoot}; use crate::prelude::S4Float; - use ndarray::prelude::{Array2, Axis}; + use ndarray::prelude::{Array1, Array2, Axis}; use ndarray::{LinalgScalar, ScalarOperand}; - use ndarray_linalg::{Eigh, IntoTriangular, UPLO}; + use ndarray_linalg::{Eigh, IntoTriangular, Scalar, UPLO}; use num::complex::{Complex, ComplexFloat}; - // use num::traits::{Float, FloatConst}; - use std::ops::Neg; + use num::traits::{FromPrimitive, Num, NumCast, Signed}; + use std::ops; + + pub fn genspace(features: usize) -> Array1 { + Array1::from_iter((0..features).map(|x| T::from(x).unwrap())) + } + + pub(crate) fn hippo(features: usize) -> Array2 + where + T: Num + NumCast + ScalarOperand + Signed + SquareRoot, + { + let base = genspace::(features).insert_axis(Axis(1)); + let p = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); + let mut a = &p * &p.t(); + a = &a.into_triangular(UPLO::Lower) - &Array2::from_diag(&genspace::(features)); + -a + } pub(crate) fn make_hippo(features: usize) -> Array2 where T: ComplexFloat + ScalarOperand, { - let base = rangespace(features).insert_axis(Axis(1)); + let base = rangespace((features, 1)); let p = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); let mut a = &p * &p.t(); a = &a.into_triangular(UPLO::Lower) - &base.diag(); @@ -55,34 +70,58 @@ pub(crate) mod utils { pub(crate) fn make_dplr_hippo(features: usize) -> DPLR where T: S4Float, + ::Real: FromPrimitive, + Complex<::Real>: + ops::Mul::Real>, Output = Complex<::Real>>, Complex<::Real>: LinalgScalar + ScalarOperand, + Array2::Real>>: ops::Add< + Array1::Real>>, + Output = Array2::Real>>, + > + ops::Mul< + Array2<::Real>, + Output = Array2::Real>>, + > + ops::Mul< + Array2::Real>>, + Output = Array2::Real>>, + > + ops::Mul< + Array2::Real>>, + Output = Array2::Real>>, + >, { - let (a, p, b) = make_nplr_hippo(features).into(); + let (a, p, b) = make_nplr_hippo::(features).into(); // - let s = &a - + p.clone() - .insert_axis(Axis(1)) - .dot(&p.clone().insert_axis(Axis(0))); + let s = { + let p2 = p.clone().insert_axis(Axis(1)); + &a + p2.dot(&p2.t()) + }; // - let sd = s.diag(); - - let a = Array2::ones(s.dim()) * sd.mean().expect("Average of diagonal is NaN"); - - // TODO: replace with eigh - let (e, v) = &(&s * T::from((Complex::<::Real>::i().neg())).unwrap()) - .eigh(UPLO::Lower) - .expect(""); - let e = e.mapv(|x| T::from(x).unwrap()); - - let a = a + &e * T::from(T::one().as_imag()).unwrap(); - let p = v.conj().t().dot(&p); - let b = v.conj().t().dot(&b); + let sd = s.diag().mapv(|i: T| Complex::new(i.re(), i.im())); + + let a = Array2::::Real>>::ones(s.dim()) + * sd.mean().expect("Average of diagonal is NaN"); + + // TODO: Fix this + // let (ee, vv) = { + // let si = + // }; + // let (e, v) = s.mapv(|i: T| T::from(Complex::new(i.re(), i.im()) * Complex::i().neg()).unwrap()) + // .eigh(UPLO::Lower) + // .expect(""); + let (e, v) = s.conj().eigh(UPLO::Lower).expect(""); + // compute the conjugate transpose of the eigenvectors + + // let a = a + &e * Complex::new(<::Real>::one(), <::Real>::one()); + // let a = a + e.mapv(|i: ::Real| Complex::new(i.re(), i.im()) * Complex::i()); + let lambda = { + let a = Array2::ones(s.dim()); + a + }; DPLR { - lambda: a, - p, - b, - v: v.clone(), + lambda, + p: v.conj().t().dot(&p), + b: v.conj().t().dot(&b), + v, } } } @@ -90,38 +129,66 @@ pub(crate) mod utils { #[cfg(test)] mod tests { use super::*; + use crate::core::prelude::Conjugate; use dplr::DPLR; use nplr::NPLR; - use num::Complex; + + use ndarray::prelude::{Array, Axis, Dimension}; + use ndarray::ScalarOperand; + use ndarray_linalg::aclose; + use num::{Num, Signed}; #[test] fn test_hippo() { let features = 10; - let a = make_hippo::(features); + let a = hippo::(features); let b = HiPPO::::new(features); assert_eq!(&a, b.as_ref()); } + fn close(a: &Array, b: &Array, atol: T) + where + D: Dimension, + T: Num + ScalarOperand + Signed + PartialOrd + std::fmt::Debug, + { + (a - b).for_each(|i| assert!(i.abs() <= atol, "Actual: {:?}\nTolerance: {:?}", i, atol)) + } + #[test] fn test_dplr() { let features = 10; - let a = make_dplr_hippo::>(features); - let b = HiPPO::>::new(features).dplr(); - let c = DPLR::>::new(features); - assert_eq!(&a, &b); - assert_eq!(&a, &c); - } + // let a = make_dplr_hippo::>(features); + // let b = HiPPO::>::new(features).dplr(); + let nplr = NPLR::::new(features); + let dplr = DPLR::::create(features); - #[test] - fn test_nplr() { - let features = 10; + let hippo = nplr.a.clone(); + + let v = dplr.v.clone(); + let vc = v.conj().t().to_owned(); - let a = make_nplr_hippo::(features); - let b = HiPPO::::new(features).nplr(); - let c = NPLR::::new(features); - assert_eq!(&a, &b); - assert_eq!(&a, &c); + let lambda = dplr.lambda.diag().to_owned().mapv(|i| i.re); + + let p = nplr.p.insert_axis(Axis(1)); + + let pc = dplr.p.insert_axis(Axis(1)); + + let a = v.dot(&lambda).dot(&vc) - &p.dot(&p.t()); + let b = v.dot(&(lambda - pc.dot(&pc.t()))).dot(&vc); + + close(&hippo, &a, 1e-4); + // close(&hippo, &b, 1e-4); + // assert_eq!(&a, &b); } + + // #[test] + // fn test_nplr() { + // let features = 10; + + // let a = HiPPO::::new(features).nplr(); + // let b = NPLR::::new(features); + // assert_eq!(&a, &b); + // } } diff --git a/ml/s4/src/hippo/nplr.rs b/ml/s4/src/hippo/nplr.rs index 6beb48bf..b3835761 100644 --- a/ml/s4/src/hippo/nplr.rs +++ b/ml/s4/src/hippo/nplr.rs @@ -7,12 +7,25 @@ //! use super::utils::*; +use crate::core::prelude::{rangespace, SquareRoot}; use ndarray::prelude::{Array1, Array2}; use ndarray::ScalarOperand; use num::complex::ComplexFloat; -// use num::traits::{Float, FloatConst}; +use num::traits::{Num, NumCast, Signed}; use serde::{Deserialize, Serialize}; +fn nplr(features: usize) -> (Array2, Array1, Array1) +where + T: Num + NumCast + ScalarOperand + Signed + SquareRoot, +{ + let hippo = hippo::(features); + + let base = genspace::(features); + let p = (&base + (T::one() / T::from(2).unwrap())).mapv(T::sqrt); + let b = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); + (hippo, p, b) +} + #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] pub struct NPLR { pub a: Array2, @@ -22,10 +35,10 @@ pub struct NPLR { impl NPLR where - T: ComplexFloat + ScalarOperand, + T: Num + NumCast + ScalarOperand + Signed + SquareRoot, { pub fn new(features: usize) -> Self { - make_nplr_hippo(features) + nplr(features).into() } } diff --git a/ml/s4/src/primitives.rs b/ml/s4/src/primitives.rs index 09ef539b..40a2b06b 100644 --- a/ml/s4/src/primitives.rs +++ b/ml/s4/src/primitives.rs @@ -5,19 +5,40 @@ pub use self::constants::*; use crate::core::specs::{Arithmetic, AsComplex, Conjugate}; use ndarray::ScalarOperand; -use ndarray_linalg::Lapack; -use num::complex::ComplexFloat; +use ndarray_linalg::{Lapack, Scalar}; +use num::complex::{Complex, ComplexFloat}; use num::traits::FromPrimitive; // use num::traits::{Float, FromPrimitive, FloatConst}; use std::ops; -pub trait S4Float: - Arithmetic + AsComplex + Conjugate + ComplexFloat + FromPrimitive + Lapack + ScalarOperand +pub trait S4Float: Arithmetic + + AsComplex + + Conjugate + + ComplexFloat + + FromPrimitive + + Lapack + + Scalar + + ScalarOperand + + ops::Mul::Real>, Output = Complex<::Real>> +where + ::Real: FromPrimitive, { } -impl S4Float for T where - T: AsComplex + Conjugate + ComplexFloat + FromPrimitive + Lapack + ScalarOperand +impl S4Float for T +where + T: Arithmetic + + Arithmetic< + Complex<::Real>, + Output = Complex<::Real>, + > + AsComplex + + Conjugate + + ComplexFloat + + FromPrimitive + + Lapack + + ScalarOperand + + ops::Mul::Real>, Output = Complex<::Real>>, + ::Real: FromPrimitive, { } diff --git a/ml/s4/src/specs.rs b/ml/s4/src/specs.rs index 8935e73a..23832f76 100644 --- a/ml/s4/src/specs.rs +++ b/ml/s4/src/specs.rs @@ -37,7 +37,7 @@ where fn fft(&self, args: &Self) -> Self::Output { let dim = args.dim(); let mut out = Self::ones(args.dim()); - let mut buffer = vec![T::zero().as_complex(); args.len()]; + let mut buffer = vec![T::zero().as_re(); args.len()]; let mut planner = FftPlanner::new(); let fft = planner.plan_fft_forward(args.len()); fft.process(buffer.as_mut_slice()); @@ -51,7 +51,7 @@ where fn ifft(&self, args: &Self) -> Self::Output { let dim = args.dim(); let mut out = Self::ones(args.dim()); - let mut buffer = vec![T::zero().as_complex(); args.len()]; + let mut buffer = vec![T::zero().as_re(); args.len()]; let mut planner = FftPlanner::new(); let fft = planner.plan_fft_inverse(args.len()); fft.process(buffer.as_mut_slice()); From 3c523b3d774d5a8ed611c193103911533cf28095 Mon Sep 17 00:00:00 2001 From: FL03 Date: Wed, 3 Jan 2024 19:08:21 -0600 Subject: [PATCH 102/118] update Signed-off-by: FL03 --- ml/s4/Cargo.toml | 2 +- ml/s4/src/hippo/dplr.rs | 130 +++++++++++++++++-------------- ml/s4/src/hippo/mod.rs | 164 +++++++++++----------------------------- ml/s4/src/hippo/nplr.rs | 3 +- 4 files changed, 121 insertions(+), 178 deletions(-) diff --git a/ml/s4/Cargo.toml b/ml/s4/Cargo.toml index 8c67013a..368e3388 100644 --- a/ml/s4/Cargo.toml +++ b/ml/s4/Cargo.toml @@ -66,7 +66,7 @@ concision-neural = { features = ["blas"], path = "../neural" } anyhow.workspace = true lazy_static.workspace = true -ndarray = { features = ["blas", "serde-1"], version = "0.15" } +ndarray = { features = ["approx", "blas", "serde-1"], version = "0.15" } ndarray-conv = "0.2" ndarray-linalg = { features = [], version = "0.16" } ndarray-rand.workspace = true diff --git a/ml/s4/src/hippo/dplr.rs b/ml/s4/src/hippo/dplr.rs index 86daf7d6..b90dad96 100644 --- a/ml/s4/src/hippo/dplr.rs +++ b/ml/s4/src/hippo/dplr.rs @@ -6,74 +6,90 @@ //! //! use super::nplr::NPLR; -use crate::core::prelude::{Conjugate, SquareRoot}; -use crate::prelude::S4Float; -use ndarray::prelude::{Array1, Array2, Axis}; -use ndarray::{LinalgScalar, ScalarOperand}; -use ndarray_linalg::{Eigh, IntoTriangular, Lapack, Scalar, UPLO}; -use num::complex::{Complex, ComplexFloat}; -use num::{FromPrimitive, Num, One, Signed}; +use crate::core::prelude::{AsComplex, Conjugate, SquareRoot}; +use ndarray::prelude::{Array, Array1, Array2, Axis}; +use ndarray::ScalarOperand; +use ndarray_linalg::{Eigh, Lapack, Scalar, UPLO}; +use num::{Complex, Num, Signed}; use serde::{Deserialize, Serialize}; -use std::ops::{Add, Mul, Neg}; +use std::ops::{Mul, Neg}; -// #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] -#[derive(Clone, Debug, PartialEq)] +pub(crate) fn dplr(features: usize) -> DPLR +where + T: AsComplex + + Conjugate + + Lapack + + Num + + Scalar + + ScalarOperand + + Signed + + SquareRoot + + Mul, Output = Complex>, + Complex: Lapack, + ::Real: Mul, Output = Complex>, +{ + let (a, p, b) = NPLR::::new(features).into(); + + // + let s = { + let p2 = p.clone().insert_axis(Axis(1)); + &a + p2.dot(&p2.t()) + }; + // + let sd = s.diag(); + + let lambda_re = Array::ones(sd.dim()) * sd.mean().expect(""); + + let (e, v) = s + .mapv(|i: T| i * Complex::i().neg()) + .eigh(UPLO::Lower) + .expect(""); + + let lambda = { + // let lambda_im = e.mapv(|i| i * Complex::i()); + let iter = lambda_re + .into_iter() + .zip(e.into_iter()) + .map(|(i, j)| Complex::new(i, T::zero()) + T::from(j).unwrap() * Complex::i()); + Array::from_iter(iter) + }; + let p = p.mapv(AsComplex::as_re); + let b = b.mapv(AsComplex::as_re); + DPLR { + lambda, + p: v.conj().t().dot(&p), + b: v.conj().t().dot(&b), + v, + } +} + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] pub struct DPLR where - T: Clone + Num + Scalar, + T: Clone + Num, { - pub lambda: Array2::Real>>, - pub p: Array1, - pub b: Array1, - pub v: Array2, + pub lambda: Array1>, + pub p: Array1>, + pub b: Array1>, + pub v: Array2>, } impl DPLR where - T: Conjugate + Lapack + Scalar + ScalarOperand + Signed + SquareRoot, - T: Add::Real>, Output = Complex<::Real>> - + Mul::Real>, Output = Complex<::Real>>, - Complex<::Real>: Mul::Real>>, - Complex<::Real>: - Mul::Real>, Output = Complex<::Real>>, - Complex: Add::Real>, Output = Complex> - + Mul::Real>, Output = Complex>, + T: AsComplex + + Conjugate + + Lapack + + Num + + Scalar + + ScalarOperand + + Signed + + SquareRoot + + Mul, Output = Complex>, + Complex: Lapack, + ::Real: Mul, Output = Complex>, { pub fn create(features: usize) -> Self { - let (a, p, b) = NPLR::::new(features).into(); - - // - let s = { - let p2 = p.clone().insert_axis(Axis(1)); - &a + p2.dot(&p2.t()) - }; - // - let sd = s.diag().mean().expect("Average of diagonal is NaN"); - - let a = Array2::ones(s.dim()) * sd; - - // TODO: Fix this - // let (ee, vv) = { - // let si = - // }; - // let (e, v) = s.mapv(|i: T| T::from(Complex::new(i.re(), i.im()) * Complex::i().neg()).unwrap()) - // .eigh(UPLO::Lower) - // .expect(""); - let (e, v) = s.conj().eigh(UPLO::Lower).expect(""); - - // let a = a + &e * Complex::new(<::Real>::one(), <::Real>::one()); - let a = a + e.mapv(|i: ::Real| { - Complex::new(i.re(), i.im()) - * Complex::new(::Real::one(), ::Real::one().neg()) - }); - let p = v.conj().t().dot(&p); - let b = v.conj().t().dot(&b); - Self { - lambda: a, - p, - b, - v: v.clone(), - } + dplr(features) } } diff --git a/ml/s4/src/hippo/mod.rs b/ml/s4/src/hippo/mod.rs index 4ecc89d6..7c272d5b 100644 --- a/ml/s4/src/hippo/mod.rs +++ b/ml/s4/src/hippo/mod.rs @@ -18,16 +18,11 @@ pub struct LowRank { } pub(crate) mod utils { - use super::dplr::DPLR; - use super::nplr::NPLR; - use crate::core::prelude::{rangespace, Conjugate, SquareRoot}; - use crate::prelude::S4Float; + use crate::core::prelude::SquareRoot; use ndarray::prelude::{Array1, Array2, Axis}; - use ndarray::{LinalgScalar, ScalarOperand}; - use ndarray_linalg::{Eigh, IntoTriangular, Scalar, UPLO}; - use num::complex::{Complex, ComplexFloat}; - use num::traits::{FromPrimitive, Num, NumCast, Signed}; - use std::ops; + use ndarray::ScalarOperand; + use ndarray_linalg::{IntoTriangular, UPLO}; + use num::traits::{Num, NumCast, Signed}; pub fn genspace(features: usize) -> Array1 { Array1::from_iter((0..features).map(|x| T::from(x).unwrap())) @@ -43,100 +38,17 @@ pub(crate) mod utils { a = &a.into_triangular(UPLO::Lower) - &Array2::from_diag(&genspace::(features)); -a } - - pub(crate) fn make_hippo(features: usize) -> Array2 - where - T: ComplexFloat + ScalarOperand, - { - let base = rangespace((features, 1)); - let p = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); - let mut a = &p * &p.t(); - a = &a.into_triangular(UPLO::Lower) - &base.diag(); - -a - } - - pub(crate) fn make_nplr_hippo(features: usize) -> NPLR - where - T: ComplexFloat + ScalarOperand, - { - let hippo = make_hippo(features); - - let base = rangespace((features,)); - let p = (&base + T::one() / T::from(2).unwrap()).mapv(T::sqrt); - let b = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); - NPLR { a: hippo, p, b } - } - - pub(crate) fn make_dplr_hippo(features: usize) -> DPLR - where - T: S4Float, - ::Real: FromPrimitive, - Complex<::Real>: - ops::Mul::Real>, Output = Complex<::Real>>, - Complex<::Real>: LinalgScalar + ScalarOperand, - Array2::Real>>: ops::Add< - Array1::Real>>, - Output = Array2::Real>>, - > + ops::Mul< - Array2<::Real>, - Output = Array2::Real>>, - > + ops::Mul< - Array2::Real>>, - Output = Array2::Real>>, - > + ops::Mul< - Array2::Real>>, - Output = Array2::Real>>, - >, - { - let (a, p, b) = make_nplr_hippo::(features).into(); - - // - let s = { - let p2 = p.clone().insert_axis(Axis(1)); - &a + p2.dot(&p2.t()) - }; - // - let sd = s.diag().mapv(|i: T| Complex::new(i.re(), i.im())); - - let a = Array2::::Real>>::ones(s.dim()) - * sd.mean().expect("Average of diagonal is NaN"); - - // TODO: Fix this - // let (ee, vv) = { - // let si = - // }; - // let (e, v) = s.mapv(|i: T| T::from(Complex::new(i.re(), i.im()) * Complex::i().neg()).unwrap()) - // .eigh(UPLO::Lower) - // .expect(""); - let (e, v) = s.conj().eigh(UPLO::Lower).expect(""); - // compute the conjugate transpose of the eigenvectors - - // let a = a + &e * Complex::new(<::Real>::one(), <::Real>::one()); - // let a = a + e.mapv(|i: ::Real| Complex::new(i.re(), i.im()) * Complex::i()); - let lambda = { - let a = Array2::ones(s.dim()); - a - }; - DPLR { - lambda, - p: v.conj().t().dot(&p), - b: v.conj().t().dot(&b), - v, - } - } } #[cfg(test)] mod tests { use super::*; - use crate::core::prelude::Conjugate; use dplr::DPLR; use nplr::NPLR; - use ndarray::prelude::{Array, Axis, Dimension}; - use ndarray::ScalarOperand; - use ndarray_linalg::aclose; - use num::{Num, Signed}; + use crate::core::prelude::Conjugate; + use ndarray::prelude::{Array, Axis}; + use num::complex::ComplexFloat; #[test] fn test_hippo() { @@ -147,40 +59,56 @@ mod tests { assert_eq!(&a, b.as_ref()); } - fn close(a: &Array, b: &Array, atol: T) - where - D: Dimension, - T: Num + ScalarOperand + Signed + PartialOrd + std::fmt::Debug, - { - (a - b).for_each(|i| assert!(i.abs() <= atol, "Actual: {:?}\nTolerance: {:?}", i, atol)) - } - #[test] - fn test_dplr() { - let features = 10; + fn test_low_rank() { + let features = 8; - // let a = make_dplr_hippo::>(features); - // let b = HiPPO::>::new(features).dplr(); let nplr = NPLR::::new(features); let dplr = DPLR::::create(features); let hippo = nplr.a.clone(); let v = dplr.v.clone(); + // compute the conjugate transpose of the eigenvectors let vc = v.conj().t().to_owned(); - - let lambda = dplr.lambda.diag().to_owned().mapv(|i| i.re); - - let p = nplr.p.insert_axis(Axis(1)); - - let pc = dplr.p.insert_axis(Axis(1)); - + // create a two-dimensional array from the diagonal of the lambda matrix + let lambda = { + let ld = dplr.lambda.diag().to_owned(); + Array::from_diag(&ld) + }; + // reshape the p values + let p = nplr.p.clone().insert_axis(Axis(1)); + let pc = dplr.p.clone().insert_axis(Axis(1)); + // compute the expected values for NPLR let a = v.dot(&lambda).dot(&vc) - &p.dot(&p.t()); - let b = v.dot(&(lambda - pc.dot(&pc.t()))).dot(&vc); + // compute the expected values for DPLR + let b = { + let tmp = lambda - pc.dot(&pc.conj().t()); + v.dot(&tmp).dot(&vc) + }; - close(&hippo, &a, 1e-4); - // close(&hippo, &b, 1e-4); - // assert_eq!(&a, &b); + let err_nplr = { + let tmp = (&a - &hippo).mapv(|i| i.abs()); + tmp.mean().unwrap() + }; + assert!( + err_nplr <= 1e-4, + "Actual: {:?}\nTolerance: {:?}", + err_nplr, + 1e-4 + ); + let err_dplr = { + let tmp = (&b - &hippo).mapv(|i| i.abs()); + println!("{:?}", &tmp); + tmp.mean().unwrap() + // tmp + }; + assert!( + err_dplr <= 1e-4, + "Actual: {:?}\nTolerance: {:?}", + err_dplr, + 1e-4 + ); } // #[test] diff --git a/ml/s4/src/hippo/nplr.rs b/ml/s4/src/hippo/nplr.rs index b3835761..b809f2d2 100644 --- a/ml/s4/src/hippo/nplr.rs +++ b/ml/s4/src/hippo/nplr.rs @@ -7,10 +7,9 @@ //! use super::utils::*; -use crate::core::prelude::{rangespace, SquareRoot}; +use crate::core::prelude::SquareRoot; use ndarray::prelude::{Array1, Array2}; use ndarray::ScalarOperand; -use num::complex::ComplexFloat; use num::traits::{Num, NumCast, Signed}; use serde::{Deserialize, Serialize}; From 3e29bd2b2e0a48e4b7918cad301acc3c9abef8b3 Mon Sep 17 00:00:00 2001 From: FL03 Date: Wed, 3 Jan 2024 19:27:56 -0600 Subject: [PATCH 103/118] update Signed-off-by: FL03 --- ml/s4/src/hippo/dplr.rs | 38 +++++++++++++++++++------------------- ml/s4/src/hippo/hippo.rs | 30 +----------------------------- ml/s4/src/hippo/nplr.rs | 2 +- ml/s4/src/primitives.rs | 38 -------------------------------------- ml/s4/src/specs.rs | 6 +----- ml/s4/src/utils.rs | 2 +- 6 files changed, 23 insertions(+), 93 deletions(-) diff --git a/ml/s4/src/hippo/dplr.rs b/ml/s4/src/hippo/dplr.rs index b90dad96..e41a27e2 100644 --- a/ml/s4/src/hippo/dplr.rs +++ b/ml/s4/src/hippo/dplr.rs @@ -11,20 +11,28 @@ use ndarray::prelude::{Array, Array1, Array2, Axis}; use ndarray::ScalarOperand; use ndarray_linalg::{Eigh, Lapack, Scalar, UPLO}; use num::{Complex, Num, Signed}; +use num::traits::NumOps; use serde::{Deserialize, Serialize}; use std::ops::{Mul, Neg}; +pub(crate) trait DPLRScalar: AsComplex + Conjugate + Scalar + ScalarOperand + Signed + SquareRoot + NumOps + NumOps, Complex> +where + Complex: Lapack, + ::Real: Mul, Output = Complex>, +{ +} + +impl DPLRScalar for T +where + T: AsComplex + Conjugate + NumOps + NumOps, Complex> + Scalar + ScalarOperand + Signed + SquareRoot, + Complex: Lapack, + ::Real: Mul, Output = Complex>, +{ +} + pub(crate) fn dplr(features: usize) -> DPLR where - T: AsComplex - + Conjugate - + Lapack - + Num - + Scalar - + ScalarOperand - + Signed - + SquareRoot - + Mul, Output = Complex>, + T: DPLRScalar, Complex: Lapack, ::Real: Mul, Output = Complex>, { @@ -76,17 +84,9 @@ where impl DPLR where - T: AsComplex - + Conjugate - + Lapack - + Num - + Scalar - + ScalarOperand - + Signed - + SquareRoot - + Mul, Output = Complex>, + T: AsComplex + Conjugate + NumOps + NumOps, Complex> + Scalar + ScalarOperand + Signed + SquareRoot, Complex: Lapack, - ::Real: Mul, Output = Complex>, + ::Real: NumOps, Complex>, { pub fn create(features: usize) -> Self { dplr(features) diff --git a/ml/s4/src/hippo/hippo.rs b/ml/s4/src/hippo/hippo.rs index c4c6aac9..bc644a50 100644 --- a/ml/s4/src/hippo/hippo.rs +++ b/ml/s4/src/hippo/hippo.rs @@ -8,9 +8,7 @@ use super::utils::genspace; use crate::core::prelude::SquareRoot; use ndarray::prelude::Array2; use ndarray::ScalarOperand; -use ndarray_linalg::{IntoTriangular, UPLO}; use num::traits::{Num, NumCast, Signed}; -// use num::traits::{Float, FloatConst}; use serde::{Deserialize, Serialize}; pub enum HiPPOs { @@ -82,33 +80,7 @@ where // T: S4Float, // { // pub fn dplr(&self) -> DPLR { -// let (a, p, b) = super::make_nplr_hippo(self.features).into(); - -// // -// let s = &a -// + p.clone() -// .insert_axis(Axis(1)) -// .dot(&p.clone().insert_axis(Axis(0))); -// // -// let sd = s.diag(); - -// let a = Array2::ones(s.dim()) * sd.mean().expect("Average of diagonal is NaN"); - -// // TODO: replace with eigh -// let (e, v) = &(&s * T::from(T::one().neg().as_im()).unwrap()) -// .eigh(UPLO::Lower) -// .expect(""); -// let e = e.mapv(|x| T::from(x).unwrap()); - -// let a = a + &e * T::from(T::one().as_im()).unwrap(); -// let p = v.conj().t().dot(&p); -// let b = v.conj().t().dot(&b); -// DPLR { -// lambda: a, -// p, -// b, -// v: v.clone(), -// } + // } // } diff --git a/ml/s4/src/hippo/nplr.rs b/ml/s4/src/hippo/nplr.rs index b809f2d2..adc5a5ad 100644 --- a/ml/s4/src/hippo/nplr.rs +++ b/ml/s4/src/hippo/nplr.rs @@ -34,7 +34,7 @@ pub struct NPLR { impl NPLR where - T: Num + NumCast + ScalarOperand + Signed + SquareRoot, + T: NumCast + ScalarOperand + Signed + SquareRoot, { pub fn new(features: usize) -> Self { nplr(features).into() diff --git a/ml/s4/src/primitives.rs b/ml/s4/src/primitives.rs index 40a2b06b..7db8f95d 100644 --- a/ml/s4/src/primitives.rs +++ b/ml/s4/src/primitives.rs @@ -3,44 +3,6 @@ Contrib: FL03 */ pub use self::constants::*; -use crate::core::specs::{Arithmetic, AsComplex, Conjugate}; -use ndarray::ScalarOperand; -use ndarray_linalg::{Lapack, Scalar}; -use num::complex::{Complex, ComplexFloat}; -use num::traits::FromPrimitive; -// use num::traits::{Float, FromPrimitive, FloatConst}; -use std::ops; - -pub trait S4Float: Arithmetic - + AsComplex - + Conjugate - + ComplexFloat - + FromPrimitive - + Lapack - + Scalar - + ScalarOperand - + ops::Mul::Real>, Output = Complex<::Real>> -where - ::Real: FromPrimitive, -{ -} - -impl S4Float for T -where - T: Arithmetic - + Arithmetic< - Complex<::Real>, - Output = Complex<::Real>, - > + AsComplex - + Conjugate - + ComplexFloat - + FromPrimitive - + Lapack - + ScalarOperand - + ops::Mul::Real>, Output = Complex<::Real>>, - ::Real: FromPrimitive, -{ -} mod constants { /// The default model size for S4 models diff --git a/ml/s4/src/specs.rs b/ml/s4/src/specs.rs index 23832f76..d88d322c 100644 --- a/ml/s4/src/specs.rs +++ b/ml/s4/src/specs.rs @@ -5,7 +5,7 @@ use crate::core::prelude::AsComplex; use ndarray::prelude::{Array, Dimension}; use num::complex::ComplexFloat; -use rustfft::{Fft, FftNum, FftPlanner}; +use rustfft::{FftNum, FftPlanner}; pub trait Scan { type Output; @@ -35,8 +35,6 @@ where type Output = Self; fn fft(&self, args: &Self) -> Self::Output { - let dim = args.dim(); - let mut out = Self::ones(args.dim()); let mut buffer = vec![T::zero().as_re(); args.len()]; let mut planner = FftPlanner::new(); let fft = planner.plan_fft_forward(args.len()); @@ -49,8 +47,6 @@ where } fn ifft(&self, args: &Self) -> Self::Output { - let dim = args.dim(); - let mut out = Self::ones(args.dim()); let mut buffer = vec![T::zero().as_re(); args.len()]; let mut planner = FftPlanner::new(); let fft = planner.plan_fft_inverse(args.len()); diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index 09ae480b..19607db0 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -10,7 +10,7 @@ use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, StandardNormal, Uniform}; use ndarray_rand::RandomExt; use num::complex::{Complex, ComplexFloat}; -use num::traits::float::{Float, FloatConst, FloatCore}; +use num::traits::float::{Float, FloatConst,}; use num::{Num, Signed}; use rustfft::{FftNum, FftPlanner}; From 2769518817e69c7c7841dfc8b02d2c82cf654ae0 Mon Sep 17 00:00:00 2001 From: FL03 Date: Wed, 3 Jan 2024 19:31:16 -0600 Subject: [PATCH 104/118] update Signed-off-by: FL03 --- core/src/params/store.rs | 4 +-- core/src/primitives.rs | 2 +- data/src/df/mod.rs | 2 +- data/src/lib.rs | 6 +---- data/src/primitives.rs | 1 - data/src/specs/mod.rs | 1 - ml/linear/src/model/module.rs | 2 +- ml/neural/src/errors/mod.rs | 2 +- ml/neural/src/func/loss/mod.rs | 2 +- ml/neural/src/func/prop/mod.rs | 2 +- ml/neural/src/layers/params.rs | 2 +- ml/neural/src/lib.rs | 1 - ml/neural/src/nn/cnn/model.rs | 2 -- ml/neural/src/nn/gnn/mod.rs | 2 +- ml/neural/src/nn/gnn/model.rs | 2 +- ml/neural/src/nn/rnn/mod.rs | 2 +- ml/neural/src/nn/rnn/model.rs | 2 -- ml/neural/src/params/mod.rs | 3 +-- ml/neural/src/primitives.rs | 2 +- ml/neural/src/specs.rs | 2 +- ml/nlp/src/lib.rs | 5 ---- ml/nlp/src/primitives.rs | 1 - ml/optim/src/cost/mod.rs | 2 +- ml/optim/src/grad/mod.rs | 12 ++++----- ml/optim/src/grad/sgd.rs | 16 +++++------ ml/optim/src/primitives.rs | 2 +- ml/s4/src/hippo/dplr.rs | 30 ++++++++++++++++++--- ml/s4/src/hippo/hippo.rs | 2 +- ml/s4/src/model/model.rs | 4 +-- ml/s4/src/ops/scan.rs | 4 +-- ml/s4/src/params/mod.rs | 2 +- ml/s4/src/specs.rs | 6 ----- ml/s4/src/utils.rs | 6 ++--- ml/transformers/src/attention/params/mod.rs | 2 +- ml/transformers/src/codec/decode/mod.rs | 2 +- ml/transformers/src/codec/encode/mod.rs | 2 +- ml/transformers/src/codec/mod.rs | 1 - ml/transformers/src/ffn/mod.rs | 2 +- ml/transformers/src/lib.rs | 4 +-- 39 files changed, 72 insertions(+), 77 deletions(-) diff --git a/core/src/params/store.rs b/core/src/params/store.rs index b337143b..83c610e6 100644 --- a/core/src/params/store.rs +++ b/core/src/params/store.rs @@ -91,8 +91,8 @@ mod tests { fn test_model_store() { let (inputs, outputs) = (5, 3); - let shapes = [(inputs, outputs), (outputs, outputs), (outputs, 1)]; + let _shapes = [(inputs, outputs), (outputs, outputs), (outputs, 1)]; - let params = ParamStore::::new(); + let _params = ParamStore::::new(); } } diff --git a/core/src/primitives.rs b/core/src/primitives.rs index 7c12ff9e..2c0e698c 100644 --- a/core/src/primitives.rs +++ b/core/src/primitives.rs @@ -2,7 +2,7 @@ Appellation: primitives Contrib: FL03 */ -pub use self::{constants::*, statics::*, types::*}; +pub use self::{constants::*, types::*}; pub use ndarray::ShapeError; pub use ndarray_rand::rand_distr::uniform::SampleUniform; diff --git a/data/src/df/mod.rs b/data/src/df/mod.rs index d5d9cfa7..57944b27 100644 --- a/data/src/df/mod.rs +++ b/data/src/df/mod.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ //! # DataFrame -pub use self::{dataframe::*, utils::*}; +pub use self::dataframe::*; pub(crate) mod dataframe; diff --git a/data/src/lib.rs b/data/src/lib.rs index 80c0aa92..5ca6a51b 100644 --- a/data/src/lib.rs +++ b/data/src/lib.rs @@ -6,7 +6,7 @@ //! #![feature(associated_type_defaults)] -pub use self::{misc::*, primitives::*, utils::*}; +pub use self::misc::*; pub(crate) mod misc; pub(crate) mod primitives; @@ -26,10 +26,6 @@ pub(crate) use concision_core as core; pub mod prelude { // pub use linfa::dataset::{Dataset, DatasetBase, DatasetView}; - pub use crate::primitives::*; - - pub use crate::utils::*; - pub use crate::datasets::*; pub use crate::df::*; pub use crate::flows::*; diff --git a/data/src/primitives.rs b/data/src/primitives.rs index 859023bb..51d14049 100644 --- a/data/src/primitives.rs +++ b/data/src/primitives.rs @@ -2,7 +2,6 @@ Appellation: primitives Contrib: FL03 */ -pub use self::{constants::*, statics::*, types::*}; mod constants {} diff --git a/data/src/specs/mod.rs b/data/src/specs/mod.rs index 602237e6..dd32aeaf 100644 --- a/data/src/specs/mod.rs +++ b/data/src/specs/mod.rs @@ -2,7 +2,6 @@ Appellation: specs Contrib: FL03 */ -pub use self::elements::*; pub(crate) mod elements; diff --git a/ml/linear/src/model/module.rs b/ml/linear/src/model/module.rs index 30aded23..501538bc 100644 --- a/ml/linear/src/model/module.rs +++ b/ml/linear/src/model/module.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use crate::core::params::{Biased, Weighted}; -use crate::neural::models::exp::{Module, ModuleParams}; +use crate::neural::models::exp::ModuleParams; use crate::neural::prelude::Forward; use ndarray::prelude::{Array2, NdFloat}; use num::Float; diff --git a/ml/neural/src/errors/mod.rs b/ml/neural/src/errors/mod.rs index 267dace8..9e51d373 100644 --- a/ml/neural/src/errors/mod.rs +++ b/ml/neural/src/errors/mod.rs @@ -2,7 +2,7 @@ Appellation: errors Contrib: FL03 */ -pub use self::{error::*, utils::*}; +pub use self::error::*; pub(crate) mod error; diff --git a/ml/neural/src/func/loss/mod.rs b/ml/neural/src/func/loss/mod.rs index b0883478..a8c79cc2 100644 --- a/ml/neural/src/func/loss/mod.rs +++ b/ml/neural/src/func/loss/mod.rs @@ -8,7 +8,7 @@ //! Overall, neural network models aim to minimize the average loss by adjusting certain hyperparameters, //! the weights and biases. -pub use self::{kinds::*, utils::*}; +pub use self::kinds::*; pub(crate) mod kinds; diff --git a/ml/neural/src/func/prop/mod.rs b/ml/neural/src/func/prop/mod.rs index 4510df4a..92ae0562 100644 --- a/ml/neural/src/func/prop/mod.rs +++ b/ml/neural/src/func/prop/mod.rs @@ -5,7 +5,7 @@ //! # Propagation //! //! This module describes the propagation of data through a neural network. -pub use self::{modes::*, results::*, utils::*}; +pub use self::{modes::*, results::*}; pub(crate) mod modes; pub(crate) mod results; diff --git a/ml/neural/src/layers/params.rs b/ml/neural/src/layers/params.rs index dab9a47b..e42757b5 100644 --- a/ml/neural/src/layers/params.rs +++ b/ml/neural/src/layers/params.rs @@ -6,7 +6,7 @@ use super::LayerShape; use crate::core::prelude::GenerateRandom; use crate::prelude::{Features, Forward, Node}; use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Array1, Array2, Axis, Dimension, Ix2, NdFloat}; +use ndarray::prelude::{Array, Array1, Array2, Axis, Dimension, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use num::Float; diff --git a/ml/neural/src/lib.rs b/ml/neural/src/lib.rs index 805da18e..b3372643 100644 --- a/ml/neural/src/lib.rs +++ b/ml/neural/src/lib.rs @@ -30,7 +30,6 @@ pub mod prelude { pub use crate::specs::*; pub use crate::utils::*; - pub use crate::cmp::*; pub use crate::errors::*; pub use crate::func::{activate::*, loss::*, prop::*, rms::*}; pub use crate::layers::*; diff --git a/ml/neural/src/nn/cnn/model.rs b/ml/neural/src/nn/cnn/model.rs index 0c92921e..4dd61560 100644 --- a/ml/neural/src/nn/cnn/model.rs +++ b/ml/neural/src/nn/cnn/model.rs @@ -2,7 +2,5 @@ Appellation: model Contrib: FL03 */ -use ndarray::prelude::Array2; -use num::Float; pub struct CNN {} diff --git a/ml/neural/src/nn/gnn/mod.rs b/ml/neural/src/nn/gnn/mod.rs index 2b8f4900..321efbe5 100644 --- a/ml/neural/src/nn/gnn/mod.rs +++ b/ml/neural/src/nn/gnn/mod.rs @@ -4,7 +4,7 @@ */ //! # Graph Neural Network //! -pub use self::{model::*, tasks::*, utils::*}; +pub use self::{model::*, tasks::*}; pub(crate) mod model; pub(crate) mod tasks; diff --git a/ml/neural/src/nn/gnn/model.rs b/ml/neural/src/nn/gnn/model.rs index 147a580e..0cea9291 100644 --- a/ml/neural/src/nn/gnn/model.rs +++ b/ml/neural/src/nn/gnn/model.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use crate::prelude::Node; -use ndarray::prelude::Array; + use num::Float; use petgraph::prelude::{Directed, Graph}; diff --git a/ml/neural/src/nn/rnn/mod.rs b/ml/neural/src/nn/rnn/mod.rs index 5f8a8d7f..d2cb74d5 100644 --- a/ml/neural/src/nn/rnn/mod.rs +++ b/ml/neural/src/nn/rnn/mod.rs @@ -5,7 +5,7 @@ //! # Reccurrant Neural Network (RNN) //! //! -pub use self::{model::*, utils::*}; +pub use self::model::*; pub(crate) mod model; diff --git a/ml/neural/src/nn/rnn/model.rs b/ml/neural/src/nn/rnn/model.rs index 08a2b48f..b3e3fd21 100644 --- a/ml/neural/src/nn/rnn/model.rs +++ b/ml/neural/src/nn/rnn/model.rs @@ -2,7 +2,5 @@ Appellation: model Contrib: FL03 */ -use ndarray::prelude::Array2; -use num::Float; pub struct RNN {} diff --git a/ml/neural/src/params/mod.rs b/ml/neural/src/params/mod.rs index 7191e882..2b4c9429 100644 --- a/ml/neural/src/params/mod.rs +++ b/ml/neural/src/params/mod.rs @@ -12,9 +12,8 @@ pub(crate) mod group; pub(crate) mod param; pub(crate) mod shapes; -use ndarray::linalg::Dot; use ndarray::prelude::{Array, Dimension, Ix2}; -use ndarray::IntoDimension; + use num::Float; pub trait Biased diff --git a/ml/neural/src/primitives.rs b/ml/neural/src/primitives.rs index 120654da..4fabd493 100644 --- a/ml/neural/src/primitives.rs +++ b/ml/neural/src/primitives.rs @@ -2,7 +2,7 @@ Appellation: primitives Contrib: FL03 */ -pub use self::{constants::*, statics::*, types::*}; +pub use self::{constants::*, types::*}; pub(crate) mod constants { pub const DEFAULT_BUFFER: usize = 1024; diff --git a/ml/neural/src/specs.rs b/ml/neural/src/specs.rs index 4374cff2..6f2f149d 100644 --- a/ml/neural/src/specs.rs +++ b/ml/neural/src/specs.rs @@ -4,7 +4,7 @@ */ use crate::core::BoxResult; use crate::func::loss::Loss; -use ndarray::prelude::{Array, Axis, Dimension, Ix2}; +use ndarray::prelude::{Array, Axis, Ix2}; use num::Float; pub trait Backward: Forward { diff --git a/ml/nlp/src/lib.rs b/ml/nlp/src/lib.rs index 8ddedbe2..75150baa 100644 --- a/ml/nlp/src/lib.rs +++ b/ml/nlp/src/lib.rs @@ -3,7 +3,6 @@ Contrib: FL03 */ //! # Natural Language Processing -pub use self::{primitives::*, specs::*, utils::*}; pub(crate) mod primitives; pub(crate) mod specs; @@ -18,8 +17,4 @@ pub use concision_core as core; pub mod prelude { pub use crate::embed::*; pub use crate::encode::*; - - pub use crate::primitives::*; - pub use crate::specs::*; - pub use crate::utils::*; } diff --git a/ml/nlp/src/primitives.rs b/ml/nlp/src/primitives.rs index 859023bb..51d14049 100644 --- a/ml/nlp/src/primitives.rs +++ b/ml/nlp/src/primitives.rs @@ -2,7 +2,6 @@ Appellation: primitives Contrib: FL03 */ -pub use self::{constants::*, statics::*, types::*}; mod constants {} diff --git a/ml/optim/src/cost/mod.rs b/ml/optim/src/cost/mod.rs index 7dac866b..75ba92e1 100644 --- a/ml/optim/src/cost/mod.rs +++ b/ml/optim/src/cost/mod.rs @@ -4,7 +4,7 @@ */ //! # cost //! -pub use self::{kinds::*, utils::*}; +pub use self::kinds::*; pub(crate) mod kinds; diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index 2732d288..c18fda19 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -39,9 +39,9 @@ pub(crate) mod utils { use std::ops::Sub; pub fn gradient_descent( - gamma: T, + _gamma: T, model: &mut M, - objective: impl Gradient, + _objective: impl Gradient, data: &Array2, targets: &Array, ) -> anyhow::Result @@ -137,17 +137,17 @@ mod tests { fn test_gradient() { let (samples, inputs, outputs) = (20, 5, 1); - let (epochs, gamma) = (10, 0.001); + let (_epochs, _gamma) = (10, 0.001); let features = LayerShape::new(inputs, outputs); // Generate some example data let x = linarr::((samples, features.inputs())).unwrap(); - let y = linarr::((samples, features.outputs())).unwrap(); + let _y = linarr::((samples, features.outputs())).unwrap(); - let mut model = Layer::::from(features).init(true); + let model = Layer::::from(features).init(true); - let pred = model.forward(&x); + let _pred = model.forward(&x); // let mut losses = Array1::zeros(epochs); // for e in 0..epochs { diff --git a/ml/optim/src/grad/sgd.rs b/ml/optim/src/grad/sgd.rs index b2f1d0ed..cb0171ca 100644 --- a/ml/optim/src/grad/sgd.rs +++ b/ml/optim/src/grad/sgd.rs @@ -38,7 +38,7 @@ where let ys = y.select(Axis(0), pos); let pred = model.forward(&xs); - let error = &pred - &ys; + let _error = &pred - &ys; for batch in (0..samples).step_by(batch_size) { let mut gradient = Array2::zeros((features.outputs(), features.inputs())); @@ -54,7 +54,7 @@ where let inner = y - &prediction; let partial_w = (-2.0 / batch_size as f64) * input.dot(&inner); - let partial_b = (-2.0 / batch_size as f64) * inner; + let _partial_b = (-2.0 / batch_size as f64) * inner; gradient -= partial_w.sum(); // let mut weights = model.weights_mut().slice_mut(s![]) // model.set_weights(weights) @@ -83,7 +83,7 @@ pub fn sgd_step( x: &Array2, y: &Array1, model: &mut Layer, - learning_rate: f64, + _learning_rate: f64, batch_size: usize, ) -> anyhow::Result where @@ -91,17 +91,17 @@ where { let layer = model.clone(); let features = layer.features(); - let (samples, _inputs) = x.dim(); + let (_samples, _inputs) = x.dim(); let mut indices: Vec = (0..features.outputs()).collect(); - let mut losses = 0.0; + let losses = 0.0; indices.shuffle(&mut rand::thread_rng()); let pos = &indices[..batch_size]; let xs = x.select(Axis(0), pos); - let ys = y.select(Axis(0), pos); + let _ys = y.select(Axis(0), pos); - let pred = model.forward(&xs); + let _pred = model.forward(&xs); Ok(losses) } @@ -114,7 +114,7 @@ pub struct Sgd { impl Sgd { pub fn step(&mut self) -> f64 { - let mut loss = 0.0; + let loss = 0.0; loss } diff --git a/ml/optim/src/primitives.rs b/ml/optim/src/primitives.rs index 01b073a1..d80ec4b6 100644 --- a/ml/optim/src/primitives.rs +++ b/ml/optim/src/primitives.rs @@ -2,7 +2,7 @@ Appellation: primitives Contrib: FL03 */ -pub use self::{constants::*, statics::*, types::*}; +pub use self::{constants::*, types::*}; mod constants { diff --git a/ml/s4/src/hippo/dplr.rs b/ml/s4/src/hippo/dplr.rs index e41a27e2..679a637c 100644 --- a/ml/s4/src/hippo/dplr.rs +++ b/ml/s4/src/hippo/dplr.rs @@ -10,12 +10,20 @@ use crate::core::prelude::{AsComplex, Conjugate, SquareRoot}; use ndarray::prelude::{Array, Array1, Array2, Axis}; use ndarray::ScalarOperand; use ndarray_linalg::{Eigh, Lapack, Scalar, UPLO}; -use num::{Complex, Num, Signed}; use num::traits::NumOps; +use num::{Complex, Num, Signed}; use serde::{Deserialize, Serialize}; use std::ops::{Mul, Neg}; -pub(crate) trait DPLRScalar: AsComplex + Conjugate + Scalar + ScalarOperand + Signed + SquareRoot + NumOps + NumOps, Complex> +pub(crate) trait DPLRScalar: + AsComplex + + Conjugate + + Scalar + + ScalarOperand + + Signed + + SquareRoot + + NumOps + + NumOps, Complex> where Complex: Lapack, ::Real: Mul, Output = Complex>, @@ -24,7 +32,14 @@ where impl DPLRScalar for T where - T: AsComplex + Conjugate + NumOps + NumOps, Complex> + Scalar + ScalarOperand + Signed + SquareRoot, + T: AsComplex + + Conjugate + + NumOps + + NumOps, Complex> + + Scalar + + ScalarOperand + + Signed + + SquareRoot, Complex: Lapack, ::Real: Mul, Output = Complex>, { @@ -84,7 +99,14 @@ where impl DPLR where - T: AsComplex + Conjugate + NumOps + NumOps, Complex> + Scalar + ScalarOperand + Signed + SquareRoot, + T: AsComplex + + Conjugate + + NumOps + + NumOps, Complex> + + Scalar + + ScalarOperand + + Signed + + SquareRoot, Complex: Lapack, ::Real: NumOps, Complex>, { diff --git a/ml/s4/src/hippo/hippo.rs b/ml/s4/src/hippo/hippo.rs index bc644a50..5e4e4220 100644 --- a/ml/s4/src/hippo/hippo.rs +++ b/ml/s4/src/hippo/hippo.rs @@ -80,7 +80,7 @@ where // T: S4Float, // { // pub fn dplr(&self) -> DPLR { - + // } // } diff --git a/ml/s4/src/model/model.rs b/ml/s4/src/model/model.rs index e3b9b483..1f97adc9 100644 --- a/ml/s4/src/model/model.rs +++ b/ml/s4/src/model/model.rs @@ -8,7 +8,7 @@ use crate::prelude::SSMStore; use ndarray::prelude::{Array1, Array2, NdFloat}; use ndarray_conv::{Conv2DFftExt, PaddingMode, PaddingSize}; use ndarray_linalg::Scalar; -use num::complex::{Complex, ComplexFloat}; +use num::complex::ComplexFloat; use num::Float; use rustfft::FftNum; // use std::collections::HashMap; @@ -68,7 +68,7 @@ impl S4 where T: Float, { - pub fn setup(mut self) -> Self { + pub fn setup(self) -> Self { self } } diff --git a/ml/s4/src/ops/scan.rs b/ml/s4/src/ops/scan.rs index 9454044a..f81367b2 100644 --- a/ml/s4/src/ops/scan.rs +++ b/ml/s4/src/ops/scan.rs @@ -6,14 +6,14 @@ use crate::params::SSMStore; use num::Float; -pub struct Scan<'a, T = f64> +pub struct Scanner<'a, T = f64> where T: Float, { model: &'a mut SSMStore, } -impl<'a, T> Scan<'a, T> +impl<'a, T> Scanner<'a, T> where T: Float, { diff --git a/ml/s4/src/params/mod.rs b/ml/s4/src/params/mod.rs index 249db5f1..3cda0748 100644 --- a/ml/s4/src/params/mod.rs +++ b/ml/s4/src/params/mod.rs @@ -7,7 +7,7 @@ pub use self::{kinds::*, store::*}; pub(crate) mod kinds; pub(crate) mod store; -use ndarray::prelude::{Array, Array1, Array2, Ix2}; +use ndarray::prelude::{Array, Array2, Ix2}; use num::Num; use std::collections::HashMap; diff --git a/ml/s4/src/specs.rs b/ml/s4/src/specs.rs index d88d322c..e3a5e10c 100644 --- a/ml/s4/src/specs.rs +++ b/ml/s4/src/specs.rs @@ -13,12 +13,6 @@ pub trait Scan { fn scan(&self, args: &T, initial_state: &S) -> Self::Output; } -pub trait StateSpace { - type Config; - - fn config(&self) -> &Self::Config; -} - pub trait NdFft { type Output; diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index 19607db0..7c55850b 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -10,7 +10,7 @@ use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, StandardNormal, Uniform}; use ndarray_rand::RandomExt; use num::complex::{Complex, ComplexFloat}; -use num::traits::float::{Float, FloatConst,}; +use num::traits::float::{Float, FloatConst}; use num::{Num, Signed}; use rustfft::{FftNum, FftPlanner}; @@ -98,12 +98,12 @@ where res } -pub fn casual_colvolution(a: &Array2, b: &Array2) -> Array2 +pub fn casual_colvolution(a: &Array2, _b: &Array2) -> Array2 where T: FftNum, { let mut planner = FftPlanner::::new(); - let fft = planner.plan_fft_forward(a.len()); + let _fft = planner.plan_fft_forward(a.len()); a.clone() } diff --git a/ml/transformers/src/attention/params/mod.rs b/ml/transformers/src/attention/params/mod.rs index 6917d2a2..a629ba23 100644 --- a/ml/transformers/src/attention/params/mod.rs +++ b/ml/transformers/src/attention/params/mod.rs @@ -15,7 +15,7 @@ //! - samples: The number of samples to draw from the attention distribution. //! //! -pub use self::{dim::*, hyperparams::*, qkv::*, utils::*}; +pub use self::{dim::*, hyperparams::*, qkv::*}; pub(crate) mod dim; pub(crate) mod hyperparams; diff --git a/ml/transformers/src/codec/decode/mod.rs b/ml/transformers/src/codec/decode/mod.rs index f9114f9b..716e4454 100644 --- a/ml/transformers/src/codec/decode/mod.rs +++ b/ml/transformers/src/codec/decode/mod.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ //! # Decode -pub use self::{decoder::*, params::*, utils::*}; +pub use self::decoder::*; pub(crate) mod decoder; pub(crate) mod params; diff --git a/ml/transformers/src/codec/encode/mod.rs b/ml/transformers/src/codec/encode/mod.rs index 046a6444..244d4f07 100644 --- a/ml/transformers/src/codec/encode/mod.rs +++ b/ml/transformers/src/codec/encode/mod.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ //! # Encode -pub use self::{encoder::*, params::*, stack::*, utils::*}; +pub use self::{encoder::*, params::*, stack::*}; pub(crate) mod encoder; pub(crate) mod params; diff --git a/ml/transformers/src/codec/mod.rs b/ml/transformers/src/codec/mod.rs index 38f5114a..5a21f740 100644 --- a/ml/transformers/src/codec/mod.rs +++ b/ml/transformers/src/codec/mod.rs @@ -3,7 +3,6 @@ Contrib: FL03 */ //! # Codec -pub use self::utils::*; pub mod decode; pub mod encode; diff --git a/ml/transformers/src/ffn/mod.rs b/ml/transformers/src/ffn/mod.rs index 7dd10d33..09fb9ba4 100644 --- a/ml/transformers/src/ffn/mod.rs +++ b/ml/transformers/src/ffn/mod.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ //! # Decode -pub use self::{network::*, params::*, utils::*}; +pub use self::{network::*, params::*}; pub(crate) mod network; pub(crate) mod params; diff --git a/ml/transformers/src/lib.rs b/ml/transformers/src/lib.rs index aa1d8e61..1a47f4ac 100644 --- a/ml/transformers/src/lib.rs +++ b/ml/transformers/src/lib.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ //! # Concision Transformers -pub use self::{primitives::*, specs::*, utils::*}; +pub use self::primitives::*; pub(crate) mod primitives; pub(crate) mod specs; @@ -26,6 +26,4 @@ pub mod prelude { pub use crate::transform::*; pub use crate::primitives::*; - pub use crate::specs::*; - pub use crate::utils::*; } From 293de695491ade8ce699c0ccd0b811e3028007e2 Mon Sep 17 00:00:00 2001 From: FL03 Date: Wed, 3 Jan 2024 20:08:59 -0600 Subject: [PATCH 105/118] update Signed-off-by: FL03 --- core/src/lib.rs | 1 + core/src/ops/mod.rs | 11 ++++++ data/src/shape/dimension.rs | 1 + data/src/shape/mod.rs | 4 ++- data/src/shape/rank.rs | 11 ++++++ data/src/shape/shape.rs | 68 +++++++++++++++++++++++++++++++++++++ data/src/store/storage.rs | 5 --- data/src/tensors/mod.rs | 22 ++++++++++-- ml/s4/src/hippo/dplr.rs | 2 +- ml/s4/src/hippo/mod.rs | 2 +- 10 files changed, 116 insertions(+), 11 deletions(-) create mode 100644 data/src/shape/rank.rs create mode 100644 data/src/shape/shape.rs diff --git a/core/src/lib.rs b/core/src/lib.rs index 2674282f..e8c81620 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -12,6 +12,7 @@ pub(crate) mod utils; pub mod errors; pub mod id; pub mod masks; +pub mod ops; pub mod params; pub mod specs; pub mod states; diff --git a/core/src/ops/mod.rs b/core/src/ops/mod.rs index b8bc97c9..56fc4fee 100644 --- a/core/src/ops/mod.rs +++ b/core/src/ops/mod.rs @@ -13,5 +13,16 @@ pub trait Operation { fn eval(&self, args: &T) -> Self::Output; } +impl Operation for F +where + F: Fn(&T) -> S, +{ + type Output = S; + + fn eval(&self, args: &T) -> Self::Output { + self(args) + } +} + #[cfg(test)] mod tests {} diff --git a/data/src/shape/dimension.rs b/data/src/shape/dimension.rs index f7c016bd..8beeb27c 100644 --- a/data/src/shape/dimension.rs +++ b/data/src/shape/dimension.rs @@ -4,3 +4,4 @@ */ pub trait Dimension {} + diff --git a/data/src/shape/mod.rs b/data/src/shape/mod.rs index d250c40b..9c263941 100644 --- a/data/src/shape/mod.rs +++ b/data/src/shape/mod.rs @@ -3,9 +3,11 @@ Contrib: FL03 */ //! # Shapes -pub use self::dimension::*; +pub use self::{dimension::*, rank::*, shape::*}; pub(crate) mod dimension; +pub(crate) mod rank; +pub(crate) mod shape; #[cfg(test)] mod tests {} diff --git a/data/src/shape/rank.rs b/data/src/shape/rank.rs new file mode 100644 index 00000000..21601107 --- /dev/null +++ b/data/src/shape/rank.rs @@ -0,0 +1,11 @@ +/* + Appellation: rank + Contrib: FL03 +*/ + + +pub enum Rank { + Zero(T), + One(Vec), + N(Vec), +} diff --git a/data/src/shape/shape.rs b/data/src/shape/shape.rs new file mode 100644 index 00000000..eef5a97e --- /dev/null +++ b/data/src/shape/shape.rs @@ -0,0 +1,68 @@ +/* + Appellation: shape + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use std::ops; + +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct Shape(Vec); + +impl Shape { + pub fn new(shape: Vec) -> Self { + Self(shape) + } + + pub fn elements(&self) -> usize { + self.0.iter().product() + } + + pub fn rank(&self) -> usize { + self.0.len() + } +} + +impl Extend for Shape { + fn extend>(&mut self, iter: I) { + self.0.extend(iter) + } +} + +impl FromIterator for Shape { + fn from_iter>(iter: I) -> Self { + Self(iter.into_iter().collect()) + } +} + +impl IntoIterator for Shape { + type Item = usize; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +impl<'a> IntoIterator for &'a mut Shape { + type Item = &'a mut usize; + type IntoIter = std::slice::IterMut<'a, usize>; + + fn into_iter(self) -> Self::IntoIter { + self.0.iter_mut() + } +} + +impl ops::Index for Shape { + type Output = usize; + + fn index(&self, index: usize) -> &Self::Output { + &self.0[index] + } +} + +impl ops::IndexMut for Shape { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.0[index] + } +} + diff --git a/data/src/store/storage.rs b/data/src/store/storage.rs index 596e5335..f2a69a73 100644 --- a/data/src/store/storage.rs +++ b/data/src/store/storage.rs @@ -5,8 +5,3 @@ pub struct Storage {} -pub enum Rank { - Zero(T), - One(Vec), - N(Vec), -} diff --git a/data/src/tensors/mod.rs b/data/src/tensors/mod.rs index e24e0393..403fd0fc 100644 --- a/data/src/tensors/mod.rs +++ b/data/src/tensors/mod.rs @@ -9,12 +9,28 @@ pub use self::tensor::*; pub(crate) mod tensor; -use ndarray::prelude::{Array, Dimension, Ix2}; +// use ndarray::prelude::{Array, Dimension, Ix2}; +use crate::core::ops::Operation; + +pub trait GradStore { + type Tensor: NdTensor; + + fn get(&self, id: &str) -> Option<&Self::Tensor>; +} pub trait NdTensor { - type Dim: Dimension = Ix2; - fn tensor(&self) -> &Array; + fn apply(&self, f: F) -> Self + where + F: Fn(T) -> T; + + fn apply_op(&self, op: impl Operation) -> Self; + + fn backward(&self) -> Self; + + fn id(&self) -> &str; + + fn tensor(&self) -> &Self; } #[cfg(test)] diff --git a/ml/s4/src/hippo/dplr.rs b/ml/s4/src/hippo/dplr.rs index 679a637c..58333e90 100644 --- a/ml/s4/src/hippo/dplr.rs +++ b/ml/s4/src/hippo/dplr.rs @@ -110,7 +110,7 @@ where Complex: Lapack, ::Real: NumOps, Complex>, { - pub fn create(features: usize) -> Self { + pub fn new(features: usize) -> Self { dplr(features) } } diff --git a/ml/s4/src/hippo/mod.rs b/ml/s4/src/hippo/mod.rs index 7c272d5b..1b6e6a8e 100644 --- a/ml/s4/src/hippo/mod.rs +++ b/ml/s4/src/hippo/mod.rs @@ -64,7 +64,7 @@ mod tests { let features = 8; let nplr = NPLR::::new(features); - let dplr = DPLR::::create(features); + let dplr = DPLR::::new(features); let hippo = nplr.a.clone(); From b59c66cdea1d7e16bd64abe78df7e1a6bfe400cd Mon Sep 17 00:00:00 2001 From: FL03 Date: Thu, 4 Jan 2024 11:56:24 -0600 Subject: [PATCH 106/118] update Signed-off-by: FL03 --- core/src/errors/kinds.rs | 2 +- core/src/ops/kinds.rs | 4 +- core/src/specs/arrays.rs | 31 +-- data/src/lib.rs | 2 + data/src/mat/matrix.rs | 15 +- data/src/shape/dimension.rs | 1 - data/src/shape/mod.rs | 13 +- data/src/shape/rank.rs | 15 +- data/src/shape/shape.rs | 80 +++++++ data/src/specs/elements.rs | 4 + data/src/specs/export.rs | 9 + data/src/specs/import.rs | 11 + data/src/specs/mod.rs | 3 + data/src/store/layout.rs | 19 ++ data/src/store/mod.rs | 47 +++- data/src/store/storage.rs | 1 - data/src/tensors/mod.rs | 1 + data/src/tensors/tensor.rs | 10 +- ml/linear/src/cmp/mod.rs | 13 + ml/linear/src/cmp/neurons/mod.rs | 95 ++++++++ ml/linear/src/cmp/neurons/node.rs | 263 +++++++++++++++++++++ ml/linear/src/cmp/neurons/perceptron.rs | 212 +++++++++++++++++ ml/linear/src/cmp/neurons/synapse.rs | 9 + ml/linear/src/{ => cmp}/params/features.rs | 0 ml/linear/src/{ => cmp}/params/kinds.rs | 0 ml/linear/src/{ => cmp}/params/mod.rs | 3 + ml/linear/src/{ => cmp}/params/store.rs | 3 +- ml/linear/src/conv/mod.rs | 6 + ml/linear/src/conv/module.rs | 6 + ml/linear/src/dense/mod.rs | 6 + ml/linear/src/dense/module.rs | 6 + ml/linear/src/lib.rs | 2 +- ml/linear/src/model/layer.rs | 8 +- ml/linear/src/model/mod.rs | 5 +- ml/neural/src/errors/error.rs | 4 +- ml/neural/src/func/activate/activator.rs | 2 +- ml/neural/src/func/activate/linear.rs | 14 +- ml/neural/src/func/activate/mod.rs | 4 +- ml/neural/src/func/block.rs | 4 +- ml/neural/src/layers/exp/sublayer.rs | 4 +- ml/neural/src/layers/layer.rs | 4 +- ml/neural/src/layers/stack.rs | 4 +- ml/neural/src/neurons/perceptron.rs | 4 +- ml/neural/src/nn/ffn/mlp.rs | 4 +- ml/optim/src/grad/mod.rs | 4 +- ml/s4/examples/sand.rs | 1 + ml/s4/src/hippo/dplr.rs | 7 +- ml/s4/src/utils.rs | 2 +- 48 files changed, 897 insertions(+), 70 deletions(-) create mode 100644 data/src/specs/export.rs create mode 100644 data/src/specs/import.rs create mode 100644 data/src/store/layout.rs create mode 100644 ml/linear/src/cmp/mod.rs create mode 100644 ml/linear/src/cmp/neurons/mod.rs create mode 100644 ml/linear/src/cmp/neurons/node.rs create mode 100644 ml/linear/src/cmp/neurons/perceptron.rs create mode 100644 ml/linear/src/cmp/neurons/synapse.rs rename ml/linear/src/{ => cmp}/params/features.rs (100%) rename ml/linear/src/{ => cmp}/params/kinds.rs (100%) rename ml/linear/src/{ => cmp}/params/mod.rs (87%) rename ml/linear/src/{ => cmp}/params/store.rs (98%) create mode 100644 ml/linear/src/conv/module.rs create mode 100644 ml/linear/src/dense/module.rs diff --git a/core/src/errors/kinds.rs b/core/src/errors/kinds.rs index 56809bb2..8bfbe4a5 100644 --- a/core/src/errors/kinds.rs +++ b/core/src/errors/kinds.rs @@ -39,7 +39,7 @@ pub enum Errors { IO, Null, Parse, - Process, + Process(ProcessError), Runtime, Syntax, Unknown, diff --git a/core/src/ops/kinds.rs b/core/src/ops/kinds.rs index 92d439f8..c2493dae 100644 --- a/core/src/ops/kinds.rs +++ b/core/src/ops/kinds.rs @@ -3,6 +3,4 @@ Contrib: FL03 */ -pub enum Ops { - -} \ No newline at end of file +pub enum Ops {} diff --git a/core/src/specs/arrays.rs b/core/src/specs/arrays.rs index 8bc4154b..ac248da1 100644 --- a/core/src/specs/arrays.rs +++ b/core/src/specs/arrays.rs @@ -66,24 +66,10 @@ where } } -pub trait RandNum: SampleUniform -where - StandardNormal: Distribution, -{ -} - -impl RandNum for T -where - T: SampleUniform, - StandardNormal: Distribution, -{ -} - pub trait GenerateRandom: Sized where D: Dimension, - T: Float + SampleUniform, - StandardNormal: Distribution, + T: Float, { fn rand(dim: impl IntoDimension, distr: IdS) -> Self where @@ -97,17 +83,26 @@ where Ok(Self::rand(dim.into_dimension(), dist)) } - fn stdnorm(dim: impl IntoDimension) -> Self { + fn stdnorm(dim: impl IntoDimension) -> Self + where + StandardNormal: Distribution, + { Self::rand(dim, StandardNormal) } - fn uniform(axis: usize, dim: impl IntoDimension) -> Self { + fn uniform(axis: usize, dim: impl IntoDimension) -> Self + where + T: SampleUniform, + { let dim = dim.into_dimension(); let dk = T::from(dim[axis]).unwrap().recip().sqrt(); Self::uniform_between(dk, dim) } - fn uniform_between(dk: T, dim: impl IntoDimension) -> Self { + fn uniform_between(dk: T, dim: impl IntoDimension) -> Self + where + T: SampleUniform, + { Self::rand(dim, Uniform::new(-dk, dk)) } } diff --git a/data/src/lib.rs b/data/src/lib.rs index 5ca6a51b..1fc128ce 100644 --- a/data/src/lib.rs +++ b/data/src/lib.rs @@ -24,11 +24,13 @@ pub mod tensors; pub(crate) use concision_core as core; pub mod prelude { + pub use crate::misc::*; // pub use linfa::dataset::{Dataset, DatasetBase, DatasetView}; pub use crate::datasets::*; pub use crate::df::*; pub use crate::flows::*; + pub use crate::shape::*; pub use crate::specs::*; pub use crate::store::*; diff --git a/data/src/mat/matrix.rs b/data/src/mat/matrix.rs index dbf2e4d7..76a4cbfa 100644 --- a/data/src/mat/matrix.rs +++ b/data/src/mat/matrix.rs @@ -2,7 +2,20 @@ Appellation: matrix Contrib: FL03 */ +use ndarray::prelude::Array2; pub struct Matrix { - store: Vec>, + store: Array2, +} + +impl Matrix { + pub fn new(store: Array2) -> Self { + Self { store } + } +} + +impl AsRef> for Matrix { + fn as_ref(&self) -> &Array2 { + &self.store + } } diff --git a/data/src/shape/dimension.rs b/data/src/shape/dimension.rs index 8beeb27c..f7c016bd 100644 --- a/data/src/shape/dimension.rs +++ b/data/src/shape/dimension.rs @@ -4,4 +4,3 @@ */ pub trait Dimension {} - diff --git a/data/src/shape/mod.rs b/data/src/shape/mod.rs index 9c263941..881f43c3 100644 --- a/data/src/shape/mod.rs +++ b/data/src/shape/mod.rs @@ -10,4 +10,15 @@ pub(crate) mod rank; pub(crate) mod shape; #[cfg(test)] -mod tests {} +mod tests { + use super::*; + + #[test] + fn test_shape() { + let mut shape = Shape::default(); + shape.extend([1, 1, 1]); + assert_eq!(shape, Shape::new(vec![1, 1, 1])); + assert_eq!(shape.elements(), 1); + assert_eq!(shape.rank(), 3); + } +} diff --git a/data/src/shape/rank.rs b/data/src/shape/rank.rs index 21601107..24bda35a 100644 --- a/data/src/shape/rank.rs +++ b/data/src/shape/rank.rs @@ -3,9 +3,20 @@ Contrib: FL03 */ - -pub enum Rank { +pub enum Ranks { Zero(T), One(Vec), N(Vec), } + +pub struct Rank(pub usize); + +impl Rank { + pub fn new(rank: usize) -> Self { + Self(rank) + } + + pub fn rank(&self) -> usize { + self.0 + } +} diff --git a/data/src/shape/shape.rs b/data/src/shape/shape.rs index eef5a97e..9d4a4071 100644 --- a/data/src/shape/shape.rs +++ b/data/src/shape/shape.rs @@ -17,9 +17,42 @@ impl Shape { self.0.iter().product() } + pub fn include(mut self, dim: usize) -> Self { + self.0.push(dim); + self + } + + pub fn push(&mut self, dim: usize) { + self.0.push(dim) + } + pub fn rank(&self) -> usize { self.0.len() } + + pub fn with_capacity(capacity: usize) -> Self { + Self(Vec::with_capacity(capacity)) + } + + pub fn zero() -> Self { + Self::default() + } + + pub fn zeros(rank: usize) -> Self { + Self(vec![0; rank]) + } +} + +impl AsRef<[usize]> for Shape { + fn as_ref(&self) -> &[usize] { + &self.0 + } +} + +impl AsMut<[usize]> for Shape { + fn as_mut(&mut self) -> &mut [usize] { + &mut self.0 + } } impl Extend for Shape { @@ -66,3 +99,50 @@ impl ops::IndexMut for Shape { } } +impl ops::Index> for Shape { + type Output = [usize]; + + fn index(&self, index: ops::Range) -> &Self::Output { + &self.0[index] + } +} + +impl ops::Index> for Shape { + type Output = [usize]; + + fn index(&self, index: ops::RangeTo) -> &Self::Output { + &self.0[index] + } +} + +impl ops::Index> for Shape { + type Output = [usize]; + + fn index(&self, index: ops::RangeFrom) -> &Self::Output { + &self.0[index] + } +} + +impl ops::Index for Shape { + type Output = [usize]; + + fn index(&self, index: ops::RangeFull) -> &Self::Output { + &self.0[index] + } +} + +impl ops::Index> for Shape { + type Output = [usize]; + + fn index(&self, index: ops::RangeInclusive) -> &Self::Output { + &self.0[index] + } +} + +impl ops::Index> for Shape { + type Output = [usize]; + + fn index(&self, index: ops::RangeToInclusive) -> &Self::Output { + &self.0[index] + } +} diff --git a/data/src/specs/elements.rs b/data/src/specs/elements.rs index f0bfb9f2..5b292598 100644 --- a/data/src/specs/elements.rs +++ b/data/src/specs/elements.rs @@ -2,3 +2,7 @@ Appellation: elements Contrib: FL03 */ +use num::complex::Complex; +use num::traits::NumOps; + +pub trait Element: NumOps + NumOps, Complex> + Sized {} diff --git a/data/src/specs/export.rs b/data/src/specs/export.rs new file mode 100644 index 00000000..d3bfb5c8 --- /dev/null +++ b/data/src/specs/export.rs @@ -0,0 +1,9 @@ +/* + Appellation: export + Contrib: FL03 +*/ +use std::path::Path; + +pub trait Export { + fn export(&self, path: impl AsRef) -> Result<(), std::io::Error>; +} diff --git a/data/src/specs/import.rs b/data/src/specs/import.rs new file mode 100644 index 00000000..d8c6610d --- /dev/null +++ b/data/src/specs/import.rs @@ -0,0 +1,11 @@ +/* + Appellation: import + Contrib: FL03 +*/ +use std::path::Path; + +pub trait Import { + type Obj; + + fn import(&mut self, path: impl AsRef) -> Result; +} diff --git a/data/src/specs/mod.rs b/data/src/specs/mod.rs index dd32aeaf..bcf89655 100644 --- a/data/src/specs/mod.rs +++ b/data/src/specs/mod.rs @@ -2,8 +2,11 @@ Appellation: specs Contrib: FL03 */ +pub use self::{elements::*, export::*, import::*}; pub(crate) mod elements; +pub(crate) mod export; +pub(crate) mod import; use ndarray::prelude::{Array1, Array2}; diff --git a/data/src/store/layout.rs b/data/src/store/layout.rs new file mode 100644 index 00000000..28780f6c --- /dev/null +++ b/data/src/store/layout.rs @@ -0,0 +1,19 @@ +/* + Appellation: layout + Contrib: FL03 +*/ +use crate::shape::Shape; + +pub struct Layout { + shape: Shape, +} + +impl Layout { + pub fn new(shape: Shape) -> Self { + Self { shape } + } + + pub fn shape(&self) -> &Shape { + &self.shape + } +} diff --git a/data/src/store/mod.rs b/data/src/store/mod.rs index a14c00b8..a62b8cfe 100644 --- a/data/src/store/mod.rs +++ b/data/src/store/mod.rs @@ -3,10 +3,13 @@ Contrib: FL03 */ //! # Store -pub use self::storage::*; +pub use self::{layout::*, storage::*}; +pub(crate) mod layout; pub(crate) mod storage; +use std::collections::{BTreeMap, HashMap}; + pub trait Store { fn get(&self, key: &K) -> Option<&V>; @@ -17,5 +20,47 @@ pub trait Store { fn remove(&mut self, key: &K) -> Option; } +impl Store for BTreeMap +where + K: Ord, +{ + fn get(&self, key: &K) -> Option<&V> { + BTreeMap::get(self, key) + } + + fn get_mut(&mut self, key: &K) -> Option<&mut V> { + BTreeMap::get_mut(self, key) + } + + fn insert(&mut self, key: K, value: V) -> Option { + BTreeMap::insert(self, key, value) + } + + fn remove(&mut self, key: &K) -> Option { + BTreeMap::remove(self, key) + } +} + +impl Store for HashMap +where + K: Eq + std::hash::Hash, +{ + fn get(&self, key: &K) -> Option<&V> { + HashMap::get(self, key) + } + + fn get_mut(&mut self, key: &K) -> Option<&mut V> { + HashMap::get_mut(self, key) + } + + fn insert(&mut self, key: K, value: V) -> Option { + HashMap::insert(self, key, value) + } + + fn remove(&mut self, key: &K) -> Option { + HashMap::remove(self, key) + } +} + #[cfg(test)] mod tests {} diff --git a/data/src/store/storage.rs b/data/src/store/storage.rs index f2a69a73..026e49f8 100644 --- a/data/src/store/storage.rs +++ b/data/src/store/storage.rs @@ -4,4 +4,3 @@ */ pub struct Storage {} - diff --git a/data/src/tensors/mod.rs b/data/src/tensors/mod.rs index 403fd0fc..1aae9f81 100644 --- a/data/src/tensors/mod.rs +++ b/data/src/tensors/mod.rs @@ -19,6 +19,7 @@ pub trait GradStore { } pub trait NdTensor { + fn affine(&self, a: T, b: T) -> Self; fn apply(&self, f: F) -> Self where diff --git a/data/src/tensors/tensor.rs b/data/src/tensors/tensor.rs index d8097141..38b82137 100644 --- a/data/src/tensors/tensor.rs +++ b/data/src/tensors/tensor.rs @@ -3,9 +3,10 @@ Contrib: FL03 */ use crate::core::id::AtomicId; +use crate::prelude::DType; use ndarray::prelude::{Array, Dimension, Ix2}; use ndarray::IntoDimension; -use num::Float; +use num::Num; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)] @@ -13,7 +14,6 @@ use serde::{Deserialize, Serialize}; pub struct Tensor where D: Dimension, - T: Float, { id: AtomicId, data: Array, @@ -22,9 +22,9 @@ where impl Tensor where D: Dimension, - T: Float, + T: Clone + Num, { - pub fn new(shape: impl IntoDimension) -> Self { + pub fn zeros(shape: impl IntoDimension) -> Self { Self { id: AtomicId::new(), data: Array::zeros(shape), @@ -35,7 +35,7 @@ where impl std::fmt::Display for Tensor where D: Dimension, - T: Float + std::fmt::Debug, + T: std::fmt::Debug, { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "{:?}", self.data) diff --git a/ml/linear/src/cmp/mod.rs b/ml/linear/src/cmp/mod.rs new file mode 100644 index 00000000..e341c207 --- /dev/null +++ b/ml/linear/src/cmp/mod.rs @@ -0,0 +1,13 @@ +/* + Appellation: cmp + Contrib: FL03 +*/ +//! # Components +//! +//! + +pub mod neurons; +pub mod params; + +#[cfg(test)] +mod tests {} diff --git a/ml/linear/src/cmp/neurons/mod.rs b/ml/linear/src/cmp/neurons/mod.rs new file mode 100644 index 00000000..730d56c0 --- /dev/null +++ b/ml/linear/src/cmp/neurons/mod.rs @@ -0,0 +1,95 @@ +/* + Appellation: neurons + Contrib: FL03 +*/ +//! # neurons +pub use self::{node::*, perceptron::*, synapse::*}; + +pub(crate) mod node; +pub(crate) mod perceptron; +pub(crate) mod synapse; + +use crate::neural::func::activate::Activate; +use ndarray::prelude::{Array0, Array1, Array2, Ix1, NdFloat}; + +pub trait ArtificialNeuron +where + T: NdFloat, +{ + type Rho: Activate; + + fn bias(&self) -> Array0; + + fn linear(&self, args: &Array2) -> Array1 { + args.dot(self.weights()) + self.bias() + } + + fn forward(&self, args: &Array2) -> Array1 { + self.rho().activate(&self.linear(args)) + } + + fn rho(&self) -> &Self::Rho; + + fn weights(&self) -> &Array1; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::neural::prelude::{softmax, Activate, Forward, Softmax}; + use ndarray::prelude::{array, Array1, Ix2, NdFloat}; + + fn _artificial( + args: &Array2, + bias: Option<&Array1>, + rho: impl Activate, + weights: &Array2, + ) -> Array2 + where + T: NdFloat, + { + let linear = if let Some(bias) = bias { + args.dot(weights) + bias + } else { + args.dot(weights) + }; + rho.activate(&linear) + } + + #[test] + fn test_neuron() { + let bias = 0.0; + + let data = array![[10.0, 10.0, 6.0, 1.0, 8.0]]; + let weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; + let neuron = Perceptron::::new(5).with_weights(weights.clone()); + + let linear = data.dot(&weights) + bias; + let exp = softmax(&linear); + + assert_eq!(exp, neuron.forward(&data)); + } + + // #[test] + // fn test_node() { + // let bias = ndarray::Array1::::zeros(4); + + // let a_data = array![10.0, 10.0, 6.0, 1.0, 8.0]; + // let a_weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; + // let a = Neuron::new(softmax, bias.clone(), a_weights.clone()); + // let node_a = Node::new(a.clone()).with_data(a_data.clone()); + + // let exp = _artificial(&a_data, Some(bias.clone()), Softmax::default(), &a_weights); + // assert_eq!(node_a.process(), exp); + + // let b_data = array![0.0, 9.0, 3.0, 5.0, 3.0]; + // let b_weights = array![2.0, 8.0, 8.0, 0.0, 3.0]; + + // let b = Neuron::new(softmax, bias.clone(), b_weights.clone()); + // let node_b = Node::new(b.clone()).with_data(b_data.clone()); + // let exp = _artificial(&b_data, Some(bias), Softmax::default(), &b_weights); + // assert_eq!(node_b.process(), exp); + + // assert_eq!(node_a.dot() + node_b.dot(), 252.0); + // } +} diff --git a/ml/linear/src/cmp/neurons/node.rs b/ml/linear/src/cmp/neurons/node.rs new file mode 100644 index 00000000..3c718b52 --- /dev/null +++ b/ml/linear/src/cmp/neurons/node.rs @@ -0,0 +1,263 @@ +/* + Appellation: node + Contrib: FL03 +*/ +use crate::core::prelude::GenerateRandom; +use crate::neural::prelude::Forward; + +use ndarray::linalg::Dot; +use ndarray::prelude::{Array, Array0, Array1, Array2, Dimension, NdFloat}; +use ndarray::{RemoveAxis, ScalarOperand}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; +use num::{Float, Num}; +use std::ops; + +#[derive(Clone, Debug, PartialEq)] +pub struct Node { + bias: Option>, + features: usize, + weights: Array1, +} + +impl Node +where + T: Clone + Num, +{ + pub fn create(biased: bool, features: usize) -> Self { + let bias = if biased { + Some(Array0::zeros(())) + } else { + None + }; + Self { + bias, + features, + weights: Array1::zeros(features), + } + } + + pub fn biased(features: usize) -> Self { + Self::create(true, features) + } + + pub fn new(features: usize) -> Self { + Self::create(false, features) + } +} +impl Node +where + T: Num, +{ + pub fn bias(&self) -> Option<&Array0> { + self.bias.as_ref() + } + + pub fn bias_mut(&mut self) -> Option<&mut Array0> { + self.bias.as_mut() + } + + pub fn features(&self) -> usize { + self.features + } + + pub fn is_biased(&self) -> bool { + self.bias.is_some() + } + + pub fn set_bias(&mut self, bias: Option>) { + self.bias = bias; + } + + pub fn set_features(&mut self, features: usize) { + self.features = features; + } + + pub fn set_weights(&mut self, weights: Array1) { + self.weights = weights; + } + + pub fn weights(&self) -> &Array1 { + &self.weights + } + + pub fn weights_mut(&mut self) -> &mut Array1 { + &mut self.weights + } + + pub fn with_bias(mut self, bias: Option>) -> Self { + self.bias = bias; + self + } + + pub fn with_features(mut self, features: usize) -> Self { + self.features = features; + self + } + + pub fn with_weights(mut self, weights: Array1) -> Self { + self.weights = weights; + self + } +} + +impl Node +where + T: Num + ScalarOperand, + Array2: Dot, Output = Array1>, +{ + pub fn linear(&self, data: &Array2) -> Array1 { + let w = self.weights().t().to_owned(); + if let Some(bias) = self.bias() { + data.dot(&w) + bias + } else { + data.dot(&w) + } + } +} +impl Node +where + T: Float + SampleUniform, + StandardNormal: Distribution, +{ + pub fn init(mut self, biased: bool) -> Self { + if biased { + self = self.init_bias(); + } + self.init_weight() + } + + pub fn init_bias(mut self) -> Self { + let dk = (T::one() / T::from(self.features).unwrap()).sqrt(); + self.bias = Some(Array0::uniform_between(dk, ())); + self + } + + pub fn init_weight(mut self) -> Self { + let features = self.features; + let dk = (T::one() / T::from(features).unwrap()).sqrt(); + self.weights = Array1::uniform_between(dk, features); + self + } +} + +impl Node +where + T: NdFloat, +{ + pub fn apply_gradient(&mut self, gamma: T, gradient: G) + where + G: Fn(&Array1) -> Array1, + { + let grad = gradient(self.weights()); + self.weights_mut().scaled_add(-gamma, &grad); + } + + pub fn activate(&self, data: &Array2, activator: A) -> Array1 + where + A: Fn(&Array1) -> Array1, + { + activator(&self.forward(data)) + } +} + +impl Forward> for Node +where + D: Dimension + RemoveAxis, + T: NdFloat, + Array: Dot, Output = Array>, + Array: ops::Add, Output = Array>, +{ + type Output = Array; + + fn forward(&self, data: &Array) -> Self::Output { + let w = self.weights().t().to_owned(); + if let Some(bias) = self.bias() { + return data.dot(&w) + bias.clone(); + } + data.dot(&w) + } +} + +impl FromIterator for Node +where + T: Float, +{ + fn from_iter(iter: I) -> Self + where + I: IntoIterator, + { + let weights = Array1::::from_iter(iter); + Self { + bias: None, + features: weights.len(), + weights, + } + } +} + +impl From<(Array1, Array0)> for Node +where + T: Float, +{ + fn from((weights, bias): (Array1, Array0)) -> Self { + Self { + bias: Some(bias), + features: weights.len(), + weights, + } + } +} + +impl From<(Array1, T)> for Node +where + T: NdFloat, +{ + fn from((weights, bias): (Array1, T)) -> Self { + Self { + bias: Some(Array0::ones(()) * bias), + features: weights.len(), + weights, + } + } +} + +impl From<(Array1, Option)> for Node +where + T: Float + ScalarOperand, +{ + fn from((weights, bias): (Array1, Option)) -> Self { + let bias = if let Some(b) = bias { + Some(Array0::ones(()) * b) + } else { + None + }; + Self { + bias, + features: weights.len(), + weights, + } + } +} + +impl From<(Array1, Option>)> for Node +where + T: Float, +{ + fn from((weights, bias): (Array1, Option>)) -> Self { + Self { + bias, + features: weights.len(), + weights, + } + } +} + +impl From> for (Array1, Option>) +where + T: Float, +{ + fn from(node: Node) -> Self { + (node.weights, node.bias) + } +} diff --git a/ml/linear/src/cmp/neurons/perceptron.rs b/ml/linear/src/cmp/neurons/perceptron.rs new file mode 100644 index 00000000..6e1bf0cc --- /dev/null +++ b/ml/linear/src/cmp/neurons/perceptron.rs @@ -0,0 +1,212 @@ +/* + Appellation: neuron + Contrib: FL03 +*/ +use super::Node; +use crate::neural::prelude::{Activate, Forward, LinearActivation}; +use ndarray::prelude::{Array0, Array1, Array2, Ix1, NdFloat}; +use ndarray_rand::rand_distr::uniform::SampleUniform; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; +use num::Float; + +/// Artificial Neuron +#[derive(Clone, Debug, PartialEq)] +pub struct Perceptron +where + A: Activate, + T: Float, +{ + activation: A, + node: Node, +} + +impl Perceptron +where + A: Activate, + T: Float, +{ + pub fn new(features: usize) -> Self + where + A: Default, + { + Self { + activation: A::default(), + node: Node::create(false, features), + } + } + + pub fn node(&self) -> &Node { + &self.node + } + + pub fn node_mut(&mut self) -> &mut Node { + &mut self.node + } + + pub fn features(&self) -> usize { + self.node.features() + } + + pub fn params(&self) -> &Node { + &self.node + } + + pub fn params_mut(&mut self) -> &mut Node { + &mut self.node + } + + pub fn rho(&self) -> &A { + &self.activation + } + + pub fn with_bias(mut self, bias: Option>) -> Self { + self.node = self.node.with_bias(bias); + self + } + + pub fn with_rho>(self, rho: B) -> Perceptron { + Perceptron { + activation: rho, + node: self.node, + } + } + + pub fn with_node(mut self, node: Node) -> Self { + self.node = node; + self + } + + pub fn with_weights(mut self, weights: Array1) -> Self { + self.node = self.node.with_weights(weights); + self + } + + pub fn weights(&self) -> &Array1 { + self.node.weights() + } + + pub fn weights_mut(&mut self) -> &mut Array1 { + self.node.weights_mut() + } + + pub fn set_weights(&mut self, weights: Array1) { + self.node.set_weights(weights); + } +} + +impl Perceptron +where + T: NdFloat, + A: Activate, +{ + pub fn apply_gradient(&mut self, gamma: T, gradient: G) + where + G: Fn(&Array1) -> Array1, + { + let grad = gradient(self.node().weights()); + self.update_with_gradient(gamma, &grad); + } + + pub fn update_with_gradient(&mut self, gamma: T, grad: &Array1) { + self.node.weights_mut().scaled_add(-gamma, grad); + } +} + +impl Perceptron +where + T: Float + SampleUniform, + A: Activate, + StandardNormal: Distribution, +{ + pub fn init(mut self, biased: bool) -> Self { + if biased { + self = self.init_bias(); + } + self.init_weight() + } + + pub fn init_bias(mut self) -> Self { + self.node = self.node.init_bias(); + self + } + + pub fn init_weight(mut self) -> Self { + self.node = self.node.init_weight(); + self + } +} + +impl Forward> for Perceptron +where + T: NdFloat, + A: Activate, +{ + type Output = Array1; + + fn forward(&self, args: &Array2) -> Self::Output { + let linstep = self.params().forward(args); + self.rho().activate(&linstep) + } +} + +impl From<(Array1, Array0)> for Perceptron +where + T: Float, + A: Activate + Default, +{ + fn from((weights, bias): (Array1, Array0)) -> Self { + Self { + activation: A::default(), + node: Node::from((weights, bias)), + } + } +} + +impl From<(Array1, T)> for Perceptron +where + T: NdFloat, + A: Activate + Default, +{ + fn from((weights, bias): (Array1, T)) -> Self { + Self { + activation: A::default(), + node: Node::from((weights, bias)), + } + } +} + +impl From<(Array1, Array0, A)> for Perceptron +where + T: Float, + A: Activate, +{ + fn from((weights, bias, activation): (Array1, Array0, A)) -> Self { + Self { + activation, + node: Node::from((weights, bias)), + } + } +} + +impl From<(Array1, T, A)> for Perceptron +where + T: NdFloat, + A: Activate, +{ + fn from((weights, bias, activation): (Array1, T, A)) -> Self { + Self { + activation, + node: Node::from((weights, bias)), + } + } +} + +impl From> for (Array1, Option>) +where + T: Float, + A: Activate, +{ + fn from(neuron: Perceptron) -> Self { + neuron.node().clone().into() + } +} diff --git a/ml/linear/src/cmp/neurons/synapse.rs b/ml/linear/src/cmp/neurons/synapse.rs new file mode 100644 index 00000000..f04a1c80 --- /dev/null +++ b/ml/linear/src/cmp/neurons/synapse.rs @@ -0,0 +1,9 @@ +/* + Appellation: synapse + Contrib: FL03 +*/ + +pub struct Synapse { + pub layer: usize, + pub node: usize, +} diff --git a/ml/linear/src/params/features.rs b/ml/linear/src/cmp/params/features.rs similarity index 100% rename from ml/linear/src/params/features.rs rename to ml/linear/src/cmp/params/features.rs diff --git a/ml/linear/src/params/kinds.rs b/ml/linear/src/cmp/params/kinds.rs similarity index 100% rename from ml/linear/src/params/kinds.rs rename to ml/linear/src/cmp/params/kinds.rs diff --git a/ml/linear/src/params/mod.rs b/ml/linear/src/cmp/params/mod.rs similarity index 87% rename from ml/linear/src/params/mod.rs rename to ml/linear/src/cmp/params/mod.rs index e2ce6bd1..cb0782bb 100644 --- a/ml/linear/src/params/mod.rs +++ b/ml/linear/src/cmp/params/mod.rs @@ -7,3 +7,6 @@ pub use self::{features::*, kinds::*, store::*}; pub(crate) mod features; pub(crate) mod kinds; pub(crate) mod store; + +#[cfg(test)] +mod tests {} diff --git a/ml/linear/src/params/store.rs b/ml/linear/src/cmp/params/store.rs similarity index 98% rename from ml/linear/src/params/store.rs rename to ml/linear/src/cmp/params/store.rs index 0b08755c..c6e29168 100644 --- a/ml/linear/src/params/store.rs +++ b/ml/linear/src/cmp/params/store.rs @@ -3,8 +3,9 @@ Contrib: FL03 */ use super::LayerShape; +use crate::cmp::neurons::Node; use crate::core::prelude::GenerateRandom; -use crate::neural::prelude::{Features, Forward, Node}; +use crate::neural::prelude::{Features, Forward}; use ndarray::linalg::Dot; use ndarray::prelude::{Array, Array1, Array2, Axis, Dimension, NdFloat}; use ndarray::{LinalgScalar, ShapeError}; diff --git a/ml/linear/src/conv/mod.rs b/ml/linear/src/conv/mod.rs index c804f96e..73681242 100644 --- a/ml/linear/src/conv/mod.rs +++ b/ml/linear/src/conv/mod.rs @@ -2,3 +2,9 @@ Appellation: conv Contrib: FL03 */ +pub use self::module::*; + +pub(crate) mod module; + +#[cfg(test)] +mod tests {} diff --git a/ml/linear/src/conv/module.rs b/ml/linear/src/conv/module.rs new file mode 100644 index 00000000..bebd9632 --- /dev/null +++ b/ml/linear/src/conv/module.rs @@ -0,0 +1,6 @@ +/* + Appellation: conv + Contrib: FL03 +*/ + +pub struct Conv; diff --git a/ml/linear/src/dense/mod.rs b/ml/linear/src/dense/mod.rs index db9dd16e..1ccf1162 100644 --- a/ml/linear/src/dense/mod.rs +++ b/ml/linear/src/dense/mod.rs @@ -2,3 +2,9 @@ Appellation: dense Contrib: FL03 */ +pub use self::module::*; + +pub(crate) mod module; + +#[cfg(test)] +mod tests {} diff --git a/ml/linear/src/dense/module.rs b/ml/linear/src/dense/module.rs new file mode 100644 index 00000000..413aac98 --- /dev/null +++ b/ml/linear/src/dense/module.rs @@ -0,0 +1,6 @@ +/* + Appellation: module + Contrib: FL03 +*/ + +pub struct Dense; diff --git a/ml/linear/src/lib.rs b/ml/linear/src/lib.rs index 9a1908a6..2faf5b39 100644 --- a/ml/linear/src/lib.rs +++ b/ml/linear/src/lib.rs @@ -7,10 +7,10 @@ //! This library implements the framework for building linear models. //! +pub mod cmp; pub mod conv; pub mod dense; pub mod model; -pub mod params; pub(crate) use concision_core as core; pub(crate) use concision_neural as neural; diff --git a/ml/linear/src/model/layer.rs b/ml/linear/src/model/layer.rs index 867ce99f..725e6f14 100644 --- a/ml/linear/src/model/layer.rs +++ b/ml/linear/src/model/layer.rs @@ -2,9 +2,11 @@ Appellation: model Contrib: FL03 */ -use crate::neural::func::activate::{Activate, Gradient}; -use crate::neural::prelude::{Features, Forward, Node, Perceptron}; -use crate::params::{LayerShape, LinearParams as LayerParams}; +use crate::cmp::neurons::{Node, Perceptron}; +use crate::cmp::params::LayerShape; +use crate::cmp::params::LinearParams as LayerParams; +use crate::neural::prelude::{Activate, Features, Forward, Gradient}; + use ndarray::prelude::{Array2, Ix1, NdFloat}; use ndarray::ShapeError; use ndarray_rand::rand_distr::uniform::SampleUniform; diff --git a/ml/linear/src/model/mod.rs b/ml/linear/src/model/mod.rs index c86b4d21..d16397bb 100644 --- a/ml/linear/src/model/mod.rs +++ b/ml/linear/src/model/mod.rs @@ -13,9 +13,10 @@ pub(crate) mod module; #[cfg(test)] mod tests { use super::*; + use crate::cmp::neurons::Node; + use crate::cmp::params::LayerShape; use crate::core::prelude::linarr; - use crate::neural::prelude::{Forward, Node, Softmax}; - use crate::params::LayerShape; + use crate::neural::prelude::{Forward, Softmax}; use ndarray::prelude::Ix2; #[test] diff --git a/ml/neural/src/errors/error.rs b/ml/neural/src/errors/error.rs index e5e7ce73..d51d1243 100644 --- a/ml/neural/src/errors/error.rs +++ b/ml/neural/src/errors/error.rs @@ -26,12 +26,12 @@ use strum::{Display, EnumIs, EnumIter, EnumVariantNames}; #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] pub enum MlError { + Compute(ComputeError), Data(String), Dimension(String), #[default] Error(String), Network(NetworkError), - Process(ProcessError), } impl std::error::Error for MlError {} @@ -103,7 +103,7 @@ pub enum PredictError { #[non_exhaustive] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] -pub enum ProcessError { +pub enum ComputeError { Arithmetic(String), #[default] Process(String), diff --git a/ml/neural/src/func/activate/activator.rs b/ml/neural/src/func/activate/activator.rs index d4af82e7..2a4f044f 100644 --- a/ml/neural/src/func/activate/activator.rs +++ b/ml/neural/src/func/activate/activator.rs @@ -40,7 +40,7 @@ where T: Clone, { pub fn linear() -> Self { - Self::new(Box::new(super::Linear::new())) + Self::new(Box::new(super::LinearActivation::new())) } } diff --git a/ml/neural/src/func/activate/linear.rs b/ml/neural/src/func/activate/linear.rs index 3836e9f6..0d6608e2 100644 --- a/ml/neural/src/func/activate/linear.rs +++ b/ml/neural/src/func/activate/linear.rs @@ -10,9 +10,9 @@ use serde::{Deserialize, Serialize}; #[derive( Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] -pub struct Linear; +pub struct LinearActivation; -impl Linear { +impl LinearActivation { pub fn new() -> Self { Self::default() } @@ -33,7 +33,7 @@ impl Linear { } pub fn linear(args: &T) -> T { - Linear::method()(args) + LinearActivation::method()(args) } pub fn method() -> fn(&T) -> T { @@ -45,7 +45,7 @@ impl Linear { } } -impl Gradient for Linear +impl Gradient for LinearActivation where D: Dimension, T: Clone + One, @@ -55,7 +55,7 @@ where } } -impl Fn<(&T,)> for Linear +impl Fn<(&T,)> for LinearActivation where T: Clone, { @@ -64,7 +64,7 @@ where } } -impl FnMut<(&T,)> for Linear +impl FnMut<(&T,)> for LinearActivation where T: Clone, { @@ -73,7 +73,7 @@ where } } -impl FnOnce<(&T,)> for Linear +impl FnOnce<(&T,)> for LinearActivation where T: Clone, { diff --git a/ml/neural/src/func/activate/mod.rs b/ml/neural/src/func/activate/mod.rs index 9bec62b9..729b9357 100644 --- a/ml/neural/src/func/activate/mod.rs +++ b/ml/neural/src/func/activate/mod.rs @@ -140,7 +140,7 @@ mod tests { let exp = array![0.0, 1.0, 2.0]; let args = array![0.0, 1.0, 2.0]; - assert_eq!(Linear::new().activate(&args), exp); - assert_eq!(Linear(&args), exp); + assert_eq!(LinearActivation::new().activate(&args), exp); + assert_eq!(LinearActivation(&args), exp); } } diff --git a/ml/neural/src/func/block.rs b/ml/neural/src/func/block.rs index 02d6669c..24974478 100644 --- a/ml/neural/src/func/block.rs +++ b/ml/neural/src/func/block.rs @@ -2,7 +2,7 @@ Appellation: block Contrib: FL03 */ -use crate::func::activate::{Activate, Linear, ReLU, Softmax}; +use crate::func::activate::{Activate, LinearActivation, ReLU, Softmax}; use num::Float; use std::marker::PhantomData; @@ -15,7 +15,7 @@ pub struct FuncBlock { method: Vec T>, } -pub struct FFNBlock +pub struct FFNBlock where I: Activate, H: Activate, diff --git a/ml/neural/src/layers/exp/sublayer.rs b/ml/neural/src/layers/exp/sublayer.rs index 4dc297c5..49f4df8b 100644 --- a/ml/neural/src/layers/exp/sublayer.rs +++ b/ml/neural/src/layers/exp/sublayer.rs @@ -3,14 +3,14 @@ Contrib: FL03 */ use crate::layers::Layer; -use crate::prelude::{Activate, Forward, LayerNorm, Linear}; +use crate::prelude::{Activate, Forward, LayerNorm, LinearActivation}; use ndarray::prelude::{Array2, NdFloat}; use num::{Float, FromPrimitive}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct Sublayer +pub struct Sublayer where A: Activate, T: Float, diff --git a/ml/neural/src/layers/layer.rs b/ml/neural/src/layers/layer.rs index 02edb0b6..92002e80 100644 --- a/ml/neural/src/layers/layer.rs +++ b/ml/neural/src/layers/layer.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use super::{LayerParams, LayerShape}; -use crate::func::activate::{Activate, Gradient, Linear}; +use crate::func::activate::{Activate, Gradient, LinearActivation}; use crate::prelude::{Features, Forward, Node, Perceptron}; use ndarray::prelude::{Array2, Ix1, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; @@ -13,7 +13,7 @@ use num::{Float, Signed}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct Layer +pub struct Layer where A: Activate, T: Float, diff --git a/ml/neural/src/layers/stack.rs b/ml/neural/src/layers/stack.rs index 6fbaedef..e5d18792 100644 --- a/ml/neural/src/layers/stack.rs +++ b/ml/neural/src/layers/stack.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use crate::layers::{Layer, LayerShape}; -use crate::prelude::{Activate, Features, Linear}; +use crate::prelude::{Activate, Features, LinearActivation}; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use num::Float; @@ -30,7 +30,7 @@ where /// A [Stack] is a collection of [Layer]s, typically used to construct the hidden /// layers of a deep neural network. #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -pub struct Stack +pub struct Stack where A: Activate, T: Float, diff --git a/ml/neural/src/neurons/perceptron.rs b/ml/neural/src/neurons/perceptron.rs index 916c1bb3..8626eee3 100644 --- a/ml/neural/src/neurons/perceptron.rs +++ b/ml/neural/src/neurons/perceptron.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use super::Node; -use crate::prelude::{Activate, Forward, Linear}; +use crate::prelude::{Activate, Forward, LinearActivation}; use ndarray::prelude::{Array0, Array1, Array2, Ix1, NdFloat}; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, StandardNormal}; @@ -11,7 +11,7 @@ use num::Float; /// Artificial Neuron #[derive(Clone, Debug, PartialEq)] -pub struct Perceptron +pub struct Perceptron where A: Activate, T: Float, diff --git a/ml/neural/src/nn/ffn/mlp.rs b/ml/neural/src/nn/ffn/mlp.rs index bf1d4d45..0fa16c6c 100644 --- a/ml/neural/src/nn/ffn/mlp.rs +++ b/ml/neural/src/nn/ffn/mlp.rs @@ -5,7 +5,7 @@ //! # Multi-Layer Perceptron //! -use crate::func::activate::{Activate, Linear, ReLU, Softmax}; +use crate::func::activate::{Activate, LinearActivation, ReLU, Softmax}; use crate::layers::{Layer, LayerShape, Stack}; use crate::prelude::{Features, Forward}; @@ -14,7 +14,7 @@ use ndarray::prelude::{Array2, Ix2, NdFloat}; use ndarray::IntoDimension; use num::Float; -pub struct MLP +pub struct MLP where T: Float, I: Activate, diff --git a/ml/optim/src/grad/mod.rs b/ml/optim/src/grad/mod.rs index c18fda19..d9ebf164 100644 --- a/ml/optim/src/grad/mod.rs +++ b/ml/optim/src/grad/mod.rs @@ -101,7 +101,7 @@ mod tests { use super::*; use crate::core::prelude::linarr; - use crate::neural::func::activate::{Linear, Sigmoid}; + use crate::neural::func::activate::{LinearActivation, Sigmoid}; use crate::neural::models::ModelParams; use crate::neural::prelude::{Features, Forward, Layer, LayerShape}; use ndarray::prelude::{Array1, Ix2}; @@ -145,7 +145,7 @@ mod tests { let x = linarr::((samples, features.inputs())).unwrap(); let _y = linarr::((samples, features.outputs())).unwrap(); - let model = Layer::::from(features).init(true); + let model = Layer::::from(features).init(true); let _pred = model.forward(&x); diff --git a/ml/s4/examples/sand.rs b/ml/s4/examples/sand.rs index 06e70af3..862f96f2 100644 --- a/ml/s4/examples/sand.rs +++ b/ml/s4/examples/sand.rs @@ -4,6 +4,7 @@ use concision_s4 as s4; use s4::randcomplex; use ndarray::prelude::Ix2; +use num::complex::Complex; fn main() -> anyhow::Result<()> { let c = randcomplex::([2, 2]); diff --git a/ml/s4/src/hippo/dplr.rs b/ml/s4/src/hippo/dplr.rs index 58333e90..495bad41 100644 --- a/ml/s4/src/hippo/dplr.rs +++ b/ml/s4/src/hippo/dplr.rs @@ -48,6 +48,7 @@ where pub(crate) fn dplr(features: usize) -> DPLR where T: DPLRScalar, + Complex: Lapack, ::Real: Mul, Output = Complex>, { @@ -55,12 +56,14 @@ where // let s = { + // reshape the p-array from NPLR into a two-dimensional matrix let p2 = p.clone().insert_axis(Axis(1)); + // compute s &a + p2.dot(&p2.t()) }; - // + // find the diagonal of s let sd = s.diag(); - + // create a matrix from the diagonals of s let lambda_re = Array::ones(sd.dim()) * sd.mean().expect(""); let (e, v) = s diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index 7c55850b..8acfd97a 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -25,7 +25,7 @@ where pub fn randcomplex(shape: impl IntoDimension) -> Array, D> where D: Dimension, - T: ComplexFloat, + T: Copy + Num, StandardNormal: Distribution, { let dim = shape.into_dimension(); From 5645165aa7f9c6a417a9e079662be76cb2d4d90e Mon Sep 17 00:00:00 2001 From: FL03 Date: Fri, 5 Jan 2024 13:50:54 -0600 Subject: [PATCH 107/118] update Signed-off-by: FL03 --- core/src/specs/arrays.rs | 34 +++++++++ core/src/specs/math.rs | 38 +++++++++- core/src/specs/numerical.rs | 72 +++++++++++++------ core/src/utils.rs | 17 ++--- data/src/misc/dtype.rs | 133 ++++++++++++++++++++++++++++++++++- data/src/shape/rank.rs | 46 ++++++++++++ data/src/store/mod.rs | 7 ++ data/src/tensors/tensor.rs | 2 + ml/s4/Cargo.toml | 8 ++- ml/s4/examples/sand.rs | 5 ++ ml/s4/src/cmp/cache.rs | 84 +++++++++++++++++++++- ml/s4/src/cmp/kernel.rs | 118 ++++++++++++++++++++++++++++++- ml/s4/src/hippo/hippo.rs | 3 +- ml/s4/src/hippo/mod.rs | 11 ++- ml/s4/src/hippo/nplr.rs | 8 +-- ml/s4/src/lib.rs | 1 + ml/s4/src/ops/convolve.rs | 21 +++--- ml/s4/src/ops/discretize.rs | 135 +++++++++++++++++++++++++----------- ml/s4/src/ssm/model.rs | 59 +++------------- ml/s4/src/utils.rs | 37 +++++++--- ml/s4/tests/dplr.rs | 82 ++++++++++++++++++++++ 21 files changed, 752 insertions(+), 169 deletions(-) create mode 100644 ml/s4/tests/dplr.rs diff --git a/core/src/specs/arrays.rs b/core/src/specs/arrays.rs index ac248da1..14003d83 100644 --- a/core/src/specs/arrays.rs +++ b/core/src/specs/arrays.rs @@ -32,6 +32,40 @@ where } } +pub enum ArangeArgs { + Arange { start: T, stop: T, step: T }, + Between { start: T, stop: T }, + Until { stop: T }, +} + +impl From<(T, T, T)> for ArangeArgs { + fn from(args: (T, T, T)) -> Self { + ArangeArgs::Arange { + start: args.0, + stop: args.1, + step: args.2, + } + } +} + +impl From<(T, T)> for ArangeArgs { + fn from(args: (T, T)) -> Self { + ArangeArgs::Between { + start: args.0, + stop: args.1, + } + } +} + +impl From for ArangeArgs +where + T: Num, +{ + fn from(args: T) -> Self { + ArangeArgs::Until { stop: args } + } +} + pub trait Arange { fn arange(start: T, stop: T, step: T) -> Self; } diff --git a/core/src/specs/math.rs b/core/src/specs/math.rs index 8c763ea7..8d4eb823 100644 --- a/core/src/specs/math.rs +++ b/core/src/specs/math.rs @@ -2,9 +2,10 @@ Appellation: math Contrib: FL03 */ +use ndarray::linalg::Dot; use ndarray::prelude::{Array, Dimension, Ix2}; use num::complex::Complex; -use num::{Float, Num, Signed}; +use num::{Float, Integer, Signed}; use std::ops; pub trait Conjugate { @@ -25,10 +26,10 @@ impl Conjugate for f64 { impl Conjugate for Complex where - T: Clone + Num + Signed, + T: Clone + Signed, { fn conj(&self) -> Self { - Complex::::new(self.re.clone(), -self.im.clone()) + Complex::::conj(self) } } @@ -130,3 +131,34 @@ where self.mapv(|x| x.sqrt()) } } + +pub trait Power { + type Output; + + fn pow(&self, rhs: Rhs) -> Self::Output; +} + +// impl Power for S where S: Pow { +// type Output = >::Output; + +// fn pow(self, rhs: T) -> Self::Output { +// >::pow(self, rhs) +// } +// } + +impl Power for Array +where + D: Dimension, + T: Clone, + Array: Dot, +{ + type Output = Self; + + fn pow(&self, rhs: usize) -> Self::Output { + let mut res = self.clone(); + for _ in 1..rhs { + res = res.dot(&self); + } + res + } +} diff --git a/core/src/specs/numerical.rs b/core/src/specs/numerical.rs index bf726adb..053cb841 100644 --- a/core/src/specs/numerical.rs +++ b/core/src/specs/numerical.rs @@ -2,7 +2,9 @@ Appellation: num Contrib: FL03 */ -use std::ops::{self, Add, Div, Mul, Sub}; +use num::complex::Complex; +use num::traits::{NumAssignOps, NumOps, Signed}; +use std::ops::{Add, Div, Mul, Sub}; pub trait Algebraic where @@ -13,11 +15,7 @@ where pub trait AlgebraicExt where - Self: Algebraic - + ops::AddAssign - + ops::DivAssign - + ops::MulAssign - + ops::SubAssign, + Self: Algebraic + NumAssignOps, { } @@ -28,36 +26,42 @@ where type Output = C; } -impl AlgebraicExt for A where - A: Algebraic - + ops::AddAssign - + ops::DivAssign - + ops::MulAssign - + ops::SubAssign +impl AlgebraicExt for A where A: Algebraic + NumAssignOps {} + +pub trait ComplexNum +where + Self: Algebraic + Algebraic, { + type DType; + + fn complex(self) -> Self; + + fn im(self) -> Self::DType; + + fn re(self) -> Self::DType; } -pub trait ComplexNum: +pub trait Imaginary: Algebraic + Algebraic { type DType; - fn imag(self) -> Self::DType; + fn im(self) -> Self::DType; - fn real(self) -> Self::DType; + fn re(self) -> Self::DType; } -impl ComplexNum for num::Complex +impl Imaginary for num::Complex where T: Clone + num::Num, { type DType = T; - fn imag(self) -> Self::DType { + fn im(self) -> Self::DType { self.im } - fn real(self) -> Self::DType { + fn re(self) -> Self::DType { self.re } } @@ -94,8 +98,36 @@ impl Number for f64 {} impl Number for S where S: ComplexNum {} -pub trait Numerical +pub trait Abs { + fn abs(&self) -> Self; +} + +impl Abs for T where - T: Number, + T: Signed, { + fn abs(&self) -> Self { + Signed::abs(self) + } +} +pub trait Scalar { + type Complex: NumOps + NumOps; + type Real: NumOps + NumOps; +} + +pub trait Numerical: Algebraic { + type Elem: Algebraic + Number; + + fn abs(&self) -> Self + where + Self::Elem: Abs, + { + self.eval(|x| x.abs()) + } + + fn conj(self) -> Self; + + fn eval(&self, f: F) -> Self + where + F: Fn(Self::Elem) -> Self::Elem; } diff --git a/core/src/utils.rs b/core/src/utils.rs index a37a6585..580f88bb 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -5,8 +5,6 @@ use ndarray::prelude::*; use ndarray::{concatenate, IntoDimension, RemoveAxis, ShapeError}; -use ndarray_rand::rand_distr::{Bernoulli, BernoulliError}; -use ndarray_rand::RandomExt; use num::cast::AsPrimitive; use num::{Float, Num, NumCast, Zero}; @@ -23,17 +21,6 @@ where res } -pub fn bernoulli( - dim: impl IntoDimension, - p: Option, -) -> Result, BernoulliError> -where - D: Dimension, -{ - let dist = Bernoulli::new(p.unwrap_or(0.5))?; - Ok(Array::random(dim.into_dimension(), dist)) -} - pub fn cauchy_dot(a: &Array, lambda: &Array, omega: &Array) -> T where D: Dimension, @@ -99,6 +86,10 @@ where out } +pub fn genspace(features: usize) -> Array1 { + Array1::from_iter((0..features).map(|x| T::from(x).unwrap())) +} + pub fn linarr(dim: impl IntoDimension) -> Result, ShapeError> where D: Dimension, diff --git a/data/src/misc/dtype.rs b/data/src/misc/dtype.rs index 49fefcff..5b8d1ad2 100644 --- a/data/src/misc/dtype.rs +++ b/data/src/misc/dtype.rs @@ -2,6 +2,9 @@ Appellation: dtype Contrib: FL03 */ +use serde::{Deserialize, Serialize}; +use smart_default::SmartDefault; +use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; pub trait DataType { fn dtype(&self) -> DType; @@ -16,12 +19,43 @@ where } } +#[derive( + Clone, + Copy, + Debug, + Deserialize, + Display, + EnumCount, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, + SmartDefault, +)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] pub enum DType { + #[default] FloatingPoint(FloatingPoint), Integer(Integer), Unsigned(Unsigned), } +impl DType { + pub fn detect(var: T) -> Self + where + T: Clone + Default + Into, + { + var.dtype() + } +} + impl From for DType { fn from(_: f32) -> Self { DType::FloatingPoint(FloatingPoint::F32) @@ -106,8 +140,30 @@ impl From for DType { } } +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumCount, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] pub enum FloatingPoint { F32, + #[default] F64, } @@ -129,12 +185,66 @@ impl From for DType { } } +pub struct Int { + size: IntSize, +} +#[derive( + Clone, + Copy, + Debug, + Deserialize, + Display, + EnumCount, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum IntSize { + S8 = 8, + S16 = 16, + S32 = 32, + S64 = 64, + S128 = 128, + SSize, +} + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumCount, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] pub enum Integer { I8, I16, I32, I64, I128, + #[default] ISIZE, } @@ -179,13 +289,34 @@ impl From for DType { DType::Integer(dtype) } } - +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumCount, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] pub enum Unsigned { U8, U16, U32, U64, U128, + #[default] USIZE, } diff --git a/data/src/shape/rank.rs b/data/src/shape/rank.rs index 24bda35a..99d94e8c 100644 --- a/data/src/shape/rank.rs +++ b/data/src/shape/rank.rs @@ -2,6 +2,10 @@ Appellation: rank Contrib: FL03 */ +//! # Rank +//! +//! The rank of a n-dimensional array describes the number of dimensions +use serde::{Deserialize, Serialize}; pub enum Ranks { Zero(T), @@ -9,6 +13,10 @@ pub enum Ranks { N(Vec), } +#[derive( + Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, +)] +#[serde(rename_all = "lowercase")] pub struct Rank(pub usize); impl Rank { @@ -20,3 +28,41 @@ impl Rank { self.0 } } + +impl AsRef for Rank { + fn as_ref(&self) -> &usize { + &self.0 + } +} + +impl AsMut for Rank { + fn as_mut(&mut self) -> &mut usize { + &mut self.0 + } +} + +impl From for Rank { + fn from(rank: usize) -> Self { + Self(rank) + } +} + +impl From for usize { + fn from(rank: Rank) -> Self { + rank.0 + } +} + +// impl TryFrom for Rank +// where +// T: NumCast, +// { +// type Error = Box; + +// fn try_from(value: T) -> Result { +// if let Some(rank) = ::from(value) { +// return Ok(Self(rank)); +// } +// Err("Could not convert to Rank".into()) +// } +// } diff --git a/data/src/store/mod.rs b/data/src/store/mod.rs index a62b8cfe..1808329f 100644 --- a/data/src/store/mod.rs +++ b/data/src/store/mod.rs @@ -9,8 +9,13 @@ pub(crate) mod layout; pub(crate) mod storage; use std::collections::{BTreeMap, HashMap}; +use std::ops; pub trait Store { + fn contains(&self, key: &K) -> bool { + self.get(key).is_some() + } + fn get(&self, key: &K) -> Option<&V>; fn get_mut(&mut self, key: &K) -> Option<&mut V>; @@ -20,6 +25,8 @@ pub trait Store { fn remove(&mut self, key: &K) -> Option; } +pub trait StoreExt: Store + ops::Index {} + impl Store for BTreeMap where K: Ord, diff --git a/data/src/tensors/tensor.rs b/data/src/tensors/tensor.rs index 38b82137..07319a9f 100644 --- a/data/src/tensors/tensor.rs +++ b/data/src/tensors/tensor.rs @@ -17,6 +17,7 @@ where { id: AtomicId, data: Array, + dtype: DType, } impl Tensor @@ -28,6 +29,7 @@ where Self { id: AtomicId::new(), data: Array::zeros(shape), + dtype: DType::default(), } } } diff --git a/ml/s4/Cargo.toml b/ml/s4/Cargo.toml index 368e3388..5941514a 100644 --- a/ml/s4/Cargo.toml +++ b/ml/s4/Cargo.toml @@ -15,9 +15,10 @@ version.workspace = true default = ["blas"] blas = [ - # "ndarray/blas", - # "concision-core/blas", - # "concision-neural/blas", + "ndarray/blas", + "concision-core/blas", + "concision-data/blas", + "concision-neural/blas", ] intel-mkl-system = [ @@ -62,6 +63,7 @@ test = true [dependencies] concision-core = { features = ["blas"], path = "../../core", version = "0.1.12" } +concision-data = { features = ["blas"], path = "../../data", version = "0.1.12" } concision-neural = { features = ["blas"], path = "../neural" } anyhow.workspace = true diff --git a/ml/s4/examples/sand.rs b/ml/s4/examples/sand.rs index 862f96f2..3e88cb1a 100644 --- a/ml/s4/examples/sand.rs +++ b/ml/s4/examples/sand.rs @@ -1,12 +1,17 @@ // use concision_core as cnc; +extern crate concision_s4; + use concision_s4 as s4; use s4::randcomplex; use ndarray::prelude::Ix2; +use ndarray_linalg::Scalar; use num::complex::Complex; fn main() -> anyhow::Result<()> { + let i = Complex::::i(); + println!("{:?}", i.add_real(1.0)); let c = randcomplex::([2, 2]); println!("{:?}", &c); diff --git a/ml/s4/src/cmp/cache.rs b/ml/s4/src/cmp/cache.rs index 3d0fa7e9..32202017 100644 --- a/ml/s4/src/cmp/cache.rs +++ b/ml/s4/src/cmp/cache.rs @@ -2,12 +2,94 @@ Appellation: cache Contrib: FL03 */ +use crate::data::prelude::Store; use ndarray::prelude::{Array, Dimension, Ix2}; // use num::{Complex, Float}; +use std::collections::HashMap; pub struct Cache where D: Dimension, { - cache: Array, + cache: HashMap>, +} + +impl Cache +where + D: Dimension, +{ + pub fn new() -> Self { + Self { + cache: HashMap::new(), + } + } +} + +impl Store> for Cache +where + D: Dimension, +{ + fn get(&self, key: &String) -> Option<&Array> { + self.cache.get(key) + } + + fn get_mut(&mut self, key: &String) -> Option<&mut Array> { + self.cache.get_mut(key) + } + + fn insert(&mut self, key: String, value: Array) -> Option> { + self.cache.insert(key, value) + } + + fn remove(&mut self, key: &String) -> Option> { + self.cache.remove(key) + } +} + +impl Extend<(String, Array)> for Cache +where + D: Dimension, +{ + fn extend)>>(&mut self, iter: I) { + for (key, value) in iter { + self.insert(key, value); + } + } +} + +impl FromIterator<(String, Array)> for Cache +where + D: Dimension, +{ + fn from_iter)>>(iter: I) -> Self { + let mut cache = Self::new(); + for (key, value) in iter { + cache.insert(key, value); + } + cache + } +} + +impl IntoIterator for Cache +where + D: Dimension, +{ + type Item = (String, Array); + type IntoIter = std::collections::hash_map::IntoIter>; + + fn into_iter(self) -> Self::IntoIter { + self.cache.into_iter() + } +} + +impl<'a, T, D> IntoIterator for &'a mut Cache +where + D: Dimension, +{ + type Item = (&'a String, &'a mut Array); + type IntoIter = std::collections::hash_map::IterMut<'a, String, Array>; + + fn into_iter(self) -> Self::IntoIter { + self.cache.iter_mut() + } } diff --git a/ml/s4/src/cmp/kernel.rs b/ml/s4/src/cmp/kernel.rs index 073f331e..360a9e12 100644 --- a/ml/s4/src/cmp/kernel.rs +++ b/ml/s4/src/cmp/kernel.rs @@ -2,8 +2,122 @@ Appellation: kernel Contrib: FL03 */ -use ndarray::prelude::Array2; -use num::Float; +use crate::core::prelude::{AsComplex, Conjugate}; +use crate::prelude::{cauchy, cauchy_complex}; +use ndarray::prelude::{Array, Array1, Array2, Ix1, NdFloat}; +use ndarray::ScalarOperand; +use ndarray_linalg::Scalar; +use num::complex::{Complex, ComplexFloat}; +use num::traits::{Float, FloatConst, FromPrimitive, NumOps, Signed, Zero}; +use rustfft::{FftNum, FftPlanner}; +use std::ops::Neg; + +pub struct DPLRParams { + pub lambda: Array1, + pub p: Array1, + pub q: Array1, + pub b: Array1, + pub c: Array1, +} + +impl DPLRParams { + pub fn new(lambda: Array1, p: Array1, q: Array1, b: Array1, c: Array1) -> Self { + Self { lambda, p, q, b, c } + } +} + +// impl DPLRParams +// where +// T: ComplexFloat, +// ::Real: NumOps + NumOps::Real>, Complex<::Real>>, +// Complex<::Real>: NumOps + NumOps<::Real, Complex<::Real>> +// { +// pub fn kernel(&self, step: T, l: usize) -> Array1<::Real> { +// let lt = T::from(l).unwrap(); +// let omega_l = { +// let f = | i: usize | -> Complex<::Real> { +// Complex::<::Real>::i().neg() * ::Real::from(i).unwrap() * ::Real::PI() / lt +// }; +// Array::from_iter((0..l).map(f)) +// }; +// } +// } + +impl DPLRParams> { + pub fn kernel_s(&self, step: f64, l: usize) -> Array1 { + let lt = l as f64; // T::from(l).unwrap(); + let omega_l = { + let f = |i: usize| -> Complex { + (Complex::i().neg() * (i as f64) * f64::PI() / lt).exp() + }; + Array::from_iter((0..l).map(f)) + }; + + let aterm = (self.c.conj(), self.q.conj()); + let bterm = (self.b.clone(), self.p.clone()); + + let g = ((&omega_l.clone().neg() + 1.0) / (&omega_l + 1.0)) * (2.0 * step.recip()); + let c = omega_l.mapv(|i| 2.0 / (1.0 + i)); + + let k00 = cauchy_complex(&(&aterm.0 * &bterm.0), &g, &self.lambda); + let k01 = cauchy_complex(&(&aterm.0 * &bterm.1), &g, &self.lambda); + let k10 = cauchy_complex(&(&aterm.1 * &bterm.0), &g, &self.lambda); + let k11 = cauchy_complex(&(&aterm.1 * &bterm.1), &g, &self.lambda); + + let at_roots = &c * (&k00 - k01 * (&k11 + 1.0).mapv(ComplexFloat::recip) * &k10); + + let mut fft_planner = FftPlanner::new(); + let fft = fft_planner.plan_fft_inverse(l); + // create a buffer to hold the complex numbers + let mut buffer = at_roots.into_raw_vec(); + fft.process(buffer.as_mut_slice()); + Array::from_iter(buffer.into_iter().map(|i| i.re())) + } +} + +pub fn kernel_dplr( + dplr: &DPLRParams<::Complex>, + step: ::Real, + l: usize, +) -> Array1<::Real> +where + T: Conjugate + FftNum + Float + Scalar>, + ::Real: + FloatConst + NumOps<::Complex, ::Complex> + ScalarOperand, + ::Complex: ScalarOperand, +{ + let lt = ::Real::from(l).unwrap(); + let two = ::Real::from(2).unwrap(); + let omega_l: Array1<::Complex> = { + let f = |i: usize| -> ::Complex { + ((::Real::from(i).unwrap() * ::Real::PI()) / lt) + .mul_complex(Complex::i().neg()) + .exp() + }; + Array::from_iter((0..l).map(f)) + }; + + let aterm = (dplr.c.conj(), dplr.q.conj()); + let bterm = (dplr.b.clone(), dplr.p.clone()); + + let g = ((&omega_l.clone().neg() + ::Real::one()) / (&omega_l + T::one())) + * (T::one() * step.recip()); + let c = omega_l.mapv(|i| two.div_complex(::Real::one().add_complex(i))); + + let k00: Array1<::Complex> = cauchy(&(&aterm.0 * &bterm.0), &g, &dplr.lambda); + let k01: Array1<::Complex> = cauchy(&(&aterm.0 * &bterm.1), &g, &dplr.lambda); + let k10: Array1<::Complex> = cauchy(&(&aterm.1 * &bterm.0), &g, &dplr.lambda); + let k11: Array1<::Complex> = cauchy(&(&aterm.1 * &bterm.1), &g, &dplr.lambda); + + let at_roots = &c * (&k00 - k01 * (&k11 + T::one()).mapv(ComplexFloat::recip) * &k10); + + let mut fft_planner = FftPlanner::new(); + let fft = fft_planner.plan_fft_inverse(l); + // create a buffer to hold the complex numbers + let mut buffer = at_roots.into_raw_vec(); + fft.process(buffer.as_mut_slice()); + Array::from_iter(buffer.into_iter().map(|i| i.re())) +} pub struct Kernel { kernal: Array2, diff --git a/ml/s4/src/hippo/hippo.rs b/ml/s4/src/hippo/hippo.rs index 5e4e4220..9ffaff30 100644 --- a/ml/s4/src/hippo/hippo.rs +++ b/ml/s4/src/hippo/hippo.rs @@ -4,8 +4,7 @@ */ // use super::dplr::DPLR; use super::nplr::NPLR; -use super::utils::genspace; -use crate::core::prelude::SquareRoot; +use crate::core::prelude::{genspace, SquareRoot}; use ndarray::prelude::Array2; use ndarray::ScalarOperand; use num::traits::{Num, NumCast, Signed}; diff --git a/ml/s4/src/hippo/mod.rs b/ml/s4/src/hippo/mod.rs index 1b6e6a8e..8a0a2bbf 100644 --- a/ml/s4/src/hippo/mod.rs +++ b/ml/s4/src/hippo/mod.rs @@ -5,7 +5,8 @@ //! # HiPPO //! //! -pub use self::{hippo::*, kinds::*, utils::*}; +pub(crate) use self::utils::*; +pub use self::{hippo::*, kinds::*}; pub(crate) mod hippo; pub(crate) mod kinds; @@ -18,16 +19,12 @@ pub struct LowRank { } pub(crate) mod utils { - use crate::core::prelude::SquareRoot; - use ndarray::prelude::{Array1, Array2, Axis}; + use crate::core::prelude::{genspace, SquareRoot}; + use ndarray::prelude::{Array2, Axis}; use ndarray::ScalarOperand; use ndarray_linalg::{IntoTriangular, UPLO}; use num::traits::{Num, NumCast, Signed}; - pub fn genspace(features: usize) -> Array1 { - Array1::from_iter((0..features).map(|x| T::from(x).unwrap())) - } - pub(crate) fn hippo(features: usize) -> Array2 where T: Num + NumCast + ScalarOperand + Signed + SquareRoot, diff --git a/ml/s4/src/hippo/nplr.rs b/ml/s4/src/hippo/nplr.rs index adc5a5ad..771c88e2 100644 --- a/ml/s4/src/hippo/nplr.rs +++ b/ml/s4/src/hippo/nplr.rs @@ -5,9 +5,9 @@ //! # Normal Plus Low Rank (NPLR) //! //! -use super::utils::*; +use super::HiPPO; -use crate::core::prelude::SquareRoot; +use crate::core::prelude::{genspace, SquareRoot}; use ndarray::prelude::{Array1, Array2}; use ndarray::ScalarOperand; use num::traits::{Num, NumCast, Signed}; @@ -17,12 +17,12 @@ fn nplr(features: usize) -> (Array2, Array1, Array1) where T: Num + NumCast + ScalarOperand + Signed + SquareRoot, { - let hippo = hippo::(features); + let hippo = HiPPO::::new(features); let base = genspace::(features); let p = (&base + (T::one() / T::from(2).unwrap())).mapv(T::sqrt); let b = (&base * T::from(2).unwrap() + T::one()).mapv(T::sqrt); - (hippo, p, b) + (hippo.into(), p, b) } #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] diff --git a/ml/s4/src/lib.rs b/ml/s4/src/lib.rs index 09faeadc..0142d999 100644 --- a/ml/s4/src/lib.rs +++ b/ml/s4/src/lib.rs @@ -19,6 +19,7 @@ pub mod params; pub mod ssm; pub(crate) use concision_core as core; +pub(crate) use concision_data as data; pub(crate) use concision_neural as neural; pub mod prelude { diff --git a/ml/s4/src/ops/convolve.rs b/ml/s4/src/ops/convolve.rs index 2ab15635..10f65d13 100644 --- a/ml/s4/src/ops/convolve.rs +++ b/ml/s4/src/ops/convolve.rs @@ -2,21 +2,20 @@ Appellation: convolve Contrib: FL03 */ +use crate::core::prelude::Power; use crate::prelude::powmat; -use ndarray::prelude::{s, Array2, Axis, NdFloat}; +use ndarray::linalg::Dot; +use ndarray::prelude::{s, Array, Array1, Array2, Axis}; +use ndarray::ScalarOperand; +use ndarray_linalg::flatten; +use num::Num; pub fn convolve() {} -pub fn k_convolve(a: &Array2, b: &Array2, c: &Array2, l: usize) -> Array2 +pub fn k_convolve(a: &Array2, b: &Array2, c: &Array2, l: usize) -> Array1 where - T: NdFloat, + T: Num + ScalarOperand, + Array2: Dot, Output = Array2> + Dot, Output = Array1>, { - let b = b.clone().remove_axis(Axis(1)); - let mut res = Array2::::zeros((l, a.shape()[0])); - for i in 0..l { - let tmp = powmat(a, i); - let out = c.dot(&tmp.dot(&b)); - res.slice_mut(s![i, ..]).assign(&out); - } - res + Array::from_iter((0..l).map(|i| c.dot(&a.pow(i).dot(b)).sum())) } diff --git a/ml/s4/src/ops/discretize.rs b/ml/s4/src/ops/discretize.rs index 922b5d24..57ef2954 100644 --- a/ml/s4/src/ops/discretize.rs +++ b/ml/s4/src/ops/discretize.rs @@ -2,11 +2,11 @@ Appellation: discretize Contrib: FL03 */ -use crate::core::prelude::Conjugate; -use crate::prelude::powmat; +use crate::core::prelude::{Conjugate, Power}; -use ndarray::{Array2, ScalarOperand}; +use ndarray::{Array, Array1, Array2, Axis, ScalarOperand}; use ndarray_linalg::{Inverse, Lapack, Scalar}; +use num::complex::ComplexFloat; use num::Float; pub fn discretize( @@ -14,7 +14,7 @@ pub fn discretize( b: &Array2, c: &Array2, step: T, -) -> anyhow::Result<(Array2, Array2, Array2)> +) -> anyhow::Result> where T: Lapack + Scalar + ScalarOperand, { @@ -24,59 +24,60 @@ where let be = (&eye - a * ss).inv().expect("Could not invert matrix"); let ab = be.dot(&(&eye + a * ss)); - let bb = (b * ss).dot(&b.t()); + let bb = (be * ss).dot(b); - Ok((ab, bb, c.clone())) + Ok((ab, bb, c.clone()).into()) } pub fn discretize_dplr( - lambda: &Array2, - p: &Array2, - q: &Array2, - b: &Array2, - c: &Array2, + lambda: &Array1, + p: &Array1, + q: &Array1, + b: &Array1, + c: &Array1, step: T, l: usize, -) -> anyhow::Result<(Array2, Array2, Array2)> +) -> anyhow::Result> where - T: Conjugate + Float + Lapack + Scalar + ScalarOperand, + T: ComplexFloat + Conjugate + Lapack + Scalar + ScalarOperand, { - let (n, _m) = lambda.dim(); - + let n = lambda.dim(); + // create an identity matrix; (n, n) let eye = Array2::::eye(n); + // compute the step size let ss = T::from(2).unwrap() * step.recip(); + // turn the parameters into two-dimensional matricies + let b2 = b.clone().insert_axis(Axis(1)); - let a = { - let tmp = Array2::from_diag(&lambda.diag()); - tmp - &p.dot(&q.conj().t()) - }; + let c2 = c.clone().insert_axis(Axis(1)); - let a0 = &eye * ss + &a; + let p2 = p.clone().insert_axis(Axis(1)); - let d = { - let tmp = lambda.mapv(|i| (ss - i).recip()); - Array2::from_diag(&tmp.diag()) - }; - - let qc = { - let tmp = q.conj(); - tmp.t().to_owned() - }; - let p2 = p.clone(); + let q2 = q.clone().insert_axis(Axis(1)); + // transpose the c matrix + let ct = c2.t(); + // compute the conjugate transpose of q + let qct = q2.conj().t().to_owned(); + // create a diagonal matrix D from the scaled eigenvalues: Dim(n, n) :: 1 / (step_size - value) + let d = Array::from_diag(&lambda.mapv(|i| (ss - i).recip())); + // create a diagonal matrix from the eigenvalues + let a = Array::from_diag(&lambda) - &p2.dot(&q2.conj().t()); + // compute A0 + let a0 = &eye * ss + &a; + // compute A1 let a1 = { - let tmp = qc.dot(&d.dot(&p2)).mapv(|i| (T::one() + i).recip()); - &d - &d.dot(&p2) * &tmp * &qc.dot(&d) + let tmp = qct.dot(&d.dot(&p2)).mapv(|i| (T::one() + i).recip()); + &d - &d.dot(&p2) * &tmp * &qct.dot(&d) }; - + // compute a-bar let ab = a0.dot(&a1); - let bb = a1.dot(b) * T::from(2).unwrap(); - let cb = { - let tmp = (&eye - powmat(&ab, l)).inv()?.conj(); - c.dot(&tmp) - }; - - Ok((ab, bb, cb.conj())) + // compute b-bar + let bb = a1.dot(&b2) * T::from(2).unwrap(); + // compute c-bar + let cb = ct.dot(&(&eye - ab.clone().pow(l)).inv()?.conj()).conj(); + // return the discretized system + Ok((ab, bb, cb).into()) } pub trait Discretize @@ -88,7 +89,59 @@ where fn discretize(&self, step: T) -> Self::Output; } -pub enum DiscretizeArgs {} +#[derive(Clone, Debug)] +pub struct Discrete { + pub a: Array2, + pub b: Array2, + pub c: Array2, +} + +impl Discrete { + pub fn new(a: Array2, b: Array2, c: Array2) -> Self { + Self { a, b, c } + } + + pub fn from_features(features: usize) -> Self + where + T: Float, + { + let a = Array2::::zeros((features, features)); + let b = Array2::::zeros((features, 1)); + let c = Array2::::zeros((1, features)); + Self::new(a, b, c) + } +} + +impl Discrete +where + T: Lapack + Scalar + ScalarOperand, +{ + pub fn discretize(&self, args: &Self, step: T) -> anyhow::Result { + discretize(&args.a, &args.b, &args.c, step) + } +} + +impl From<(Array2, Array2, Array2)> for Discrete { + fn from((a, b, c): (Array2, Array2, Array2)) -> Self { + Self::new(a, b, c) + } +} + +impl From> for (Array2, Array2, Array2) { + fn from(discrete: Discrete) -> Self { + (discrete.a, discrete.b, discrete.c) + } +} + +pub enum DiscretizeArgs { + DPLR { + lambda: Array1, + p: Array1, + q: Array1, + b: Array1, + c: Array1, + }, +} pub struct Discretizer { pub step: T, diff --git a/ml/s4/src/ssm/model.rs b/ml/s4/src/ssm/model.rs index 094e6cbc..90fc6aa2 100644 --- a/ml/s4/src/ssm/model.rs +++ b/ml/s4/src/ssm/model.rs @@ -4,65 +4,22 @@ */ use super::SSMConfig; use crate::neural::Forward; +use crate::ops::Discrete; use crate::params::{SSMParams::*, SSMStore}; use crate::prelude::{discretize, k_convolve}; -use ndarray::prelude::{Array1, Array2, NdFloat}; +use ndarray::prelude::{Array1, Array2, Axis, NdFloat}; use ndarray_conv::{Conv2DFftExt, PaddingMode, PaddingSize}; use ndarray_linalg::{Lapack, Scalar}; use num::Float; use rustfft::FftNum; -#[derive(Clone, Debug)] -pub struct Discrete { - pub a: Array2, - pub b: Array2, - pub c: Array2, -} - -impl Discrete -where - T: Float, -{ - pub fn new(a: Array2, b: Array2, c: Array2) -> Self { - Self { a, b, c } - } - - pub fn from_features(features: usize) -> Self - where - T: Default, - { - let a = Array2::::eye(features); - let b = Array2::::zeros((features, 1)); - let c = Array2::::zeros((features, features)); - Self { a, b, c } - } -} - -impl From<(Array2, Array2, Array2)> for Discrete -where - T: Float, -{ - fn from((a, b, c): (Array2, Array2, Array2)) -> Self { - Self { a, b, c } - } -} - -impl From> for (Array2, Array2, Array2) -where - T: Float, -{ - fn from(discrete: Discrete) -> Self { - (discrete.a, discrete.b, discrete.c) - } -} - pub struct SSM where T: Float, { cache: Array1, config: SSMConfig, - kernel: Array2, + kernel: Array1, params: SSMStore, ssm: Discrete, } @@ -78,7 +35,7 @@ where let features = config.features(); let cache = Array1::::zeros(features); - let kernel = Array2::::zeros((features, features)); + let kernel = Array1::::zeros(features); let params = SSMStore::from_features(features); Self { cache, @@ -97,11 +54,11 @@ where &mut self.config } - pub fn kernel(&self) -> &Array2 { + pub fn kernel(&self) -> &Array1 { &self.kernel } - pub fn kernel_mut(&mut self) -> &mut Array2 { + pub fn kernel_mut(&mut self) -> &mut Array1 { &mut self.kernel } @@ -144,7 +101,7 @@ where { let mode = PaddingMode::<2, T>::Const(T::zero()); let size = PaddingSize::Full; - if let Some(res) = u.conv_2d_fft(&self.kernel, size, mode) { + if let Some(res) = u.conv_2d_fft(&self.kernel.clone().insert_axis(Axis(1)), size, mode) { Ok(res) } else { Err(anyhow::anyhow!("convolution failed")) @@ -156,7 +113,7 @@ where Ok(discrete.into()) } - pub fn gen_filter(&self) -> Array2 { + pub fn gen_filter(&self) -> Array1 { k_convolve( &self.params[A], &self.params[B], diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index 8acfd97a..42f12012 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -3,6 +3,7 @@ Contrib: FL03 */ use crate::core::prelude::{AsComplex, Conjugate}; +use ndarray::linalg::Dot; use ndarray::prelude::*; use ndarray::{IntoDimension, ScalarOperand}; use ndarray_linalg::Scalar; @@ -10,9 +11,9 @@ use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, StandardNormal, Uniform}; use ndarray_rand::RandomExt; use num::complex::{Complex, ComplexFloat}; -use num::traits::float::{Float, FloatConst}; -use num::{Num, Signed}; +use num::traits::{Float, FloatConst, FromPrimitive, Num, Signed}; use rustfft::{FftNum, FftPlanner}; +use std::ops::Neg; pub fn stdnorm(shape: impl IntoDimension) -> Array where @@ -38,10 +39,11 @@ where res } -pub fn cauchy(v: &Array, omega: &Array, lambda: &Array) -> Array +pub fn cauchy(v: &Array, omega: &Array, lambda: &Array) -> Array where - D: Dimension, - T: Clone + Num + ScalarOperand + Signed, + A: Dimension, + B: Dimension, + T: Num + Neg + ScalarOperand, { let cdot = |b: T| (v / (lambda * T::one().neg() + b)).sum(); omega.mapv(cdot) @@ -86,7 +88,22 @@ where pub fn powmat(a: &Array2, n: usize) -> Array2 where - T: Float + 'static, + T: Clone + 'static, + Array2: Dot, Output = Array2>, +{ + if !a.is_square() { + panic!("Matrix must be square"); + } + let mut res = a.clone(); + for _ in 1..n { + res = res.dot(a); + } + res +} + +pub fn powmatc(a: &Array2, n: usize) -> Array2 +where + T: ComplexFloat + 'static, { if !a.is_square() { panic!("Matrix must be square"); @@ -156,10 +173,10 @@ pub fn kernel_dplr( c: &Array2, step: T, l: usize, -) -> Array1::Real>> +) -> Array1<::Real> where T: AsComplex + ComplexFloat + Conjugate + FloatConst + Scalar + ScalarOperand, - ::Real: NdFloat + Num + Signed + num::FromPrimitive + num::Zero, + ::Real: NdFloat + FromPrimitive + Signed, ::Complex: ComplexFloat, { let omega_l = { @@ -182,7 +199,7 @@ where let k10 = cauchy_complex(&(&aterm.1 * bterm.0), &g, lambda); let k11 = cauchy_complex(&(&aterm.1 * bterm.1), &g, lambda); - let at_roots = &c * (&k00 - k01 * (&k11 + T::one()).mapv(|i| T::one() / i) * &k10); + let at_roots = &c * (&k00 - k01 * (&k11 + T::one()).mapv(::recip) * &k10); let mut fft_planner = FftPlanner::new(); let fft = fft_planner.plan_fft_inverse(l); @@ -190,5 +207,5 @@ where .mapv(|i| Complex::new(i.re(), i.im())) .into_raw_vec(); fft.process(buffer.as_mut_slice()); - Array::from_vec(buffer) + Array::from_iter(buffer.into_iter().map(|i| i.re())) } diff --git a/ml/s4/tests/dplr.rs b/ml/s4/tests/dplr.rs new file mode 100644 index 00000000..77b73092 --- /dev/null +++ b/ml/s4/tests/dplr.rs @@ -0,0 +1,82 @@ +#[cfg(test)] +extern crate concision_core; +extern crate concision_s4; + +use concision_core as core; +use concision_s4 as s4; + +use ndarray::prelude::*; +use ndarray_linalg::flatten; +use num::complex::ComplexFloat; + +use core::prelude::{AsComplex, Conjugate, GenerateRandom, Power}; +use s4::cmp::{kernel_dplr, DPLRParams}; +use s4::hippo::dplr::DPLR; +use s4::ops::{discretize, k_convolve}; +use s4::prelude::randcomplex; + +#[test] +fn test_gen_dplr() { + let (features, samples) = (4, 16); + + let eye = Array2::::eye(features); + + let step = (samples as f64).recip(); + + let dplr = DPLR::::new(features); + + let b2 = dplr.b.clone().insert_axis(Axis(1)); + + let p2 = dplr.p.clone().insert_axis(Axis(1)); + + let a = Array::from_diag(&dplr.lambda) - p2.dot(&p2.conj().t()); + + let c = randcomplex::(features); + let c2 = c.clone().insert_axis(Axis(0)); + + let discrete = { + let tmp = discretize(&a, &b2, &c2, step.as_re()); + assert!(tmp.is_ok(), "discretize failed: {:?}", tmp.err().unwrap()); + tmp.unwrap() + }; + + let (ab, bb, cb) = discrete.into(); + // + let ak = k_convolve(&ab, &bb, &cb.conj(), samples); + println!("Ak: {:?}", ak.shape()); + // + let cc = (&eye - ab.pow(samples)).conj().t().dot(&flatten(cb)); + // + let params = DPLRParams::new( + dplr.lambda.clone(), + dplr.p.clone(), + dplr.p.clone(), + dplr.b.clone(), + cc, + ); + // + let kernal = kernel_dplr::(¶ms, step, samples); + println!("Kernal: {:?}", kernal.shape()); + + let a_real = ak.mapv(|i| i.re()); + let err = (&a_real - &kernal).mapv(|i| i.abs()); + assert!( + err.mean().unwrap() <= 1e-4, + "Error: {:?}\nTolerance: {:?}", + err.mean().unwrap(), + 1e-4 + ); +} + +#[test] +fn test_discretize_dplr() { + let (features, samples) = (8, 16); + + let step = (samples as f64).recip(); + + let dplr = DPLR::::new(features); + + let c = Array1::::stdnorm(features); + + // let kernal = kernel_dplr(lambda, p, q, b, c, step, l) +} From 71a23b443e8086e9dd77440bf4f319f4e6ac6b71 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 6 Jan 2024 12:27:06 -0600 Subject: [PATCH 108/118] update Signed-off-by: FL03 --- concision/examples/basic.rs | 14 ++- core/src/ops/fft/fft.rs | 30 +++++++ core/src/ops/fft/mod.rs | 173 ++++++++++++++++++++++++++++++++++++ core/src/ops/fft/modes.rs | 40 +++++++++ core/src/ops/fft/plan.rs | 85 ++++++++++++++++++ core/src/ops/mod.rs | 2 + core/src/specs/arrays.rs | 105 ++++++++++++++++++---- core/src/specs/init.rs | 4 + core/src/specs/math.rs | 12 +-- core/src/specs/mod.rs | 6 ++ core/src/specs/numerical.rs | 61 ++++++------- core/src/utils.rs | 8 +- ml/s4/Cargo.toml | 1 + ml/s4/examples/sand.rs | 34 +++++-- ml/s4/src/cmp/kernel.rs | 87 ++++++++++-------- ml/s4/src/cmp/mod.rs | 4 +- ml/s4/src/ops/convolve.rs | 30 ++++++- ml/s4/src/ops/discretize.rs | 1 + ml/s4/src/params/store.rs | 8 +- ml/s4/src/utils.rs | 168 +++++++--------------------------- ml/s4/tests/dplr.rs | 58 +++++++----- 21 files changed, 656 insertions(+), 275 deletions(-) create mode 100644 core/src/ops/fft/fft.rs create mode 100644 core/src/ops/fft/mod.rs create mode 100644 core/src/ops/fft/modes.rs create mode 100644 core/src/ops/fft/plan.rs diff --git a/concision/examples/basic.rs b/concision/examples/basic.rs index 76f97644..9acb2fc9 100644 --- a/concision/examples/basic.rs +++ b/concision/examples/basic.rs @@ -1,9 +1,21 @@ extern crate concision; -use concision::prelude::BoxResult; +use concision as cnc; + +use cnc::core::ops::fft::*; +use cnc::prelude::{Arange, AsComplex, BoxResult}; + +use ndarray::prelude::*; fn main() -> BoxResult { println!("Welcome to concision!"); + let samples = 8; + let arr = Array1::::arange(samples).mapv(AsComplex::as_re); + let buff = arr.clone().into_raw_vec(); + let plan = FftPlan::new(samples); + println!("Permutations: {:?}", plan.plan()); + let res = ifft(buff.as_slice(), &plan); + println!("{:?}", &res); Ok(()) } diff --git a/core/src/ops/fft/fft.rs b/core/src/ops/fft/fft.rs new file mode 100644 index 00000000..a2ec35e8 --- /dev/null +++ b/core/src/ops/fft/fft.rs @@ -0,0 +1,30 @@ +/* + Appellation: fft + Contrib: FL03 +*/ +use super::{FftDirection, FftPlan}; +// use crate::prelude::AsComplex; +// use num::complex::{Complex, ComplexFloat}; +// use num::traits::{Float, FloatConst, NumAssignOps, NumOps}; +// use num::traits::real::Real; +// use std::ops::Neg; + + +pub struct Fft { + direction: FftDirection, + plan: FftPlan, +} + +impl Fft { + pub fn new(direction: FftDirection, plan: FftPlan) -> Self { + Self { direction, plan } + } + + pub fn direction(&self) -> FftDirection { + self.direction + } + + pub fn plan(&self) -> &FftPlan { + &self.plan + } +} \ No newline at end of file diff --git a/core/src/ops/fft/mod.rs b/core/src/ops/fft/mod.rs new file mode 100644 index 00000000..4f0214d4 --- /dev/null +++ b/core/src/ops/fft/mod.rs @@ -0,0 +1,173 @@ +/* + Appellation: fft + Contrib: FL03 +*/ +//! # Fast Fourier Transform +//! +//! +pub use self::{fft::*, modes::*, plan::*, utils::*}; + +pub(crate) mod fft; +pub(crate) mod modes; +pub(crate) mod plan; + +pub(crate) mod utils { + use super::FftPlan; + use crate::prelude::AsComplex; + use num::complex::{Complex, ComplexFloat}; + use num::traits::{Float, FloatConst, NumAssignOps, NumOps}; + + pub(crate) fn fast_fourier_transform_input_permutation(length: usize) -> Vec { + let mut result = Vec::new(); + result.reserve_exact(length); + for i in 0..length { + result.push(i); + } + let mut reverse = 0_usize; + let mut position = 1_usize; + while position < length { + let mut bit = length >> 1; + while bit & reverse != 0 { + reverse ^= bit; + bit >>= 1; + } + reverse ^= bit; + // This is equivalent to adding 1 to a reversed number + if position < reverse { + // Only swap each element once + result.swap(position, reverse); + } + position += 1; + } + result + } + + pub fn fft(input: impl AsRef<[T]>, input_permutation: impl AsRef<[usize]>) -> Vec> + where + T: AsComplex + Float + FloatConst + NumOps + NumOps, Complex> + NumAssignOps, + { + let input = input.as_ref(); + + let n = input.len(); + + let mut result = Vec::new(); + result.reserve_exact(n); + for position in input_permutation.as_ref() { + result.push(input[*position].as_re()); + } + let mut segment_length = 1_usize; + while segment_length < n { + segment_length <<= 1; + let angle = T::TAU() / T::from(segment_length).unwrap(); + let w_len = Complex::new(angle.cos(), angle.sin()); + for segment_start in (0..n).step_by(segment_length) { + let mut w = Complex::new(T::one(), T::zero()); + for position in segment_start..(segment_start + segment_length / 2) { + let a = result[position]; + let b = result[position + segment_length / 2] * w; + result[position] = a + b; + result[position + segment_length / 2] = a - b; + w *= w_len; + } + } + } + result + } + + pub fn ifft(input: &[S], input_permutation: &FftPlan) -> Vec + where + S: ComplexFloat + NumOps + NumOps + NumOps>, + T: Float + FloatConst + NumOps + NumOps, + { + let n = input.len(); + let mut result = Vec::new(); + result.reserve_exact(n); + for position in input_permutation.clone().into_iter() { + result.push(input[position]); + } + let mut segment_length = 1_usize; + while segment_length < n { + segment_length <<= 1; + let angle = T::TAU().neg() / T::from(segment_length).unwrap(); + let w_len = Complex::new(ComplexFloat::cos(angle), ComplexFloat::sin(angle)); + for segment_start in (0..n).step_by(segment_length) { + let mut w = S::one(); + for position in segment_start..(segment_start + segment_length / 2) { + let a = result[position]; + let b = result[position + segment_length / 2] * w; + result[position] = a + b; + result[position + segment_length / 2] = a - b; + w = w * w_len; + } + } + } + let scale = T::from(n).unwrap().recip(); + result.iter().map(|x| x.re() * scale).collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use num::Signed; + + fn almost_equal(a: T, b: T, epsilon: T) -> bool + where + T: PartialOrd + Signed, + { + (a - b).abs() < epsilon + } + + const EPSILON: f64 = 1e-6; + + #[test] + fn test_plan() { + let samples = 16; + + let plan = FftPlan::new(samples); + assert_eq!(plan.plan(), fast_fourier_transform_input_permutation(16).as_slice()); + } + + #[test] + fn small_polynomial_returns_self() { + let polynomial = vec![1.0f64, 1.0, 0.0, 2.5]; + let permutation = FftPlan::new(polynomial.len()); + let fft = fft(&polynomial, &permutation); + let ifft = ifft(&fft, &permutation); + for (x, y) in ifft.iter().zip(polynomial.iter()) { + assert!(almost_equal(*x, *y, EPSILON)); + } + } + + #[test] + fn square_small_polynomial() { + let mut polynomial = vec![1.0f64, 1.0, 0.0, 2.0]; + polynomial.append(&mut vec![0.0; 4]); + let permutation = FftPlan::new(polynomial.len()); + let mut fft = fft(&polynomial, &permutation); + fft.iter_mut().for_each(|num| *num *= *num); + let ifft = ifft(&fft, &permutation); + let expected = [1.0, 2.0, 1.0, 4.0, 4.0, 0.0, 4.0, 0.0, 0.0]; + for (x, y) in ifft.iter().zip(expected.iter()) { + assert!(almost_equal(*x, *y, EPSILON)); + } + } + + #[test] + #[ignore] + fn square_big_polynomial() { + // This test case takes ~1050ms on my machine in unoptimized mode, + // but it takes ~70ms in release mode. + let n = 1 << 17; // ~100_000 + let mut polynomial = vec![1.0f64; n]; + polynomial.append(&mut vec![0.0f64; n]); + let permutation = FftPlan::new(polynomial.len()); + let mut fft = fft(&polynomial, &permutation); + fft.iter_mut().for_each(|num| *num *= *num); + let ifft = ifft(&fft, &permutation); + let expected = (0..((n << 1) - 1)).map(|i| std::cmp::min(i + 1, (n << 1) - 1 - i) as f64); + for (&x, y) in ifft.iter().zip(expected) { + assert!(almost_equal(x, y, EPSILON)); + } + } +} diff --git a/core/src/ops/fft/modes.rs b/core/src/ops/fft/modes.rs new file mode 100644 index 00000000..a7410a46 --- /dev/null +++ b/core/src/ops/fft/modes.rs @@ -0,0 +1,40 @@ +/* + Appellation: modes + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; + +#[derive(Clone, Copy, Debug, Default, Deserialize, Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum FftDirection { + #[default] + Forward = 0, + Inverse = 1, +} + +impl FftDirection { + pub fn forward() -> Self { + Self::Forward + } + + pub fn inverse() -> Self { + Self::Inverse + } +} + +impl From for FftDirection { + fn from(direction: usize) -> Self { + match direction % Self::COUNT { + 0 => Self::Forward, + _ => Self::Inverse, + } + } +} +impl From for usize { + fn from(direction: FftDirection) -> Self { + direction as usize + } +} \ No newline at end of file diff --git a/core/src/ops/fft/plan.rs b/core/src/ops/fft/plan.rs new file mode 100644 index 00000000..277899d9 --- /dev/null +++ b/core/src/ops/fft/plan.rs @@ -0,0 +1,85 @@ +/* + Appellation: plan + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct FftPlan { + plan: Vec, +} + +impl FftPlan { + pub fn new(n: usize) -> Self { + + let mut permute = Vec::new(); + permute.reserve_exact(n); + permute.extend(0..n); + + let mut reverse = 0; + let mut position = 1; + while position < n { + let mut bit = n >> 1; + while bit & reverse != 0 { + reverse ^= bit; + bit >>= 1; + } + reverse ^= bit; + // This is equivalent to adding 1 to a reversed number + if position < reverse { + // Only swap each element once + permute.swap(position, reverse); + } + position += 1; + } + Self { plan: permute } + } + + pub fn plan(&self) -> &[usize] { + &self.plan + } +} + +impl AsRef<[usize]> for FftPlan { + fn as_ref(&self) -> &[usize] { + &self.plan + } +} + +impl AsMut<[usize]> for FftPlan { + fn as_mut(&mut self) -> &mut [usize] { + &mut self.plan + } +} + +impl Extend for FftPlan { + fn extend>(&mut self, iter: T) { + self.plan.extend(iter); + } +} + +impl FromIterator for FftPlan { + fn from_iter>(iter: T) -> Self { + Self { + plan: Vec::from_iter(iter), + } + } +} + +impl IntoIterator for FftPlan { + type Item = usize; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.plan.into_iter() + } +} + +impl<'a> IntoIterator for &'a mut FftPlan { + type Item = &'a mut usize; + type IntoIter = std::slice::IterMut<'a, usize>; + + fn into_iter(self) -> Self::IntoIter { + self.plan.iter_mut() + } +} diff --git a/core/src/ops/mod.rs b/core/src/ops/mod.rs index 56fc4fee..545a0a33 100644 --- a/core/src/ops/mod.rs +++ b/core/src/ops/mod.rs @@ -7,6 +7,8 @@ pub use self::kinds::*; pub(crate) mod kinds; +pub mod fft; + pub trait Operation { type Output; diff --git a/core/src/specs/arrays.rs b/core/src/specs/arrays.rs index 14003d83..bd34df9c 100644 --- a/core/src/specs/arrays.rs +++ b/core/src/specs/arrays.rs @@ -10,7 +10,8 @@ use distr::{Bernoulli, BernoulliError, Distribution, StandardNormal, Uniform}; use ndarray_rand::rand_distr as distr; use ndarray_rand::RandomExt; -use num::{Float, Num}; +use num::{Float, Num, ToPrimitive}; +use num::traits::real::Real; use std::ops; pub trait Affine: Sized { @@ -38,6 +39,67 @@ pub enum ArangeArgs { Until { stop: T }, } +impl ArangeArgs where T: Copy + Num { + /// Returns the start value of the range. + pub fn start(&self) -> T { + match self { + ArangeArgs::Arange { start, .. } => *start, + ArangeArgs::Between { start, .. } => *start, + ArangeArgs::Until { .. } => T::zero(), + } + } + /// Returns the stop value of the range. + pub fn stop(&self) -> T { + match self { + ArangeArgs::Arange { stop, .. } => *stop, + ArangeArgs::Between { stop, .. } => *stop, + ArangeArgs::Until { stop } => *stop, + } + } + /// Returns the step value of the range. + pub fn step(&self) -> T { + match self { + ArangeArgs::Arange { step, .. } => *step, + ArangeArgs::Between { .. } => T::one(), + ArangeArgs::Until { .. } => T::one(), + } + } + /// Returns the number of steps between the given boundaries + pub fn steps(&self) -> usize where T: Real { + match self { + ArangeArgs::Arange { start, stop, step } => { + let n = ((*stop - *start) / *step).ceil().to_usize().unwrap(); + n + } + ArangeArgs::Between { start, stop } => { + let n = (*stop - *start).to_usize().unwrap(); + n + } + ArangeArgs::Until { stop } => { + let n = stop.to_usize().unwrap(); + n + } + } + } +} + +impl From> for ArangeArgs { + fn from(args: ops::Range) -> Self { + ArangeArgs::Between { + start: args.start, + stop: args.end, + } + } +} + + + +impl From> for ArangeArgs { + fn from(args: ops::RangeFrom) -> Self { + ArangeArgs::Until { stop: args.start } + } +} + impl From<(T, T, T)> for ArangeArgs { fn from(args: (T, T, T)) -> Self { ArangeArgs::Arange { @@ -61,42 +123,55 @@ impl From for ArangeArgs where T: Num, { - fn from(args: T) -> Self { - ArangeArgs::Until { stop: args } + fn from(stop: T) -> Self { + ArangeArgs::Until { stop } } } pub trait Arange { - fn arange(start: T, stop: T, step: T) -> Self; + fn arange(args: impl Into>) -> Self; } impl Arange for Vec where T: Float, { - fn arange(start: T, stop: T, step: T) -> Self { - let n = ((stop - start) / step).ceil().to_usize().unwrap(); - (0..n).map(|i| start + step * T::from(i).unwrap()).collect() + fn arange(args: impl Into>) -> Self { + let args = args.into(); + let n: usize = args.stop().to_usize().expect("Failed to convert 'stop' to a usize"); + (0..n).map(|i| args.start() + args.step() * T::from(i).unwrap()).collect() } } -impl Arange for Array +impl Arange for Array where + S: Copy + Num + ToPrimitive, T: Float, { - fn arange(start: T, stop: T, step: T) -> Self { - let n = ((stop - start) / step).ceil().to_usize().unwrap(); - Array::from_shape_fn(n, |i| start + step * T::from(i).unwrap()) + fn arange(args: impl Into>) -> Self { + let args = args.into(); + let n: usize = args.stop().to_usize().expect("Failed to convert 'stop' to a usize"); + let start = T::from(args.start()).unwrap(); + let step = T::from(args.step()).unwrap(); + + Array::from_iter((0..n).map(|i| start + step * T::from(i).unwrap())) } } -impl Arange for Array +impl Arange for Array where + S: Copy + Num + ToPrimitive, T: Float, { - fn arange(start: T, stop: T, step: T) -> Self { - let n = ((stop - start) / step).ceil().to_usize().unwrap(); - Array::from_shape_fn((n, 1), |(i, ..)| start + step * T::from(i).unwrap()) + fn arange(args: impl Into>) -> Self { + let args = args.into(); + let start = T::from(args.start()).unwrap(); + let step = T::from(args.step()).unwrap(); + let n: usize = args.stop().to_usize().expect("Failed to convert 'stop' to a usize"); + let f = | (i, _j) | { + start + step * T::from(i).unwrap() + }; + Array::from_shape_fn((n, 1), f) } } diff --git a/core/src/specs/init.rs b/core/src/specs/init.rs index 6fc1ee0b..1dece2e6 100644 --- a/core/src/specs/init.rs +++ b/core/src/specs/init.rs @@ -7,6 +7,10 @@ pub trait Init { fn init(&mut self) -> Self; } +pub trait InitRandom { + fn genrand(&mut self) -> T; +} + pub trait Rand {} pub trait RandComplex {} diff --git a/core/src/specs/math.rs b/core/src/specs/math.rs index 8d4eb823..4ad2bdb1 100644 --- a/core/src/specs/math.rs +++ b/core/src/specs/math.rs @@ -5,7 +5,7 @@ use ndarray::linalg::Dot; use ndarray::prelude::{Array, Dimension, Ix2}; use num::complex::Complex; -use num::{Float, Integer, Signed}; +use num::{Float, Num, Signed}; use std::ops; pub trait Conjugate { @@ -146,15 +146,17 @@ pub trait Power { // } // } -impl Power for Array +impl Power for Array where - D: Dimension, - T: Clone, - Array: Dot, + T: Clone + Num, + Array: Dot, { type Output = Self; fn pow(&self, rhs: usize) -> Self::Output { + if rhs == 0 { + return Array::eye(self.shape()[0]); + } let mut res = self.clone(); for _ in 1..rhs { res = res.dot(&self); diff --git a/core/src/specs/mod.rs b/core/src/specs/mod.rs index fe3bd7da..fe3482c6 100644 --- a/core/src/specs/mod.rs +++ b/core/src/specs/mod.rs @@ -62,6 +62,12 @@ mod tests { use super::*; use ndarray::prelude::*; + #[test] + fn test_arange() { + let exp = array![0.0, 1.0, 2.0, 3.0, 4.0]; + assert_eq!(&exp, &Array1::::arange(5)) + } + #[test] fn test_as_complex() { let x = 1.0; diff --git a/core/src/specs/numerical.rs b/core/src/specs/numerical.rs index 053cb841..d4174e51 100644 --- a/core/src/specs/numerical.rs +++ b/core/src/specs/numerical.rs @@ -3,65 +3,56 @@ Contrib: FL03 */ use num::complex::Complex; -use num::traits::{NumAssignOps, NumOps, Signed}; -use std::ops::{Add, Div, Mul, Sub}; +use num::traits::{Num, NumAssignOps, NumOps, Signed}; +// use num::traits::real::Real; -pub trait Algebraic -where - Self: Add + Div + Mul + Sub + Sized, -{ - type Output; +pub trait Algebraic: NumOps + Sized { } -pub trait AlgebraicExt +pub trait AlgebraicExt where - Self: Algebraic + NumAssignOps, + Self: Algebraic + NumAssignOps + Sized, { } -impl Algebraic for A +impl Algebraic for A where - A: Add + Div + Mul + Sub, + A: NumOps + Sized, { - type Output = C; } -impl AlgebraicExt for A where A: Algebraic + NumAssignOps {} - -pub trait ComplexNum -where - Self: Algebraic + Algebraic, -{ - type DType; +pub trait ComplexNum: Sized { + type Real: Algebraic + Algebraic; fn complex(self) -> Self; - fn im(self) -> Self::DType; + fn im(self) -> Self::Real; - fn re(self) -> Self::DType; + fn re(self) -> Self::Real; } -pub trait Imaginary: - Algebraic + Algebraic +pub trait Imaginary: Sized +where + T: Algebraic + Algebraic, { - type DType; + type Complex: Algebraic + Algebraic; - fn im(self) -> Self::DType; + fn im(self) -> T; - fn re(self) -> Self::DType; + fn re(self) -> T; } -impl Imaginary for num::Complex +impl Imaginary for Complex where - T: Clone + num::Num, + T: Algebraic + Algebraic, Complex> + Clone + Num, { - type DType = T; + type Complex = Complex; - fn im(self) -> Self::DType { + fn im(self) -> T { self.im } - fn re(self) -> Self::DType { + fn re(self) -> T { self.re } } @@ -96,7 +87,7 @@ impl Number for f32 {} impl Number for f64 {} -impl Number for S where S: ComplexNum {} +impl Number for S where S: ComplexNum {} pub trait Abs { fn abs(&self) -> Self; @@ -111,12 +102,12 @@ where } } pub trait Scalar { - type Complex: NumOps + NumOps; + type Complex: NumOps + NumOps; type Real: NumOps + NumOps; } -pub trait Numerical: Algebraic { - type Elem: Algebraic + Number; +pub trait Numerical: Sized { + type Elem: Algebraic + Number; fn abs(&self) -> Self where diff --git a/core/src/utils.rs b/core/src/utils.rs index 580f88bb..2c7e456a 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -5,7 +5,10 @@ use ndarray::prelude::*; use ndarray::{concatenate, IntoDimension, RemoveAxis, ShapeError}; +// use ndarray_rand::RandomExt; +// use ndarray_rand::rand_distr::Distribution; use num::cast::AsPrimitive; +// use num::complex::{Complex, ComplexDistribution}; use num::{Float, Num, NumCast, Zero}; pub fn arange(a: T, b: T, h: T) -> Array1 @@ -110,6 +113,7 @@ where Array::linspace(T::zero(), T::from(n - 1).unwrap(), n).into_shape(dim) } + /// creates a matrix from the given shape filled with numerical elements [0, n) spaced evenly by 1 pub fn rangespace(dim: impl IntoDimension) -> Array where @@ -120,12 +124,12 @@ where let iter = (0..dim.size()).map(|i| T::from(i).unwrap()); Array::from_shape_vec(dim, iter.collect()).unwrap() } - +/// Round the given value to the given number of decimal places. pub fn round_to(val: T, decimals: usize) -> T { let factor = T::from(10).expect("").powi(decimals as i32); (val * factor).round() / factor } - +/// Creates a larger array from an iterator of smaller arrays. pub fn stack_iter(iter: impl IntoIterator>) -> Array2 where T: Clone + Num, diff --git a/ml/s4/Cargo.toml b/ml/s4/Cargo.toml index 5941514a..5b47742e 100644 --- a/ml/s4/Cargo.toml +++ b/ml/s4/Cargo.toml @@ -74,6 +74,7 @@ ndarray-linalg = { features = [], version = "0.16" } ndarray-rand.workspace = true ndarray-stats.workspace = true num.workspace = true +rand = "0.8" rustfft = { features = [], version = "6" } serde.workspace = true serde_json.workspace = true diff --git a/ml/s4/examples/sand.rs b/ml/s4/examples/sand.rs index 3e88cb1a..58f66d3c 100644 --- a/ml/s4/examples/sand.rs +++ b/ml/s4/examples/sand.rs @@ -1,20 +1,36 @@ // use concision_core as cnc; extern crate concision_s4; +use concision_core as core; use concision_s4 as s4; -use s4::randcomplex; - -use ndarray::prelude::Ix2; -use ndarray_linalg::Scalar; -use num::complex::Complex; +use core::prelude::{Arange, AsComplex}; +use s4::prelude::cauchy; +use s4::ssm::{SSM, SSMConfig}; +use ndarray::prelude::*; +use rustfft::FftPlanner; fn main() -> anyhow::Result<()> { - let i = Complex::::i(); - println!("{:?}", i.add_real(1.0)); - let c = randcomplex::([2, 2]); + let (features, samples) = (4, 16); + + let arr = Array::::arange(samples).mapv(AsComplex::as_re); + let mut planner = FftPlanner::::new(); + let fft = planner.plan_fft_inverse(samples); + let mut buffer = arr.to_vec(); + println!("Buffer: {:?}", &buffer); + fft.process(buffer.as_mut_slice()); + + let res = Array::from_vec(buffer); + + println!("FFT:\n\n{:?}\n", res); + + let config = SSMConfig::new(true, features, samples); + let _ssm = SSM::::create(config); - println!("{:?}", &c); + let a = Array1::::arange(features * features).into_shape((features, features))? + 1.0; + let b = Array1::::arange(features).insert_axis(Axis(1)); + let c = Array1::::arange(features).insert_axis(Axis(0)) * 2.0; + println!("Cauchy:\n\n{:?}", cauchy(&a, &b, &c)); Ok(()) } diff --git a/ml/s4/src/cmp/kernel.rs b/ml/s4/src/cmp/kernel.rs index 360a9e12..5c9d2b04 100644 --- a/ml/s4/src/cmp/kernel.rs +++ b/ml/s4/src/cmp/kernel.rs @@ -2,13 +2,14 @@ Appellation: kernel Contrib: FL03 */ -use crate::core::prelude::{AsComplex, Conjugate}; -use crate::prelude::{cauchy, cauchy_complex}; -use ndarray::prelude::{Array, Array1, Array2, Ix1, NdFloat}; +use crate::core::ops::fft::*; +use crate::core::prelude::Conjugate; +use crate::prelude::cauchy; +use ndarray::prelude::{Array, Array1, Array2,}; use ndarray::ScalarOperand; use ndarray_linalg::Scalar; use num::complex::{Complex, ComplexFloat}; -use num::traits::{Float, FloatConst, FromPrimitive, NumOps, Signed, Zero}; +use num::traits::{Float, FloatConst, NumOps,}; use rustfft::{FftNum, FftPlanner}; use std::ops::Neg; @@ -45,13 +46,7 @@ impl DPLRParams { impl DPLRParams> { pub fn kernel_s(&self, step: f64, l: usize) -> Array1 { - let lt = l as f64; // T::from(l).unwrap(); - let omega_l = { - let f = |i: usize| -> Complex { - (Complex::i().neg() * (i as f64) * f64::PI() / lt).exp() - }; - Array::from_iter((0..l).map(f)) - }; + let omega_l = omega_l::(l); let aterm = (self.c.conj(), self.q.conj()); let bterm = (self.b.clone(), self.p.clone()); @@ -59,10 +54,10 @@ impl DPLRParams> { let g = ((&omega_l.clone().neg() + 1.0) / (&omega_l + 1.0)) * (2.0 * step.recip()); let c = omega_l.mapv(|i| 2.0 / (1.0 + i)); - let k00 = cauchy_complex(&(&aterm.0 * &bterm.0), &g, &self.lambda); - let k01 = cauchy_complex(&(&aterm.0 * &bterm.1), &g, &self.lambda); - let k10 = cauchy_complex(&(&aterm.1 * &bterm.0), &g, &self.lambda); - let k11 = cauchy_complex(&(&aterm.1 * &bterm.1), &g, &self.lambda); + let k00 = cauchy(&(&aterm.0 * &bterm.0), &g, &self.lambda); + let k01 = cauchy(&(&aterm.0 * &bterm.1), &g, &self.lambda); + let k10 = cauchy(&(&aterm.1 * &bterm.0), &g, &self.lambda); + let k11 = cauchy(&(&aterm.1 * &bterm.1), &g, &self.lambda); let at_roots = &c * (&k00 - k01 * (&k11 + 1.0).mapv(ComplexFloat::recip) * &k10); @@ -75,6 +70,25 @@ impl DPLRParams> { } } +pub fn omega_l(l: usize) -> Array1<::Complex> +where + T: Scalar>, + ::Real: Float + FloatConst + NumOps<::Real, ::Real>, + ::Complex: ScalarOperand, +{ + let lt = ::Real::from(l).unwrap(); + let f = |i: usize| -> ::Complex { + let im = ::Real::PI() + .mul_complex(Complex::new(T::zero(), T::from(2).unwrap().neg())); + ::Real::from(i) + .unwrap() + .div_real(lt) + .mul_complex(im) + .exp() + }; + Array::from_iter((0..l).map(f)) +} + pub fn kernel_dplr( dplr: &DPLRParams<::Complex>, step: ::Real, @@ -86,37 +100,32 @@ where FloatConst + NumOps<::Complex, ::Complex> + ScalarOperand, ::Complex: ScalarOperand, { - let lt = ::Real::from(l).unwrap(); + let one = ::Real::one(); let two = ::Real::from(2).unwrap(); - let omega_l: Array1<::Complex> = { - let f = |i: usize| -> ::Complex { - ((::Real::from(i).unwrap() * ::Real::PI()) / lt) - .mul_complex(Complex::i().neg()) - .exp() - }; - Array::from_iter((0..l).map(f)) - }; - + // get the lambda matrix + let lambda = dplr.lambda.clone(); + // generate omega + let omega_l: Array1<::Complex> = omega_l::(l); + // collect the relevant terms for A let aterm = (dplr.c.conj(), dplr.q.conj()); + // collect the relevant terms for B let bterm = (dplr.b.clone(), dplr.p.clone()); - let g = ((&omega_l.clone().neg() + ::Real::one()) / (&omega_l + T::one())) - * (T::one() * step.recip()); - let c = omega_l.mapv(|i| two.div_complex(::Real::one().add_complex(i))); + let g = omega_l.mapv(|i| (one - i) / (one + i)) * (two / step); + let c = omega_l.mapv(|i| two.div_complex(one.add_complex(i))); - let k00: Array1<::Complex> = cauchy(&(&aterm.0 * &bterm.0), &g, &dplr.lambda); - let k01: Array1<::Complex> = cauchy(&(&aterm.0 * &bterm.1), &g, &dplr.lambda); - let k10: Array1<::Complex> = cauchy(&(&aterm.1 * &bterm.0), &g, &dplr.lambda); - let k11: Array1<::Complex> = cauchy(&(&aterm.1 * &bterm.1), &g, &dplr.lambda); + let k00: Array1<::Complex> = cauchy(&(&aterm.0 * &bterm.0), &g, &lambda); + let k01: Array1<::Complex> = cauchy(&(&aterm.0 * &bterm.1), &g, &lambda); + let k10: Array1<::Complex> = cauchy(&(&aterm.1 * &bterm.0), &g, &lambda); + let k11: Array1<::Complex> = cauchy(&(&aterm.1 * &bterm.1), &g, &lambda); - let at_roots = &c * (&k00 - k01 * (&k11 + T::one()).mapv(ComplexFloat::recip) * &k10); + let at_roots = &c * (&k00 - k01 * &k11.mapv(|i| one / (i + one)) * &k10); - let mut fft_planner = FftPlanner::new(); - let fft = fft_planner.plan_fft_inverse(l); - // create a buffer to hold the complex numbers - let mut buffer = at_roots.into_raw_vec(); - fft.process(buffer.as_mut_slice()); - Array::from_iter(buffer.into_iter().map(|i| i.re())) + let buffer = at_roots.into_raw_vec(); + println!("Roots:\n\n{:?}\n", &buffer); + let permute = FftPlan::new(l); + let res = ifft(buffer.as_slice(), &permute); + Array::from_vec(res) } pub struct Kernel { diff --git a/ml/s4/src/cmp/mod.rs b/ml/s4/src/cmp/mod.rs index 4f09370f..f27d829a 100644 --- a/ml/s4/src/cmp/mod.rs +++ b/ml/s4/src/cmp/mod.rs @@ -5,7 +5,7 @@ //! # Components //! //! -pub use self::{cache::*, kernel::*}; +pub use self::cache::*; pub(crate) mod cache; -pub(crate) mod kernel; +pub mod kernel; diff --git a/ml/s4/src/ops/convolve.rs b/ml/s4/src/ops/convolve.rs index 10f65d13..62fba100 100644 --- a/ml/s4/src/ops/convolve.rs +++ b/ml/s4/src/ops/convolve.rs @@ -2,12 +2,11 @@ Appellation: convolve Contrib: FL03 */ -use crate::core::prelude::Power; +use crate::core::prelude::{concat_iter, Power}; use crate::prelude::powmat; use ndarray::linalg::Dot; use ndarray::prelude::{s, Array, Array1, Array2, Axis}; use ndarray::ScalarOperand; -use ndarray_linalg::flatten; use num::Num; pub fn convolve() {} @@ -17,5 +16,30 @@ where T: Num + ScalarOperand, Array2: Dot, Output = Array2> + Dot, Output = Array1>, { - Array::from_iter((0..l).map(|i| c.dot(&a.pow(i).dot(b)).sum())) + let b = b.clone().remove_axis(Axis(1)); + let mut res = Array2::::zeros((l, a.shape()[0])); + for i in 0..l { + let out = c.dot(&a.pow(i).dot(&b)); + res.slice_mut(s![i, ..]).assign(&out); + } + res.remove_axis(Axis(1)) } + + +pub fn k_conv(a: &Array2, b: &Array2, c: &Array2, l: usize) -> Array1 +where + T: Num + ScalarOperand, + Array2: Dot, Output = Array2> + Dot, Output = Array1>, +{ + let mut store = Vec::new(); + for i in 0..l { + let tmp = c.dot(&a.pow(i).dot(b)); + store.extend(tmp); + + } + Array::from_iter(store) +} + +pub struct Filter { + +} \ No newline at end of file diff --git a/ml/s4/src/ops/discretize.rs b/ml/s4/src/ops/discretize.rs index 57ef2954..4a0fb74d 100644 --- a/ml/s4/src/ops/discretize.rs +++ b/ml/s4/src/ops/discretize.rs @@ -29,6 +29,7 @@ where Ok((ab, bb, c.clone()).into()) } + pub fn discretize_dplr( lambda: &Array1, p: &Array1, diff --git a/ml/s4/src/params/store.rs b/ml/s4/src/params/store.rs index 25d21608..8ba3e78c 100644 --- a/ml/s4/src/params/store.rs +++ b/ml/s4/src/params/store.rs @@ -196,13 +196,7 @@ where T: Float, { fn from(store: SSMStore) -> Self { - let mut map = HashMap::new(); - - map.insert(SSMParams::A, store.a); - map.insert(SSMParams::B, store.b); - map.insert(SSMParams::C, store.c); - map.insert(SSMParams::D, store.d); - map + HashMap::from_iter(store.into_iter()) } } diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index 42f12012..7fa0064e 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -2,111 +2,45 @@ Appellation: utils Contrib: FL03 */ -use crate::core::prelude::{AsComplex, Conjugate}; -use ndarray::linalg::Dot; use ndarray::prelude::*; use ndarray::{IntoDimension, ScalarOperand}; -use ndarray_linalg::Scalar; +use ndarray::linalg::Dot; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, StandardNormal, Uniform}; use ndarray_rand::RandomExt; -use num::complex::{Complex, ComplexFloat}; -use num::traits::{Float, FloatConst, FromPrimitive, Num, Signed}; -use rustfft::{FftNum, FftPlanner}; +use num::complex::{Complex, ComplexDistribution, ComplexFloat}; +use num::traits::Num; use std::ops::Neg; -pub fn stdnorm(shape: impl IntoDimension) -> Array -where - D: Dimension, - StandardNormal: Distribution, -{ - Array::random(shape, StandardNormal) -} - -pub fn randcomplex(shape: impl IntoDimension) -> Array, D> -where - D: Dimension, - T: Copy + Num, - StandardNormal: Distribution, -{ - let dim = shape.into_dimension(); - let re = Array::random(dim.clone(), StandardNormal); - let im = Array::random(dim.clone(), StandardNormal); - let mut res = Array::zeros(dim); - ndarray::azip!((re in &re, im in &im, res in &mut res) { - *res = Complex::new(*re, *im); - }); - res -} - -pub fn cauchy(v: &Array, omega: &Array, lambda: &Array) -> Array +pub fn cauchy(a: &Array, b: &Array, c: &Array) -> Array where A: Dimension, B: Dimension, T: Num + Neg + ScalarOperand, { - let cdot = |b: T| (v / (lambda * T::one().neg() + b)).sum(); - omega.mapv(cdot) + let cdot = |b: T| (a / (c * T::one().neg() + b)).sum(); + b.mapv(cdot) } -pub fn cauchy_complex( - v: &Array, - omega: &Array, - lambda: &Array, -) -> Array +pub fn logstep(a: T, b: T, shape: impl IntoDimension) -> Array where D: Dimension, - S: Dimension, - T: ComplexFloat + ScalarOperand, + T: NdFloat + SampleUniform, { - let cdot = |b: T| (v / (lambda * T::one().neg() + b)).sum(); - omega.mapv(cdot) + Array::random(shape, Uniform::new(a, b)) * (b.ln() - a.ln()) + a.ln() } -// pub fn eig_sym(args: &Array2) -> (Array1, Array2) { -// let sym = args.clone().into_nalgebra().symmetric_eigen(); -// ( -// sym.eigenvalues.into_ndarray1(), -// sym.eigenvectors.into_ndarray2(), -// ) -// } - -// pub fn eig_csym(args: &Array2>) -> (Array1, Array2>) { -// let sym = args.clone().into_nalgebra().symmetric_eigen(); -// let (eig, v) = (sym.eigenvalues, sym.eigenvectors); -// (eig.into_ndarray1(), v.into_ndarray2()) -// } - -// pub fn eigh(args: &Array2) -> (Array1, Array2) { -// let na = args.clone().into_nalgebra(); -// let sym = na.symmetric_eigen(); -// let v = sym.eigenvectors; -// let eig = sym.eigenvalues.into_ndarray1(); -// let eigval = v.into_ndarray2(); -// (eig, eigval) -// } pub fn powmat(a: &Array2, n: usize) -> Array2 where - T: Clone + 'static, + T: Clone + Num + 'static, Array2: Dot, Output = Array2>, { if !a.is_square() { panic!("Matrix must be square"); } - let mut res = a.clone(); - for _ in 1..n { - res = res.dot(a); - } - res -} - -pub fn powmatc(a: &Array2, n: usize) -> Array2 -where - T: ComplexFloat + 'static, -{ - if !a.is_square() { - panic!("Matrix must be square"); + if n == 0 { + return Array2::::eye(a.nrows()); } let mut res = a.clone(); for _ in 1..n { @@ -115,30 +49,39 @@ where res } -pub fn casual_colvolution(a: &Array2, _b: &Array2) -> Array2 +/// Generate a random array of complex numbers with real and imaginary parts in the range [0, 1) +pub fn randc(shape: impl IntoDimension) -> Array, D> where - T: FftNum, + D: Dimension, + T: Distribution + Num, + ComplexDistribution: Distribution>, { - let mut planner = FftPlanner::::new(); - let _fft = planner.plan_fft_forward(a.len()); - - a.clone() + let distr = ComplexDistribution::::new(T::one(), T::one()); + Array::random(shape, distr) } -pub fn logstep(a: T, b: T, shape: impl IntoDimension) -> Array +pub fn randcomplex(shape: impl IntoDimension) -> Array, D> where D: Dimension, - T: NdFloat + SampleUniform, + T: Copy + Num, + StandardNormal: Distribution, { - Array::random(shape, Uniform::new(a, b)) * (b.ln() - a.ln()) + a.ln() + let dim = shape.into_dimension(); + let re = Array::random(dim.clone(), StandardNormal); + let im = Array::random(dim.clone(), StandardNormal); + let mut res = Array::zeros(dim); + ndarray::azip!((re in &re, im in &im, res in &mut res) { + *res = Complex::new(*re, *im); + }); + res } -pub fn logstep_init(a: T, b: T) -> impl Fn(D) -> Array +pub fn stdnorm(shape: impl IntoDimension) -> Array where D: Dimension, - T: NdFloat + SampleUniform, + StandardNormal: Distribution, { - move |shape| logstep(a, b, shape) + Array::random(shape, StandardNormal) } pub fn scanner( @@ -164,48 +107,3 @@ where } res } - -pub fn kernel_dplr( - lambda: &Array2, - p: &Array2, - q: &Array2, - b: &Array2, - c: &Array2, - step: T, - l: usize, -) -> Array1<::Real> -where - T: AsComplex + ComplexFloat + Conjugate + FloatConst + Scalar + ScalarOperand, - ::Real: NdFloat + FromPrimitive + Signed, - ::Complex: ComplexFloat, -{ - let omega_l = { - let f = |i: usize| { - T::from(i).unwrap() - * T::from(Complex::new(T::one(), -T::PI() / T::from(l).unwrap())).unwrap() - }; - Array::from_iter((0..l).map(f)) - }; - let aterm = (c.conj(), q.conj()); - let bterm = (b, p); - - let two = T::from(2).unwrap(); - - let g = ((&omega_l * T::one().neg() + T::one()) / (&omega_l + T::one())) * (two / step); - let c = (&omega_l + T::one()).mapv(|i| two / i); - - let k00 = cauchy_complex(&(&aterm.0 * bterm.0), &g, lambda); - let k01 = cauchy_complex(&(&aterm.0 * bterm.1), &g, lambda); - let k10 = cauchy_complex(&(&aterm.1 * bterm.0), &g, lambda); - let k11 = cauchy_complex(&(&aterm.1 * bterm.1), &g, lambda); - - let at_roots = &c * (&k00 - k01 * (&k11 + T::one()).mapv(::recip) * &k10); - - let mut fft_planner = FftPlanner::new(); - let fft = fft_planner.plan_fft_inverse(l); - let mut buffer = at_roots - .mapv(|i| Complex::new(i.re(), i.im())) - .into_raw_vec(); - fft.process(buffer.as_mut_slice()); - Array::from_iter(buffer.into_iter().map(|i| i.re())) -} diff --git a/ml/s4/tests/dplr.rs b/ml/s4/tests/dplr.rs index 77b73092..58fe4408 100644 --- a/ml/s4/tests/dplr.rs +++ b/ml/s4/tests/dplr.rs @@ -6,14 +6,36 @@ use concision_core as core; use concision_s4 as s4; use ndarray::prelude::*; +use ndarray::IntoDimension; use ndarray_linalg::flatten; +use ndarray_rand::RandomExt; +use ndarray_rand::rand::{rngs::StdRng, SeedableRng}; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; +use ndarray_rand::rand_distr::uniform::{SampleUniform, Uniform}; use num::complex::ComplexFloat; use core::prelude::{AsComplex, Conjugate, GenerateRandom, Power}; -use s4::cmp::{kernel_dplr, DPLRParams}; +use s4::cmp::kernel::{kernel_dplr, DPLRParams}; use s4::hippo::dplr::DPLR; -use s4::ops::{discretize, k_convolve}; -use s4::prelude::randcomplex; +use s4::ops::{discretize, k_conv}; + +const RNGKEY: u64 = 1; + +fn seeded_uniform(key: u64, start: T, stop: T, shape: impl IntoDimension) -> Array +where + D: Dimension, + T: SampleUniform, +{ + Array::random_using(shape, Uniform::new(start, stop), &mut StdRng::seed_from_u64(key)) +} + +fn seeded_stdnorm(key: u64, shape: impl IntoDimension) -> Array +where + D: Dimension, + StandardNormal: Distribution, +{ + Array::random_using(shape, StandardNormal, &mut StdRng::seed_from_u64(key)) +} #[test] fn test_gen_dplr() { @@ -25,14 +47,16 @@ fn test_gen_dplr() { let dplr = DPLR::::new(features); + let lambda = dplr.lambda.clone(); + let b2 = dplr.b.clone().insert_axis(Axis(1)); let p2 = dplr.p.clone().insert_axis(Axis(1)); - let a = Array::from_diag(&dplr.lambda) - p2.dot(&p2.conj().t()); + let a = Array::from_diag(&lambda) - p2.dot(&p2.conj().t()); - let c = randcomplex::(features); - let c2 = c.clone().insert_axis(Axis(0)); + let c = seeded_stdnorm(RNGKEY, features); + let c2 = c.clone().insert_axis(Axis(0)).mapv(AsComplex::as_re); let discrete = { let tmp = discretize(&a, &b2, &c2, step.as_re()); @@ -42,13 +66,15 @@ fn test_gen_dplr() { let (ab, bb, cb) = discrete.into(); // - let ak = k_convolve(&ab, &bb, &cb.conj(), samples); - println!("Ak: {:?}", ak.shape()); + let ak = k_conv(&ab, &bb, &cb.conj(), samples); // - let cc = (&eye - ab.pow(samples)).conj().t().dot(&flatten(cb)); + let cc = { + let tmp = flatten(cb); + (&eye - ab.pow(samples)).conj().t().dot(&tmp) + }; // let params = DPLRParams::new( - dplr.lambda.clone(), + lambda, dplr.p.clone(), dplr.p.clone(), dplr.b.clone(), @@ -68,15 +94,3 @@ fn test_gen_dplr() { ); } -#[test] -fn test_discretize_dplr() { - let (features, samples) = (8, 16); - - let step = (samples as f64).recip(); - - let dplr = DPLR::::new(features); - - let c = Array1::::stdnorm(features); - - // let kernal = kernel_dplr(lambda, p, q, b, c, step, l) -} From 341ec9ad9d25c6239c3b7b37febcaa69d2cf1e6b Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 7 Jan 2024 13:43:47 -0600 Subject: [PATCH 109/118] update Signed-off-by: FL03 --- core/src/ops/fft/fft.rs | 4 +- core/src/ops/fft/mod.rs | 52 +++++++-- core/src/ops/kinds.rs | 32 +++++- core/src/specs/arrays.rs | 43 ++++--- core/src/specs/iter.rs | 8 ++ core/src/specs/math/mod.rs | 62 ++++++++++ core/src/specs/{ => math}/numerical.rs | 14 ++- core/src/specs/{math.rs => math/ops.rs} | 10 +- core/src/specs/math/scalar.rs | 11 ++ core/src/specs/mod.rs | 100 ++++++++-------- core/src/utils.rs | 145 ++++++++++++++---------- data/src/flows/direction.rs | 33 +++++- data/src/tensors/mod.rs | 28 ++++- data/src/tensors/mode.rs | 60 ++++++++++ data/src/tensors/tensor.rs | 30 +++++ ml/s4/src/cmp/kernel.rs | 42 +++---- ml/s4/src/hippo/dplr.rs | 19 ++++ ml/s4/src/ops/convolve.rs | 35 ++---- ml/s4/src/ops/discretize.rs | 92 +++++++++++---- ml/s4/src/ops/mod.rs | 56 ++++++++- ml/s4/src/ssm/model.rs | 9 +- ml/s4/src/utils.rs | 47 +------- ml/s4/tests/dplr.rs | 58 ++++------ 23 files changed, 692 insertions(+), 298 deletions(-) create mode 100644 core/src/specs/iter.rs create mode 100644 core/src/specs/math/mod.rs rename core/src/specs/{ => math}/numerical.rs (91%) rename core/src/specs/{math.rs => math/ops.rs} (93%) create mode 100644 core/src/specs/math/scalar.rs create mode 100644 data/src/tensors/mode.rs diff --git a/core/src/ops/fft/fft.rs b/core/src/ops/fft/fft.rs index a2ec35e8..dbeb3598 100644 --- a/core/src/ops/fft/fft.rs +++ b/core/src/ops/fft/fft.rs @@ -10,12 +10,12 @@ use super::{FftDirection, FftPlan}; // use std::ops::Neg; -pub struct Fft { +pub struct FastFourierTransform { direction: FftDirection, plan: FftPlan, } -impl Fft { +impl FastFourierTransform { pub fn new(direction: FftDirection, plan: FftPlan) -> Self { Self { direction, plan } } diff --git a/core/src/ops/fft/mod.rs b/core/src/ops/fft/mod.rs index 4f0214d4..f0a75351 100644 --- a/core/src/ops/fft/mod.rs +++ b/core/src/ops/fft/mod.rs @@ -11,11 +11,16 @@ pub(crate) mod fft; pub(crate) mod modes; pub(crate) mod plan; +pub trait Fft { + fn fft(&self) -> Vec; + fn ifft(&self) -> Vec; +} + pub(crate) mod utils { use super::FftPlan; use crate::prelude::AsComplex; use num::complex::{Complex, ComplexFloat}; - use num::traits::{Float, FloatConst, NumAssignOps, NumOps}; + use num::traits::{Float, FloatConst, NumAssignOps, NumOps,}; pub(crate) fn fast_fourier_transform_input_permutation(length: usize) -> Vec { let mut result = Vec::new(); @@ -74,10 +79,41 @@ pub(crate) mod utils { result } - pub fn ifft(input: &[S], input_permutation: &FftPlan) -> Vec + pub fn ifft(input: &[Complex], input_permutation: &FftPlan) -> Vec> + where + T: Float + FloatConst + NumOps, Complex>, + Complex: ComplexFloat, + { + let n = input.len(); + let mut result = Vec::new(); + result.reserve_exact(n); + for position in input_permutation.clone().into_iter() { + result.push(input[position]); + } + let mut segment_length = 1_usize; + while segment_length < n { + segment_length <<= 1; + let angle = T::TAU().neg() / T::from(segment_length).unwrap(); + let w_len = Complex::new(T::cos(angle), T::sin(angle)); + for segment_start in (0..n).step_by(segment_length) { + let mut w = Complex::new(T::one(), T::zero()); + for position in segment_start..(segment_start + segment_length / 2) { + let a = result[position]; + let b = result[position + segment_length / 2] * w; + result[position] = a + b; + result[position + segment_length / 2] = a - b; + w = w * w_len; + } + } + } + let scale = T::from(n).unwrap().recip(); + result.iter().map(|x| x * scale).collect() + } + + pub fn ifftr(input: &[S], input_permutation: &FftPlan) -> Vec where - S: ComplexFloat + NumOps + NumOps + NumOps>, - T: Float + FloatConst + NumOps + NumOps, + T: Float + FloatConst + NumOps + NumOps, Complex>, + S: ComplexFloat + NumOps + NumOps>, { let n = input.len(); let mut result = Vec::new(); @@ -89,7 +125,7 @@ pub(crate) mod utils { while segment_length < n { segment_length <<= 1; let angle = T::TAU().neg() / T::from(segment_length).unwrap(); - let w_len = Complex::new(ComplexFloat::cos(angle), ComplexFloat::sin(angle)); + let w_len = Complex::new(T::cos(angle), T::sin(angle)); for segment_start in (0..n).step_by(segment_length) { let mut w = S::one(); for position in segment_start..(segment_start + segment_length / 2) { @@ -133,7 +169,7 @@ mod tests { let polynomial = vec![1.0f64, 1.0, 0.0, 2.5]; let permutation = FftPlan::new(polynomial.len()); let fft = fft(&polynomial, &permutation); - let ifft = ifft(&fft, &permutation); + let ifft = ifftr(&fft, &permutation); for (x, y) in ifft.iter().zip(polynomial.iter()) { assert!(almost_equal(*x, *y, EPSILON)); } @@ -146,7 +182,7 @@ mod tests { let permutation = FftPlan::new(polynomial.len()); let mut fft = fft(&polynomial, &permutation); fft.iter_mut().for_each(|num| *num *= *num); - let ifft = ifft(&fft, &permutation); + let ifft = ifftr(&fft, &permutation); let expected = [1.0, 2.0, 1.0, 4.0, 4.0, 0.0, 4.0, 0.0, 0.0]; for (x, y) in ifft.iter().zip(expected.iter()) { assert!(almost_equal(*x, *y, EPSILON)); @@ -164,7 +200,7 @@ mod tests { let permutation = FftPlan::new(polynomial.len()); let mut fft = fft(&polynomial, &permutation); fft.iter_mut().for_each(|num| *num *= *num); - let ifft = ifft(&fft, &permutation); + let ifft = ifftr(&fft, &permutation); let expected = (0..((n << 1) - 1)).map(|i| std::cmp::min(i + 1, (n << 1) - 1 - i) as f64); for (&x, y) in ifft.iter().zip(expected) { assert!(almost_equal(x, y, EPSILON)); diff --git a/core/src/ops/kinds.rs b/core/src/ops/kinds.rs index c2493dae..20bf1a47 100644 --- a/core/src/ops/kinds.rs +++ b/core/src/ops/kinds.rs @@ -3,4 +3,34 @@ Contrib: FL03 */ -pub enum Ops {} +pub enum Op {} + +pub enum CompareOp { + Eq, + Ge, + Gt, + Le, + Lt, + Ne, +} + +pub enum BinaryOp { + Add, + Div, + Maximum, + Minimum, + Mul, + Sub, +} + +pub trait BinaryOperation { + type Output; + + fn eval(&self, lhs: T, rhs: T) -> Self::Output; +} + +pub trait UnaryOperation { + type Output; + + fn eval(&self, arg: T) -> Self::Output; +} \ No newline at end of file diff --git a/core/src/specs/arrays.rs b/core/src/specs/arrays.rs index bd34df9c..e1d9f97d 100644 --- a/core/src/specs/arrays.rs +++ b/core/src/specs/arrays.rs @@ -2,15 +2,14 @@ Appellation: base Contrib: FL03 */ -use ndarray::prelude::{Array, Axis, Dimension, Ix1, Ix2, NdFloat}; +use ndarray::prelude::{Array, Axis, Dimension, Ix1, Ix2,}; use ndarray::{IntoDimension, ScalarOperand, ShapeError}; -// use ndarray::linalg::Dot; -use distr::uniform::SampleUniform; -use distr::{Bernoulli, BernoulliError, Distribution, StandardNormal, Uniform}; -use ndarray_rand::rand_distr as distr; +use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Distribution, StandardNormal,}; +use ndarray_rand::rand_distr::uniform::{Uniform, SampleUniform}; +use ndarray_rand::rand::rngs::StdRng; +use ndarray_rand::rand::{Rng, SeedableRng}; use ndarray_rand::RandomExt; - -use num::{Float, Num, ToPrimitive}; +use num::traits::{Float, Num, NumAssignOps, ToPrimitive}; use num::traits::real::Real; use std::ops; @@ -184,6 +183,11 @@ where where IdS: Distribution; + fn rand_using(dim: impl IntoDimension, distr: IdS, rng: &mut R) -> Self + where + IdS: Distribution, + R: Rng; + fn bernoulli(dim: impl IntoDimension, p: Option) -> Result where Bernoulli: Distribution, @@ -199,6 +203,14 @@ where Self::rand(dim, StandardNormal) } + fn normal_from_key(key: u64, dim: impl IntoDimension,) -> Self + where + StandardNormal: Distribution, + R: Rng, + { + Self::rand_using(dim.into_dimension(), StandardNormal, &mut StdRng::seed_from_u64(key)) + } + fn uniform(axis: usize, dim: impl IntoDimension) -> Self where T: SampleUniform, @@ -244,6 +256,14 @@ where { Self::random(dim.into_dimension(), distr) } + + fn rand_using(dim: impl IntoDimension, distr: IdS, rng: &mut R) -> Self + where + IdS: Distribution, + R: Rng, + { + Self::random_using(dim.into_dimension(), distr, rng) + } } pub trait IntoAxis { @@ -259,19 +279,16 @@ where } } -pub trait Inverse: Sized -where - T: Float, -{ +pub trait Inverse: Sized { fn inverse(&self) -> Option; } impl Inverse for Array where - T: NdFloat, + T: Copy + Num + NumAssignOps + ScalarOperand, { fn inverse(&self) -> Option { - crate::compute_inverse(self) + super::utils::inverse(self) } } diff --git a/core/src/specs/iter.rs b/core/src/specs/iter.rs new file mode 100644 index 00000000..98343fdb --- /dev/null +++ b/core/src/specs/iter.rs @@ -0,0 +1,8 @@ +/* + Appellation: base + Contrib: FL03 +*/ + +pub trait Walk { + fn walk(&self, other: &T) -> bool; +} diff --git a/core/src/specs/math/mod.rs b/core/src/specs/math/mod.rs new file mode 100644 index 00000000..3f33b844 --- /dev/null +++ b/core/src/specs/math/mod.rs @@ -0,0 +1,62 @@ +/* + Appellation: math + Contrib: FL03 +*/ +pub use self::{numerical::*, ops::*, scalar::*}; + +pub(crate) mod numerical; +pub(crate) mod ops; +pub(crate) mod scalar; + + +use num::complex::Complex; +use num::traits::{Float, Num}; + +pub trait AsComplex: Sized { + fn as_complex(self, real: bool) -> Complex; + + fn as_re(self) -> Complex { + self.as_complex(true) + } + + fn as_im(self) -> Complex { + self.as_complex(false) + } +} + +impl AsComplex for T +where + T: Num, +{ + fn as_complex(self, real: bool) -> Complex { + match real { + true => Complex::new(self, Self::zero()), + false => Complex::new(Self::zero(), self), + } + } +} + +pub trait RoundTo { + fn round_to(&self, places: usize) -> Self; +} + +impl RoundTo for T +where + T: Float, +{ + fn round_to(&self, places: usize) -> Self { + crate::round_to(*self, places) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_as_complex() { + let x = 1.0; + let y = x.as_re(); + assert_eq!(y, Complex::new(1.0, 0.0)); + } +} \ No newline at end of file diff --git a/core/src/specs/numerical.rs b/core/src/specs/math/numerical.rs similarity index 91% rename from core/src/specs/numerical.rs rename to core/src/specs/math/numerical.rs index d4174e51..3d9b201f 100644 --- a/core/src/specs/numerical.rs +++ b/core/src/specs/math/numerical.rs @@ -101,11 +101,19 @@ where Signed::abs(self) } } -pub trait Scalar { - type Complex: NumOps + NumOps; - type Real: NumOps + NumOps; + +pub trait Reciprocal { + fn recip(self) -> Self; +} + +impl Reciprocal for T where T: Num + NumOps { + fn recip(self) -> Self { + Self::one() / self + } } + + pub trait Numerical: Sized { type Elem: Algebraic + Number; diff --git a/core/src/specs/math.rs b/core/src/specs/math/ops.rs similarity index 93% rename from core/src/specs/math.rs rename to core/src/specs/math/ops.rs index 4ad2bdb1..672e8cac 100644 --- a/core/src/specs/math.rs +++ b/core/src/specs/math/ops.rs @@ -1,5 +1,5 @@ /* - Appellation: math + Appellation: ops Contrib: FL03 */ use ndarray::linalg::Dot; @@ -154,11 +154,11 @@ where type Output = Self; fn pow(&self, rhs: usize) -> Self::Output { - if rhs == 0 { - return Array::eye(self.shape()[0]); + if !self.is_square() { + panic!("Matrix must be square to be raised to a power"); } - let mut res = self.clone(); - for _ in 1..rhs { + let mut res = Array::eye(self.shape()[0]); + for _ in 0..rhs { res = res.dot(&self); } res diff --git a/core/src/specs/math/scalar.rs b/core/src/specs/math/scalar.rs new file mode 100644 index 00000000..fc926a30 --- /dev/null +++ b/core/src/specs/math/scalar.rs @@ -0,0 +1,11 @@ +/* + Appellation: scalar + Contrib: FL03 +*/ +// use super::Algebraic; +use num::traits::NumOps; + +pub trait Scalar { + type Complex: NumOps + NumOps; + type Real: NumOps + NumOps; +} \ No newline at end of file diff --git a/core/src/specs/mod.rs b/core/src/specs/mod.rs index fe3482c6..ea5399fa 100644 --- a/core/src/specs/mod.rs +++ b/core/src/specs/mod.rs @@ -2,57 +2,66 @@ Appellation: specs Contrib: FL03 */ -pub use self::{arrays::*, base::*, init::*, math::*, numerical::*}; +pub use self::{arrays::*, base::*, init::*, iter::*, math::*,}; pub(crate) mod arrays; pub(crate) mod base; pub(crate) mod init; +pub(crate) mod iter; pub(crate) mod math; -pub(crate) mod numerical; -use num::complex::Complex; -use num::traits::{Float, Num}; +pub trait Named { + fn name(&self) -> &str; +} -pub trait AsComplex: Sized { - fn as_complex(self, real: bool) -> Complex; - fn as_re(self) -> Complex { - self.as_complex(true) - } +pub(crate) mod utils { + use ndarray::prelude::{s, Array2}; + use ndarray::ScalarOperand; + use num::traits::{Num, NumAssignOps}; - fn as_im(self) -> Complex { - self.as_complex(false) - } -} + pub fn inverse(matrix: &Array2) -> Option> where T: Copy + Num + NumAssignOps + ScalarOperand { + let (rows, cols) = matrix.dim(); -impl AsComplex for T -where - T: Num, -{ - fn as_complex(self, real: bool) -> Complex { - let (re, im): (Self, Self) = if real { - (self, Self::zero()) - } else { - (Self::zero(), self) - }; - Complex::new(re, im) - } -} + if !matrix.is_square() { + return None; // Matrix must be square for inversion + } -pub trait Named { - fn name(&self) -> &str; -} + let identity = Array2::eye(rows); -pub trait RoundTo { - fn round_to(&self, places: usize) -> Self; -} + // Construct an augmented matrix by concatenating the original matrix with an identity matrix + let mut aug = Array2::zeros((rows, 2 * cols)); + aug.slice_mut(s![.., ..cols]).assign(matrix); + aug.slice_mut(s![.., cols..]).assign(&identity); -impl RoundTo for T -where - T: Float, -{ - fn round_to(&self, places: usize) -> Self { - crate::round_to(*self, places) + // Perform Gaussian elimination to reduce the left half to the identity matrix + for i in 0..rows { + let pivot = aug[[i, i]]; + + if pivot == T::zero() { + return None; // Matrix is singular + } + + aug + .slice_mut(s![i, ..]) + .mapv_inplace(|x| x / pivot); + + for j in 0..rows { + if i != j { + let am = aug.clone(); + let factor = aug[[j, i]]; + let rhs = am.slice(s![i, ..]); + aug + .slice_mut(s![j, ..]) + .zip_mut_with(&rhs, |x, &y| *x -= y * factor); + } + } + } + + // Extract the inverted matrix from the augmented matrix + let inverted = aug.slice(s![.., cols..]); + + Some(inverted.to_owned()) } } @@ -68,13 +77,6 @@ mod tests { assert_eq!(&exp, &Array1::::arange(5)) } - #[test] - fn test_as_complex() { - let x = 1.0; - let y = x.as_re(); - assert_eq!(y, Complex::new(1.0, 0.0)); - } - #[test] fn test_affine() { let x = array![[0.0, 1.0], [2.0, 3.0]]; @@ -82,4 +84,12 @@ mod tests { let y = x.affine(4.0, -2.0).unwrap(); assert_eq!(y, array![[-2.0, 2.0], [6.0, 10.0]]); } + + #[test] + fn test_matrix_power() { + let x = array![[1.0, 2.0], [3.0, 4.0]]; + assert_eq!(x.pow(0), Array2::::eye(2)); + assert_eq!(x.pow(1), x); + assert_eq!(x.pow(2), x.dot(&x)); + } } diff --git a/core/src/utils.rs b/core/src/utils.rs index 2c7e456a..c14eb992 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -4,12 +4,16 @@ */ use ndarray::prelude::*; -use ndarray::{concatenate, IntoDimension, RemoveAxis, ShapeError}; -// use ndarray_rand::RandomExt; -// use ndarray_rand::rand_distr::Distribution; +use ndarray::{concatenate, IntoDimension, RemoveAxis, ScalarOperand, ShapeError}; +use ndarray::linalg::Dot; +use ndarray_rand::RandomExt; +use ndarray_rand::rand::SeedableRng; +use ndarray_rand::rand::rngs::StdRng; +use ndarray_rand::rand_distr::{Distribution, StandardNormal}; +use rand::distributions::uniform::{SampleUniform, Uniform}; use num::cast::AsPrimitive; // use num::complex::{Complex, ComplexDistribution}; -use num::{Float, Num, NumCast, Zero}; +use num::{Complex, Float, FromPrimitive, Num, NumCast, Signed, Zero}; pub fn arange(a: T, b: T, h: T) -> Array1 where @@ -24,58 +28,12 @@ where res } -pub fn cauchy_dot(a: &Array, lambda: &Array, omega: &Array) -> T -where - D: Dimension, - T: NdFloat, -{ - (a / (omega - lambda)).sum() -} - -pub fn compute_inverse(matrix: &Array2) -> Option> { - let (rows, cols) = matrix.dim(); - - if !matrix.is_square() { - return None; // Matrix must be square for inversion - } - - let identity = Array2::eye(rows); - - // Concatenate the original matrix with an identity matrix - let mut augmented_matrix = Array2::zeros((rows, 2 * cols)); - augmented_matrix.slice_mut(s![.., ..cols]).assign(matrix); - augmented_matrix.slice_mut(s![.., cols..]).assign(&identity); - - // Perform Gaussian elimination to reduce the left half to the identity matrix - for i in 0..rows { - let pivot = augmented_matrix[[i, i]]; - - if pivot == T::zero() { - return None; // Matrix is singular - } - - augmented_matrix - .slice_mut(s![i, ..]) - .mapv_inplace(|x| x / pivot); - - for j in 0..rows { - if i != j { - let am = augmented_matrix.clone(); - let factor = augmented_matrix[[j, i]]; - let rhs = am.slice(s![i, ..]); - augmented_matrix - .slice_mut(s![j, ..]) - .zip_mut_with(&rhs, |x, &y| *x -= y * factor); - } - } - } - - // Extract the inverted matrix from the augmented matrix - let inverted = augmented_matrix.slice(s![.., cols..]); - - Some(inverted.to_owned()) +pub fn assert_atol(a: &Array, b: &Array, tol: T) where D: Dimension, T: FromPrimitive + PartialOrd + ScalarOperand + Signed + std::fmt::Debug { + let err = (b - a).mapv(|i| i.abs()).mean().unwrap(); + assert!(err <= tol, "Error: {:?}", err); } +/// Creates an n-dimensional array from an iterator of n dimensional arrays. pub fn concat_iter(axis: usize, iter: impl IntoIterator>) -> Array where D: RemoveAxis, @@ -106,14 +64,43 @@ where pub fn linspace(dim: impl IntoDimension) -> Result, ShapeError> where D: Dimension, - T: Float, + T: NumCast, { let dim = dim.into_dimension(); let n = dim.as_array_view().product(); - Array::linspace(T::zero(), T::from(n - 1).unwrap(), n).into_shape(dim) + Array::from_iter((0..n).map(|x| T::from(x).unwrap())).into_shape(dim) +} +/// Raise a matrix to a power +pub fn powmat(a: &Array2, n: usize) -> Array2 +where + T: Clone + Num + 'static, + Array2: Dot, Output = Array2>, +{ + if !a.is_square() { + panic!("Matrix must be square"); + } + let mut res = Array2::::eye(a.nrows()); + for _ in 0..n { + res = res.dot(a); + } + res +} +/// +pub fn randcomplex(shape: impl IntoDimension) -> Array, D> +where + D: Dimension, + T: Copy + Num, + StandardNormal: Distribution, +{ + let dim = shape.into_dimension(); + let re = Array::random(dim.clone(), StandardNormal); + let im = Array::random(dim.clone(), StandardNormal); + let mut res = Array::zeros(dim); + ndarray::azip!((re in &re, im in &im, res in &mut res) { + *res = Complex::new(*re, *im); + }); + res } - - /// creates a matrix from the given shape filled with numerical elements [0, n) spaced evenly by 1 pub fn rangespace(dim: impl IntoDimension) -> Array where @@ -129,6 +116,7 @@ pub fn round_to(val: T, decimals: usize) -> T { let factor = T::from(10).expect("").powi(decimals as i32); (val * factor).round() / factor } + /// Creates a larger array from an iterator of smaller arrays. pub fn stack_iter(iter: impl IntoIterator>) -> Array2 where @@ -144,7 +132,46 @@ where } res } - +/// Creates a random array from a uniform distribution using a given key +pub fn seeded_uniform(key: u64, start: T, stop: T, shape: impl IntoDimension) -> Array +where + D: Dimension, + T: SampleUniform, +{ + Array::random_using(shape, Uniform::new(start, stop), &mut StdRng::seed_from_u64(key)) +} +/// +pub fn seeded_stdnorm(key: u64, shape: impl IntoDimension) -> Array +where + D: Dimension, + StandardNormal: Distribution, +{ + Array::random_using(shape, StandardNormal, &mut StdRng::seed_from_u64(key)) +} +/// +pub fn randc_normal(key: u64, shape: impl IntoDimension) -> Array, D> +where + D: Dimension, + T: Copy + Num, + StandardNormal: Distribution, +{ + let dim = shape.into_dimension(); + let re = seeded_stdnorm(key, dim.clone()); + let im = seeded_stdnorm(key, dim.clone()); + let mut res = Array::zeros(dim); + ndarray::azip!((re in &re, im in &im, res in &mut res) { + *res = Complex::new(*re, *im); + }); + res +} +/// Given a shape, generate a random array using the StandardNormal distribution +pub fn stdnorm(shape: impl IntoDimension) -> Array +where + D: Dimension, + StandardNormal: Distribution, +{ + Array::random(shape, StandardNormal) +} /// Returns the upper triangular portion of a matrix. pub fn triu(a: &Array2) -> Array2 where diff --git a/data/src/flows/direction.rs b/data/src/flows/direction.rs index 68bff080..179fe6b9 100644 --- a/data/src/flows/direction.rs +++ b/data/src/flows/direction.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; +use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; #[derive( Clone, @@ -12,6 +12,7 @@ use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; Default, Deserialize, Display, + EnumCount, EnumIs, EnumIter, EnumString, @@ -27,7 +28,33 @@ use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] pub enum Direction { + Backward = 0, #[default] - Forward, - Backward, + Forward = 1, +} + +impl Direction { + /// A functional alias for [Direction::Backward]. + pub fn backward() -> Self { + Self::Backward + } + /// A functional alias for [Direction::Forward]. + pub fn forward() -> Self { + Self::Forward + } +} + +impl From for usize { + fn from(direction: Direction) -> Self { + direction as usize + } +} + +impl From for Direction { + fn from(index: usize) -> Self { + match index % Self::COUNT { + 0 => Self::Backward, + _ => Self::Forward, + } + } } diff --git a/data/src/tensors/mod.rs b/data/src/tensors/mod.rs index 1aae9f81..92269f0e 100644 --- a/data/src/tensors/mod.rs +++ b/data/src/tensors/mod.rs @@ -5,12 +5,14 @@ //! # Tensors //! //! A tensor is a generalization of vectors and matrices to potentially higher dimensions. -pub use self::tensor::*; +pub use self::{mode::*, tensor::*}; +pub(crate) mod mode; pub(crate) mod tensor; // use ndarray::prelude::{Array, Dimension, Ix2}; use crate::core::ops::Operation; +use num::traits::{Num, NumOps}; pub trait GradStore { type Tensor: NdTensor; @@ -18,6 +20,20 @@ pub trait GradStore { fn get(&self, id: &str) -> Option<&Self::Tensor>; } +pub trait ComplexN: Num + NumOps { + type Real: NumOps; + + fn im(&self) -> Self::Real; + + fn re(&self) -> Self::Real; +} + +pub trait TensorScalar { + type Complex: ComplexN; + type Real: Num + NumOps + NumOps; + +} + pub trait NdTensor { fn affine(&self, a: T, b: T) -> Self; @@ -25,12 +41,20 @@ pub trait NdTensor { where F: Fn(T) -> T; - fn apply_op(&self, op: impl Operation) -> Self; + fn apply_op(&self, op: Op) -> >::Output where Op: Operation, Self: Sized { + op.eval(self) + } fn backward(&self) -> Self; fn id(&self) -> &str; + fn is_variable(&self) -> bool { + self.mode().is_variable() + } + + fn mode(&self) -> TensorKind; + fn tensor(&self) -> &Self; } diff --git a/data/src/tensors/mode.rs b/data/src/tensors/mode.rs new file mode 100644 index 00000000..5fbc1975 --- /dev/null +++ b/data/src/tensors/mode.rs @@ -0,0 +1,60 @@ +/* + Appellation: mode + Contrib: FL03 +*/ +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumCount, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum TensorKind { + #[default] + Standard = 0, + Variable = 1, +} + +impl TensorKind { + /// A functional alias for [TensorKind::Standard]. + pub fn standard() -> Self { + Self::Standard + } + /// A functional alias for [TensorKind::Variable]. + pub fn variable() -> Self { + Self::Variable + } +} + +impl From for usize { + fn from(var: TensorKind) -> Self { + var as usize + } +} + +impl From for TensorKind { + fn from(index: usize) -> Self { + match index % Self::COUNT { + 0 => Self::Standard, + _ => Self::Variable, + } + } +} \ No newline at end of file diff --git a/data/src/tensors/tensor.rs b/data/src/tensors/tensor.rs index 07319a9f..d42f7f58 100644 --- a/data/src/tensors/tensor.rs +++ b/data/src/tensors/tensor.rs @@ -2,6 +2,7 @@ Appellation: tensor Contrib: FL03 */ +use super::TensorKind; use crate::core::id::AtomicId; use crate::prelude::DType; use ndarray::prelude::{Array, Dimension, Ix2}; @@ -18,6 +19,34 @@ where id: AtomicId, data: Array, dtype: DType, + mode: TensorKind, +} + +impl Tensor +where + D: Dimension, +{ + pub fn new(data: Array) -> Self { + Self { + id: AtomicId::new(), + data, + dtype: DType::default(), + mode: TensorKind::default(), + } + } + + pub fn mode(&self) -> TensorKind { + self.mode + } + + pub fn set_mode(&mut self, mode: TensorKind) { + self.mode = mode; + } + + pub fn as_variable(mut self) -> Self { + self.mode = TensorKind::Variable; + self + } } impl Tensor @@ -30,6 +59,7 @@ where id: AtomicId::new(), data: Array::zeros(shape), dtype: DType::default(), + mode: TensorKind::default(), } } } diff --git a/ml/s4/src/cmp/kernel.rs b/ml/s4/src/cmp/kernel.rs index 5c9d2b04..ad7a2bc5 100644 --- a/ml/s4/src/cmp/kernel.rs +++ b/ml/s4/src/cmp/kernel.rs @@ -5,12 +5,12 @@ use crate::core::ops::fft::*; use crate::core::prelude::Conjugate; use crate::prelude::cauchy; -use ndarray::prelude::{Array, Array1, Array2,}; +use ndarray::prelude::{Array, Array1,}; use ndarray::ScalarOperand; use ndarray_linalg::Scalar; use num::complex::{Complex, ComplexFloat}; use num::traits::{Float, FloatConst, NumOps,}; -use rustfft::{FftNum, FftPlanner}; +use rustfft::FftPlanner; use std::ops::Neg; pub struct DPLRParams { @@ -73,7 +73,7 @@ impl DPLRParams> { pub fn omega_l(l: usize) -> Array1<::Complex> where T: Scalar>, - ::Real: Float + FloatConst + NumOps<::Real, ::Real>, + ::Real: Float + FloatConst, ::Complex: ScalarOperand, { let lt = ::Real::from(l).unwrap(); @@ -95,10 +95,10 @@ pub fn kernel_dplr( l: usize, ) -> Array1<::Real> where - T: Conjugate + FftNum + Float + Scalar>, + T: Conjugate + Float + Scalar>, ::Real: FloatConst + NumOps<::Complex, ::Complex> + ScalarOperand, - ::Complex: ScalarOperand, + ::Complex: Conjugate + ScalarOperand, { let one = ::Real::one(); let two = ::Real::from(2).unwrap(); @@ -122,33 +122,33 @@ where let at_roots = &c * (&k00 - k01 * &k11.mapv(|i| one / (i + one)) * &k10); let buffer = at_roots.into_raw_vec(); - println!("Roots:\n\n{:?}\n", &buffer); let permute = FftPlan::new(l); - let res = ifft(buffer.as_slice(), &permute); + let res = ifftr(buffer.as_slice(), &permute); Array::from_vec(res) } pub struct Kernel { - kernal: Array2, + kernal: Array1, } -impl Kernel -where - T: Float, -{ - pub fn new(kernal: Array2) -> Self { +impl Kernel { + pub fn new(kernal: Array1) -> Self { Self { kernal } } - pub fn square(features: usize) -> Self - where - T: Default, - { - let kernal = Array2::::default((features, features)); - Self::new(kernal) + pub fn kernal(&self) -> &Array1 { + &self.kernal } +} - pub fn kernal(&self) -> &Array2 { - &self.kernal +impl Kernel +where + T: Scalar>, + ::Real: Conjugate + Float + FloatConst + NumOps<::Complex, ::Complex> + ScalarOperand, + ::Complex: Conjugate + ScalarOperand +{ + pub fn dplr(dplr: &DPLRParams<::Complex>, step: ::Real, l: usize,) -> Self { + let kernal = kernel_dplr::(dplr, step, l); + Self::new(kernal) } } diff --git a/ml/s4/src/hippo/dplr.rs b/ml/s4/src/hippo/dplr.rs index 495bad41..43b07f27 100644 --- a/ml/s4/src/hippo/dplr.rs +++ b/ml/s4/src/hippo/dplr.rs @@ -128,3 +128,22 @@ where // make_dplr_hippo(features) // } // } + +impl From<(Array1>, Array1>, Array1>, Array2>)> for DPLR +where + T: Clone + Num, +{ + fn from((lambda, p, b, v): (Array1>, Array1>, Array1>, Array2>)) -> Self { + + DPLR { lambda, p, b, v } + } +} + +impl From> for (Array1>, Array1>, Array1>, Array2>) +where + T: Clone + Num, +{ + fn from(dplr: DPLR) -> Self { + (dplr.lambda, dplr.p, dplr.b, dplr.v) + } +} \ No newline at end of file diff --git a/ml/s4/src/ops/convolve.rs b/ml/s4/src/ops/convolve.rs index 62fba100..e6b0e84e 100644 --- a/ml/s4/src/ops/convolve.rs +++ b/ml/s4/src/ops/convolve.rs @@ -2,42 +2,27 @@ Appellation: convolve Contrib: FL03 */ -use crate::core::prelude::{concat_iter, Power}; -use crate::prelude::powmat; +use crate::core::prelude::Power; use ndarray::linalg::Dot; -use ndarray::prelude::{s, Array, Array1, Array2, Axis}; +use ndarray::prelude::{Array, Array1, Array2,}; use ndarray::ScalarOperand; use num::Num; -pub fn convolve() {} - -pub fn k_convolve(a: &Array2, b: &Array2, c: &Array2, l: usize) -> Array1 -where - T: Num + ScalarOperand, - Array2: Dot, Output = Array2> + Dot, Output = Array1>, -{ - let b = b.clone().remove_axis(Axis(1)); - let mut res = Array2::::zeros((l, a.shape()[0])); - for i in 0..l { - let out = c.dot(&a.pow(i).dot(&b)); - res.slice_mut(s![i, ..]).assign(&out); - } - res.remove_axis(Axis(1)) -} - - +/// Generates a large convolution kernal pub fn k_conv(a: &Array2, b: &Array2, c: &Array2, l: usize) -> Array1 where T: Num + ScalarOperand, - Array2: Dot, Output = Array2> + Dot, Output = Array1>, + Array2: Dot, Output = Array2>, { + let f = | i: usize | { + c.dot(&a.pow(i).dot(b)) + }; + let mut store = Vec::new(); for i in 0..l { - let tmp = c.dot(&a.pow(i).dot(b)); - store.extend(tmp); - + store.extend(f(i)); } - Array::from_iter(store) + Array::from_vec(store) } pub struct Filter { diff --git a/ml/s4/src/ops/discretize.rs b/ml/s4/src/ops/discretize.rs index 4a0fb74d..79f57c2b 100644 --- a/ml/s4/src/ops/discretize.rs +++ b/ml/s4/src/ops/discretize.rs @@ -7,7 +7,7 @@ use crate::core::prelude::{Conjugate, Power}; use ndarray::{Array, Array1, Array2, Axis, ScalarOperand}; use ndarray_linalg::{Inverse, Lapack, Scalar}; use num::complex::ComplexFloat; -use num::Float; +use num::traits::{Float, NumOps}; pub fn discretize( a: &Array2, @@ -18,8 +18,9 @@ pub fn discretize( where T: Lapack + Scalar + ScalarOperand, { + let (n, ..) = a.dim(); let ss = step / T::from(2).unwrap(); // half step - let eye = Array2::::eye(a.shape()[0]); + let eye = Array2::::eye(n); let be = (&eye - a * ss).inv().expect("Could not invert matrix"); @@ -29,56 +30,99 @@ where Ok((ab, bb, c.clone()).into()) } +pub fn discrete( + a: &Array2, + b: &Array2, + c: &Array2, + step: S, +) -> anyhow::Result> +where + S: Scalar + ScalarOperand + NumOps, + T: ComplexFloat + Lapack + NumOps, +{ + let (n, ..) = a.dim(); + let ss = step / S::from(2).unwrap(); // half step + let eye = Array2::::eye(n); + + let bl = (&eye - a * ss).inv()?; + + let ab = bl.dot(&(&eye + a * ss)); + let bb = (bl * ss).dot(b); + + Ok((ab, bb, c.clone()).into()) +} + +pub fn disc( + a: &Array2, + b: &Array2, + c: &Array2, + step: S, +) -> anyhow::Result<(Array2, Array2, Array2)> +where + C: ComplexFloat + Lapack + NumOps, + S: Scalar + ScalarOperand + NumOps, + T: ComplexFloat + Lapack + NumOps + NumOps, +{ + let (n, ..) = a.dim(); + let ss = step / S::from(2).unwrap(); // half step + let eye = Array2::::eye(n); + + let bl = (&eye - a * ss).inv()?; + + let ab = bl.dot(&(&eye + a * ss)); + let bb = (bl * ss).dot(b); -pub fn discretize_dplr( - lambda: &Array1, - p: &Array1, - q: &Array1, - b: &Array1, - c: &Array1, + Ok((ab, bb, c.clone())) +} + + + +pub fn discretize_dplr( + lambda: &Array1, + p: &Array1, + q: &Array1, + b: &Array1, + c: &Array1, step: T, l: usize, -) -> anyhow::Result> +) -> anyhow::Result> where - T: ComplexFloat + Conjugate + Lapack + Scalar + ScalarOperand, + T: Float + Conjugate + Lapack + NumOps + Scalar + ScalarOperand, + S: ComplexFloat + Conjugate + Lapack + NumOps, { let n = lambda.dim(); // create an identity matrix; (n, n) - let eye = Array2::::eye(n); + let eye = Array2::::eye(n); // compute the step size let ss = T::from(2).unwrap() * step.recip(); // turn the parameters into two-dimensional matricies let b2 = b.clone().insert_axis(Axis(1)); - let c2 = c.clone().insert_axis(Axis(1)); + let c2 = c.clone().insert_axis(Axis(0)); let p2 = p.clone().insert_axis(Axis(1)); - - let q2 = q.clone().insert_axis(Axis(1)); - // transpose the c matrix - let ct = c2.t(); // compute the conjugate transpose of q - let qct = q2.conj().t().to_owned(); + let qct = q.clone().conj().t().to_owned().insert_axis(Axis(0)); // create a diagonal matrix D from the scaled eigenvalues: Dim(n, n) :: 1 / (step_size - value) let d = Array::from_diag(&lambda.mapv(|i| (ss - i).recip())); // create a diagonal matrix from the eigenvalues - let a = Array::from_diag(&lambda) - &p2.dot(&q2.conj().t()); + let a = Array::from_diag(&lambda) - &p2.dot(&q.clone().insert_axis(Axis(1)).conj().t()); // compute A0 let a0 = &eye * ss + &a; // compute A1 let a1 = { let tmp = qct.dot(&d.dot(&p2)).mapv(|i| (T::one() + i).recip()); - &d - &d.dot(&p2) * &tmp * &qct.dot(&d) + &d - (&d.dot(&p2) * tmp * &qct.dot(&d)) }; // compute a-bar - let ab = a0.dot(&a1); + let ab = a1.dot(&a0); // compute b-bar let bb = a1.dot(&b2) * T::from(2).unwrap(); // compute c-bar - let cb = ct.dot(&(&eye - ab.clone().pow(l)).inv()?.conj()).conj(); + let cb = c2.dot(&(&eye - ab.clone().pow(l)).inv()?.conj()); // return the discretized system - Ok((ab, bb, cb).into()) + Ok((ab, bb, cb.conj()).into()) } pub trait Discretize @@ -117,8 +161,8 @@ impl Discrete where T: Lapack + Scalar + ScalarOperand, { - pub fn discretize(&self, args: &Self, step: T) -> anyhow::Result { - discretize(&args.a, &args.b, &args.c, step) + pub fn discretize(&self, step: T) -> anyhow::Result { + discretize(&self.a, &self.b, &self.c, step) } } diff --git a/ml/s4/src/ops/mod.rs b/ml/s4/src/ops/mod.rs index 3e9ca368..93d4aa5e 100644 --- a/ml/s4/src/ops/mod.rs +++ b/ml/s4/src/ops/mod.rs @@ -10,4 +10,58 @@ pub(crate) mod gen; pub(crate) mod scan; #[cfg(test)] -mod tests {} +mod tests { + use super::*; + use crate::core::prelude::{assert_atol, randc_normal}; + use num::complex::ComplexFloat; + + use crate::cmp::kernel::{kernel_dplr, DPLRParams}; + use crate::hippo::dplr::DPLR; + + const FEATURES: usize = 8; + const RNGKEY: u64 = 1; + const SAMPLES: usize = 16; + + + #[test] + #[ignore = "TODO: fix this test"] + fn test_conversion() { + + let step = (SAMPLES as f64).recip(); + + let dplr = DPLR::::new(FEATURES); + let (lambda, p, b, _) = dplr.clone().into(); + + // let c = randcomplex(features); + let c = randc_normal(RNGKEY, FEATURES); + + let kernel = { + let params = DPLRParams::new(lambda.clone(), p.clone(), p.clone(), b.clone(), c.clone()); + kernel_dplr::(¶ms, step, SAMPLES) + }; + + let discrete = discretize_dplr(&lambda, &p, &p, &b, &c, step, SAMPLES).expect(""); + let (ab, bb, cb) = discrete.into(); + + let k2 = k_conv(&ab, &bb, &cb, SAMPLES); + let k2r = k2.mapv(|i| i.re()); + + assert_atol(&kernel, &k2r, 1e-4); + } + + #[test] + fn test_discretize() { + let step = (SAMPLES as f64).recip(); + + let c = randc_normal(RNGKEY, FEATURES); + + let dplr = DPLR::::new(FEATURES); + let (lambda, p, b, _) = dplr.clone().into(); + + let _discrete = { + let tmp = discretize_dplr(&lambda, &p, &p, &b, &c, step, SAMPLES); + assert!(tmp.is_ok(), "Error: {:?}", tmp.err()); + tmp.unwrap() + }; + } +} diff --git a/ml/s4/src/ssm/model.rs b/ml/s4/src/ssm/model.rs index 90fc6aa2..496a3a96 100644 --- a/ml/s4/src/ssm/model.rs +++ b/ml/s4/src/ssm/model.rs @@ -6,7 +6,7 @@ use super::SSMConfig; use crate::neural::Forward; use crate::ops::Discrete; use crate::params::{SSMParams::*, SSMStore}; -use crate::prelude::{discretize, k_convolve}; +use crate::prelude::{discretize, k_conv}; use ndarray::prelude::{Array1, Array2, Axis, NdFloat}; use ndarray_conv::{Conv2DFftExt, PaddingMode, PaddingSize}; use ndarray_linalg::{Lapack, Scalar}; @@ -81,12 +81,7 @@ where self.ssm = self.discretize(self.config().step_size()).expect(""); self } -} -impl SSM -where - T: NdFloat + Lapack + Scalar, -{ pub fn scan( &self, u: &Array2, @@ -114,7 +109,7 @@ where } pub fn gen_filter(&self) -> Array1 { - k_convolve( + k_conv( &self.params[A], &self.params[B], &self.params[C], diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index 7fa0064e..e00ae78e 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -4,11 +4,10 @@ */ use ndarray::prelude::*; use ndarray::{IntoDimension, ScalarOperand}; -use ndarray::linalg::Dot; use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::{Distribution, StandardNormal, Uniform}; +use ndarray_rand::rand_distr::{Distribution, Uniform}; use ndarray_rand::RandomExt; -use num::complex::{Complex, ComplexDistribution, ComplexFloat}; +use num::complex::{Complex, ComplexDistribution,}; use num::traits::Num; use std::ops::Neg; @@ -31,24 +30,6 @@ where } -pub fn powmat(a: &Array2, n: usize) -> Array2 -where - T: Clone + Num + 'static, - Array2: Dot, Output = Array2>, -{ - if !a.is_square() { - panic!("Matrix must be square"); - } - if n == 0 { - return Array2::::eye(a.nrows()); - } - let mut res = a.clone(); - for _ in 1..n { - res = res.dot(a); - } - res -} - /// Generate a random array of complex numbers with real and imaginary parts in the range [0, 1) pub fn randc(shape: impl IntoDimension) -> Array, D> where @@ -60,30 +41,6 @@ where Array::random(shape, distr) } -pub fn randcomplex(shape: impl IntoDimension) -> Array, D> -where - D: Dimension, - T: Copy + Num, - StandardNormal: Distribution, -{ - let dim = shape.into_dimension(); - let re = Array::random(dim.clone(), StandardNormal); - let im = Array::random(dim.clone(), StandardNormal); - let mut res = Array::zeros(dim); - ndarray::azip!((re in &re, im in &im, res in &mut res) { - *res = Complex::new(*re, *im); - }); - res -} - -pub fn stdnorm(shape: impl IntoDimension) -> Array -where - D: Dimension, - StandardNormal: Distribution, -{ - Array::random(shape, StandardNormal) -} - pub fn scanner( a: &Array2, b: &Array2, diff --git a/ml/s4/tests/dplr.rs b/ml/s4/tests/dplr.rs index 58fe4408..dcd13b6f 100644 --- a/ml/s4/tests/dplr.rs +++ b/ml/s4/tests/dplr.rs @@ -6,38 +6,20 @@ use concision_core as core; use concision_s4 as s4; use ndarray::prelude::*; -use ndarray::IntoDimension; use ndarray_linalg::flatten; -use ndarray_rand::RandomExt; -use ndarray_rand::rand::{rngs::StdRng, SeedableRng}; -use ndarray_rand::rand_distr::{Distribution, StandardNormal}; -use ndarray_rand::rand_distr::uniform::{SampleUniform, Uniform}; use num::complex::ComplexFloat; -use core::prelude::{AsComplex, Conjugate, GenerateRandom, Power}; +use core::prelude::{AsComplex, Conjugate, Power,}; use s4::cmp::kernel::{kernel_dplr, DPLRParams}; use s4::hippo::dplr::DPLR; -use s4::ops::{discretize, k_conv}; +use s4::ops::{discrete, k_conv}; const RNGKEY: u64 = 1; -fn seeded_uniform(key: u64, start: T, stop: T, shape: impl IntoDimension) -> Array -where - D: Dimension, - T: SampleUniform, -{ - Array::random_using(shape, Uniform::new(start, stop), &mut StdRng::seed_from_u64(key)) -} -fn seeded_stdnorm(key: u64, shape: impl IntoDimension) -> Array -where - D: Dimension, - StandardNormal: Distribution, -{ - Array::random_using(shape, StandardNormal, &mut StdRng::seed_from_u64(key)) -} #[test] +// #[ignore = "TODO: fix this test"] fn test_gen_dplr() { let (features, samples) = (4, 16); @@ -46,20 +28,31 @@ fn test_gen_dplr() { let step = (samples as f64).recip(); let dplr = DPLR::::new(features); + let (lambda, p, b, _v) = dplr.into(); + + println!("{:?}", &p); - let lambda = dplr.lambda.clone(); - let b2 = dplr.b.clone().insert_axis(Axis(1)); + let b2 = b.clone().insert_axis(Axis(1)); - let p2 = dplr.p.clone().insert_axis(Axis(1)); + let p2 = p.clone().insert_axis(Axis(1)); let a = Array::from_diag(&lambda) - p2.dot(&p2.conj().t()); - let c = seeded_stdnorm(RNGKEY, features); - let c2 = c.clone().insert_axis(Axis(0)).mapv(AsComplex::as_re); + // let c = { + // let tmp = seeded_uniform(RNGKEY, 0.0, 1.0, (1, features)); + // println!("C:\n\n{:#?}\n", &tmp); + // tmp.mapv(AsComplex::as_re) + // }; + let c = { + let tmp = array![[0.02185547, 0.20907068, 0.23742378, 0.3723395]]; + println!("C:\n\n{:#?}\n", &tmp); + tmp.mapv(AsComplex::as_re) + }; + // TODO: figure out why several of the signs are wrong let discrete = { - let tmp = discretize(&a, &b2, &c2, step.as_re()); + let tmp = discrete(&a, &b2, &c, step); assert!(tmp.is_ok(), "discretize failed: {:?}", tmp.err().unwrap()); tmp.unwrap() }; @@ -68,16 +61,13 @@ fn test_gen_dplr() { // let ak = k_conv(&ab, &bb, &cb.conj(), samples); // - let cc = { - let tmp = flatten(cb); - (&eye - ab.pow(samples)).conj().t().dot(&tmp) - }; + let cc = (&eye - ab.pow(samples)).conj().t().dot(&flatten(cb)); // let params = DPLRParams::new( lambda, - dplr.p.clone(), - dplr.p.clone(), - dplr.b.clone(), + p.clone(), + p.clone(), + b.clone(), cc, ); // From 1eaa176cdf35241891e2a0c4ce47bb5f18dce082 Mon Sep 17 00:00:00 2001 From: FL03 Date: Mon, 8 Jan 2024 15:18:23 -0600 Subject: [PATCH 110/118] update Signed-off-by: FL03 --- core/src/ops/fft/fft.rs | 3 +- core/src/ops/fft/mod.rs | 7 ++- core/src/ops/fft/modes.rs | 22 +++++++++- core/src/ops/fft/plan.rs | 1 - core/src/ops/kinds.rs | 2 +- core/src/specs/arrays.rs | 57 ++++++++++++++++-------- core/src/specs/math/mod.rs | 5 +-- core/src/specs/math/numerical.rs | 18 +++----- core/src/specs/math/scalar.rs | 2 +- core/src/specs/mod.rs | 15 +++---- core/src/utils.rs | 27 +++++++++--- data/src/tensors/mod.rs | 7 ++- data/src/tensors/mode.rs | 2 +- ml/s4/examples/sand.rs | 4 +- ml/s4/src/cmp/kernel.rs | 75 ++++++++++++-------------------- ml/s4/src/hippo/dplr.rs | 28 +++++++++--- ml/s4/src/ops/convolve.rs | 12 ++--- ml/s4/src/ops/discretize.rs | 33 ++------------ ml/s4/src/ops/mod.rs | 5 +-- ml/s4/src/utils.rs | 3 +- ml/s4/tests/dplr.rs | 14 +----- 21 files changed, 175 insertions(+), 167 deletions(-) diff --git a/core/src/ops/fft/fft.rs b/core/src/ops/fft/fft.rs index dbeb3598..65dfa1c6 100644 --- a/core/src/ops/fft/fft.rs +++ b/core/src/ops/fft/fft.rs @@ -9,7 +9,6 @@ use super::{FftDirection, FftPlan}; // use num::traits::real::Real; // use std::ops::Neg; - pub struct FastFourierTransform { direction: FftDirection, plan: FftPlan, @@ -27,4 +26,4 @@ impl FastFourierTransform { pub fn plan(&self) -> &FftPlan { &self.plan } -} \ No newline at end of file +} diff --git a/core/src/ops/fft/mod.rs b/core/src/ops/fft/mod.rs index f0a75351..e7d8dffb 100644 --- a/core/src/ops/fft/mod.rs +++ b/core/src/ops/fft/mod.rs @@ -20,7 +20,7 @@ pub(crate) mod utils { use super::FftPlan; use crate::prelude::AsComplex; use num::complex::{Complex, ComplexFloat}; - use num::traits::{Float, FloatConst, NumAssignOps, NumOps,}; + use num::traits::{Float, FloatConst, NumAssignOps, NumOps}; pub(crate) fn fast_fourier_transform_input_permutation(length: usize) -> Vec { let mut result = Vec::new(); @@ -161,7 +161,10 @@ mod tests { let samples = 16; let plan = FftPlan::new(samples); - assert_eq!(plan.plan(), fast_fourier_transform_input_permutation(16).as_slice()); + assert_eq!( + plan.plan(), + fast_fourier_transform_input_permutation(16).as_slice() + ); } #[test] diff --git a/core/src/ops/fft/modes.rs b/core/src/ops/fft/modes.rs index a7410a46..270db8e4 100644 --- a/core/src/ops/fft/modes.rs +++ b/core/src/ops/fft/modes.rs @@ -5,7 +5,25 @@ use serde::{Deserialize, Serialize}; use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; -#[derive(Clone, Copy, Debug, Default, Deserialize, Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumCount, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] #[repr(usize)] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] @@ -37,4 +55,4 @@ impl From for usize { fn from(direction: FftDirection) -> Self { direction as usize } -} \ No newline at end of file +} diff --git a/core/src/ops/fft/plan.rs b/core/src/ops/fft/plan.rs index 277899d9..2cbf9b4b 100644 --- a/core/src/ops/fft/plan.rs +++ b/core/src/ops/fft/plan.rs @@ -11,7 +11,6 @@ pub struct FftPlan { impl FftPlan { pub fn new(n: usize) -> Self { - let mut permute = Vec::new(); permute.reserve_exact(n); permute.extend(0..n); diff --git a/core/src/ops/kinds.rs b/core/src/ops/kinds.rs index 20bf1a47..273a4f01 100644 --- a/core/src/ops/kinds.rs +++ b/core/src/ops/kinds.rs @@ -33,4 +33,4 @@ pub trait UnaryOperation { type Output; fn eval(&self, arg: T) -> Self::Output; -} \ No newline at end of file +} diff --git a/core/src/specs/arrays.rs b/core/src/specs/arrays.rs index e1d9f97d..ec1e9229 100644 --- a/core/src/specs/arrays.rs +++ b/core/src/specs/arrays.rs @@ -2,15 +2,15 @@ Appellation: base Contrib: FL03 */ -use ndarray::prelude::{Array, Axis, Dimension, Ix1, Ix2,}; +use ndarray::prelude::{Array, Axis, Dimension, Ix1, Ix2}; use ndarray::{IntoDimension, ScalarOperand, ShapeError}; -use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Distribution, StandardNormal,}; -use ndarray_rand::rand_distr::uniform::{Uniform, SampleUniform}; use ndarray_rand::rand::rngs::StdRng; use ndarray_rand::rand::{Rng, SeedableRng}; +use ndarray_rand::rand_distr::uniform::{SampleUniform, Uniform}; +use ndarray_rand::rand_distr::{Bernoulli, BernoulliError, Distribution, StandardNormal}; use ndarray_rand::RandomExt; -use num::traits::{Float, Num, NumAssignOps, ToPrimitive}; use num::traits::real::Real; +use num::traits::{Float, Num, NumAssignOps, ToPrimitive}; use std::ops; pub trait Affine: Sized { @@ -38,7 +38,10 @@ pub enum ArangeArgs { Until { stop: T }, } -impl ArangeArgs where T: Copy + Num { +impl ArangeArgs +where + T: Copy + Num, +{ /// Returns the start value of the range. pub fn start(&self) -> T { match self { @@ -64,7 +67,10 @@ impl ArangeArgs where T: Copy + Num { } } /// Returns the number of steps between the given boundaries - pub fn steps(&self) -> usize where T: Real { + pub fn steps(&self) -> usize + where + T: Real, + { match self { ArangeArgs::Arange { start, stop, step } => { let n = ((*stop - *start) / *step).ceil().to_usize().unwrap(); @@ -91,8 +97,6 @@ impl From> for ArangeArgs { } } - - impl From> for ArangeArgs { fn from(args: ops::RangeFrom) -> Self { ArangeArgs::Until { stop: args.start } @@ -137,8 +141,13 @@ where { fn arange(args: impl Into>) -> Self { let args = args.into(); - let n: usize = args.stop().to_usize().expect("Failed to convert 'stop' to a usize"); - (0..n).map(|i| args.start() + args.step() * T::from(i).unwrap()).collect() + let n: usize = args + .stop() + .to_usize() + .expect("Failed to convert 'stop' to a usize"); + (0..n) + .map(|i| args.start() + args.step() * T::from(i).unwrap()) + .collect() } } @@ -149,7 +158,10 @@ where { fn arange(args: impl Into>) -> Self { let args = args.into(); - let n: usize = args.stop().to_usize().expect("Failed to convert 'stop' to a usize"); + let n: usize = args + .stop() + .to_usize() + .expect("Failed to convert 'stop' to a usize"); let start = T::from(args.start()).unwrap(); let step = T::from(args.step()).unwrap(); @@ -166,10 +178,11 @@ where let args = args.into(); let start = T::from(args.start()).unwrap(); let step = T::from(args.step()).unwrap(); - let n: usize = args.stop().to_usize().expect("Failed to convert 'stop' to a usize"); - let f = | (i, _j) | { - start + step * T::from(i).unwrap() - }; + let n: usize = args + .stop() + .to_usize() + .expect("Failed to convert 'stop' to a usize"); + let f = |(i, _j)| start + step * T::from(i).unwrap(); Array::from_shape_fn((n, 1), f) } } @@ -183,7 +196,11 @@ where where IdS: Distribution; - fn rand_using(dim: impl IntoDimension, distr: IdS, rng: &mut R) -> Self + fn rand_using( + dim: impl IntoDimension, + distr: IdS, + rng: &mut R, + ) -> Self where IdS: Distribution, R: Rng; @@ -203,12 +220,16 @@ where Self::rand(dim, StandardNormal) } - fn normal_from_key(key: u64, dim: impl IntoDimension,) -> Self + fn normal_from_key(key: u64, dim: impl IntoDimension) -> Self where StandardNormal: Distribution, R: Rng, { - Self::rand_using(dim.into_dimension(), StandardNormal, &mut StdRng::seed_from_u64(key)) + Self::rand_using( + dim.into_dimension(), + StandardNormal, + &mut StdRng::seed_from_u64(key), + ) } fn uniform(axis: usize, dim: impl IntoDimension) -> Self diff --git a/core/src/specs/math/mod.rs b/core/src/specs/math/mod.rs index 3f33b844..2d0c72b9 100644 --- a/core/src/specs/math/mod.rs +++ b/core/src/specs/math/mod.rs @@ -8,7 +8,6 @@ pub(crate) mod numerical; pub(crate) mod ops; pub(crate) mod scalar; - use num::complex::Complex; use num::traits::{Float, Num}; @@ -52,11 +51,11 @@ where #[cfg(test)] mod tests { use super::*; - + #[test] fn test_as_complex() { let x = 1.0; let y = x.as_re(); assert_eq!(y, Complex::new(1.0, 0.0)); } -} \ No newline at end of file +} diff --git a/core/src/specs/math/numerical.rs b/core/src/specs/math/numerical.rs index 3d9b201f..7e5b68c2 100644 --- a/core/src/specs/math/numerical.rs +++ b/core/src/specs/math/numerical.rs @@ -6,8 +6,7 @@ use num::complex::Complex; use num::traits::{Num, NumAssignOps, NumOps, Signed}; // use num::traits::real::Real; -pub trait Algebraic: NumOps + Sized { -} +pub trait Algebraic: NumOps + Sized {} pub trait AlgebraicExt where @@ -15,11 +14,7 @@ where { } -impl Algebraic for A -where - A: NumOps + Sized, -{ -} +impl Algebraic for A where A: NumOps + Sized {} pub trait ComplexNum: Sized { type Real: Algebraic + Algebraic; @@ -32,7 +27,7 @@ pub trait ComplexNum: Sized { } pub trait Imaginary: Sized -where +where T: Algebraic + Algebraic, { type Complex: Algebraic + Algebraic; @@ -106,14 +101,15 @@ pub trait Reciprocal { fn recip(self) -> Self; } -impl Reciprocal for T where T: Num + NumOps { +impl Reciprocal for T +where + T: Num + NumOps, +{ fn recip(self) -> Self { Self::one() / self } } - - pub trait Numerical: Sized { type Elem: Algebraic + Number; diff --git a/core/src/specs/math/scalar.rs b/core/src/specs/math/scalar.rs index fc926a30..bbc540cf 100644 --- a/core/src/specs/math/scalar.rs +++ b/core/src/specs/math/scalar.rs @@ -8,4 +8,4 @@ use num::traits::NumOps; pub trait Scalar { type Complex: NumOps + NumOps; type Real: NumOps + NumOps; -} \ No newline at end of file +} diff --git a/core/src/specs/mod.rs b/core/src/specs/mod.rs index ea5399fa..b1dbee82 100644 --- a/core/src/specs/mod.rs +++ b/core/src/specs/mod.rs @@ -2,7 +2,7 @@ Appellation: specs Contrib: FL03 */ -pub use self::{arrays::*, base::*, init::*, iter::*, math::*,}; +pub use self::{arrays::*, base::*, init::*, iter::*, math::*}; pub(crate) mod arrays; pub(crate) mod base; @@ -14,13 +14,15 @@ pub trait Named { fn name(&self) -> &str; } - pub(crate) mod utils { use ndarray::prelude::{s, Array2}; use ndarray::ScalarOperand; use num::traits::{Num, NumAssignOps}; - pub fn inverse(matrix: &Array2) -> Option> where T: Copy + Num + NumAssignOps + ScalarOperand { + pub fn inverse(matrix: &Array2) -> Option> + where + T: Copy + Num + NumAssignOps + ScalarOperand, + { let (rows, cols) = matrix.dim(); if !matrix.is_square() { @@ -42,17 +44,14 @@ pub(crate) mod utils { return None; // Matrix is singular } - aug - .slice_mut(s![i, ..]) - .mapv_inplace(|x| x / pivot); + aug.slice_mut(s![i, ..]).mapv_inplace(|x| x / pivot); for j in 0..rows { if i != j { let am = aug.clone(); let factor = aug[[j, i]]; let rhs = am.slice(s![i, ..]); - aug - .slice_mut(s![j, ..]) + aug.slice_mut(s![j, ..]) .zip_mut_with(&rhs, |x, &y| *x -= y * factor); } } diff --git a/core/src/utils.rs b/core/src/utils.rs index c14eb992..e1f3b345 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -3,15 +3,15 @@ Contrib: FL03 */ +use ndarray::linalg::Dot; use ndarray::prelude::*; use ndarray::{concatenate, IntoDimension, RemoveAxis, ScalarOperand, ShapeError}; -use ndarray::linalg::Dot; -use ndarray_rand::RandomExt; -use ndarray_rand::rand::SeedableRng; use ndarray_rand::rand::rngs::StdRng; +use ndarray_rand::rand::SeedableRng; use ndarray_rand::rand_distr::{Distribution, StandardNormal}; -use rand::distributions::uniform::{SampleUniform, Uniform}; +use ndarray_rand::RandomExt; use num::cast::AsPrimitive; +use rand::distributions::uniform::{SampleUniform, Uniform}; // use num::complex::{Complex, ComplexDistribution}; use num::{Complex, Float, FromPrimitive, Num, NumCast, Signed, Zero}; @@ -28,7 +28,11 @@ where res } -pub fn assert_atol(a: &Array, b: &Array, tol: T) where D: Dimension, T: FromPrimitive + PartialOrd + ScalarOperand + Signed + std::fmt::Debug { +pub fn assert_atol(a: &Array, b: &Array, tol: T) +where + D: Dimension, + T: FromPrimitive + PartialOrd + ScalarOperand + Signed + std::fmt::Debug, +{ let err = (b - a).mapv(|i| i.abs()).mean().unwrap(); assert!(err <= tol, "Error: {:?}", err); } @@ -133,12 +137,21 @@ where res } /// Creates a random array from a uniform distribution using a given key -pub fn seeded_uniform(key: u64, start: T, stop: T, shape: impl IntoDimension) -> Array +pub fn seeded_uniform( + key: u64, + start: T, + stop: T, + shape: impl IntoDimension, +) -> Array where D: Dimension, T: SampleUniform, { - Array::random_using(shape, Uniform::new(start, stop), &mut StdRng::seed_from_u64(key)) + Array::random_using( + shape, + Uniform::new(start, stop), + &mut StdRng::seed_from_u64(key), + ) } /// pub fn seeded_stdnorm(key: u64, shape: impl IntoDimension) -> Array diff --git a/data/src/tensors/mod.rs b/data/src/tensors/mod.rs index 92269f0e..46fbbf9a 100644 --- a/data/src/tensors/mod.rs +++ b/data/src/tensors/mod.rs @@ -31,7 +31,6 @@ pub trait ComplexN: Num + NumOps { pub trait TensorScalar { type Complex: ComplexN; type Real: Num + NumOps + NumOps; - } pub trait NdTensor { @@ -41,7 +40,11 @@ pub trait NdTensor { where F: Fn(T) -> T; - fn apply_op(&self, op: Op) -> >::Output where Op: Operation, Self: Sized { + fn apply_op(&self, op: Op) -> >::Output + where + Op: Operation, + Self: Sized, + { op.eval(self) } diff --git a/data/src/tensors/mode.rs b/data/src/tensors/mode.rs index 5fbc1975..ee153054 100644 --- a/data/src/tensors/mode.rs +++ b/data/src/tensors/mode.rs @@ -57,4 +57,4 @@ impl From for TensorKind { _ => Self::Variable, } } -} \ No newline at end of file +} diff --git a/ml/s4/examples/sand.rs b/ml/s4/examples/sand.rs index 58f66d3c..69eef64c 100644 --- a/ml/s4/examples/sand.rs +++ b/ml/s4/examples/sand.rs @@ -5,10 +5,10 @@ use concision_core as core; use concision_s4 as s4; use core::prelude::{Arange, AsComplex}; -use s4::prelude::cauchy; -use s4::ssm::{SSM, SSMConfig}; use ndarray::prelude::*; use rustfft::FftPlanner; +use s4::prelude::cauchy; +use s4::ssm::{SSMConfig, SSM}; fn main() -> anyhow::Result<()> { let (features, samples) = (4, 16); diff --git a/ml/s4/src/cmp/kernel.rs b/ml/s4/src/cmp/kernel.rs index ad7a2bc5..b967a671 100644 --- a/ml/s4/src/cmp/kernel.rs +++ b/ml/s4/src/cmp/kernel.rs @@ -5,11 +5,11 @@ use crate::core::ops::fft::*; use crate::core::prelude::Conjugate; use crate::prelude::cauchy; -use ndarray::prelude::{Array, Array1,}; +use ndarray::prelude::{Array, Array1}; use ndarray::ScalarOperand; use ndarray_linalg::Scalar; use num::complex::{Complex, ComplexFloat}; -use num::traits::{Float, FloatConst, NumOps,}; +use num::traits::{Float, FloatConst, NumOps}; use rustfft::FftPlanner; use std::ops::Neg; @@ -44,45 +44,18 @@ impl DPLRParams { // } // } -impl DPLRParams> { - pub fn kernel_s(&self, step: f64, l: usize) -> Array1 { - let omega_l = omega_l::(l); - - let aterm = (self.c.conj(), self.q.conj()); - let bterm = (self.b.clone(), self.p.clone()); - - let g = ((&omega_l.clone().neg() + 1.0) / (&omega_l + 1.0)) * (2.0 * step.recip()); - let c = omega_l.mapv(|i| 2.0 / (1.0 + i)); - - let k00 = cauchy(&(&aterm.0 * &bterm.0), &g, &self.lambda); - let k01 = cauchy(&(&aterm.0 * &bterm.1), &g, &self.lambda); - let k10 = cauchy(&(&aterm.1 * &bterm.0), &g, &self.lambda); - let k11 = cauchy(&(&aterm.1 * &bterm.1), &g, &self.lambda); - - let at_roots = &c * (&k00 - k01 * (&k11 + 1.0).mapv(ComplexFloat::recip) * &k10); - - let mut fft_planner = FftPlanner::new(); - let fft = fft_planner.plan_fft_inverse(l); - // create a buffer to hold the complex numbers - let mut buffer = at_roots.into_raw_vec(); - fft.process(buffer.as_mut_slice()); - Array::from_iter(buffer.into_iter().map(|i| i.re())) - } -} - pub fn omega_l(l: usize) -> Array1<::Complex> where T: Scalar>, - ::Real: Float + FloatConst, - ::Complex: ScalarOperand, + ::Real: FloatConst + NumOps<::Complex, ::Complex>, + ::Complex: + ComplexFloat::Real> + NumOps<::Real> + ScalarOperand, { - let lt = ::Real::from(l).unwrap(); let f = |i: usize| -> ::Complex { - let im = ::Real::PI() - .mul_complex(Complex::new(T::zero(), T::from(2).unwrap().neg())); - ::Real::from(i) + let im = T::PI().mul_complex(Complex::i() * T::from(2).unwrap()); + T::from(i) .unwrap() - .div_real(lt) + .div_real(T::from(l).unwrap()) .mul_complex(im) .exp() }; @@ -100,8 +73,8 @@ where FloatConst + NumOps<::Complex, ::Complex> + ScalarOperand, ::Complex: Conjugate + ScalarOperand, { - let one = ::Real::one(); - let two = ::Real::from(2).unwrap(); + let one = T::one(); + let two = T::from(2).unwrap(); // get the lambda matrix let lambda = dplr.lambda.clone(); // generate omega @@ -111,15 +84,15 @@ where // collect the relevant terms for B let bterm = (dplr.b.clone(), dplr.p.clone()); - let g = omega_l.mapv(|i| (one - i) / (one + i)) * (two / step); - let c = omega_l.mapv(|i| two.div_complex(one.add_complex(i))); - + let g = omega_l.mapv(|i| (one - i) * (one + i).recip()) * (two * step.recip()); + let c = omega_l.mapv(|i| two * (one + i).recip()); + // compute the cauchy matrix let k00: Array1<::Complex> = cauchy(&(&aterm.0 * &bterm.0), &g, &lambda); let k01: Array1<::Complex> = cauchy(&(&aterm.0 * &bterm.1), &g, &lambda); let k10: Array1<::Complex> = cauchy(&(&aterm.1 * &bterm.0), &g, &lambda); let k11: Array1<::Complex> = cauchy(&(&aterm.1 * &bterm.1), &g, &lambda); - - let at_roots = &c * (&k00 - k01 * &k11.mapv(|i| one / (i + one)) * &k10); + // compute the roots of unity + let at_roots = &c * (&k00 - k01 * &k11.mapv(|i| (i + one).recip()) * &k10); let buffer = at_roots.into_raw_vec(); let permute = FftPlan::new(l); @@ -141,13 +114,21 @@ impl Kernel { } } -impl Kernel -where +impl Kernel +where T: Scalar>, - ::Real: Conjugate + Float + FloatConst + NumOps<::Complex, ::Complex> + ScalarOperand, - ::Complex: Conjugate + ScalarOperand + ::Real: Conjugate + + Float + + FloatConst + + NumOps<::Complex, ::Complex> + + ScalarOperand, + ::Complex: Conjugate + ScalarOperand, { - pub fn dplr(dplr: &DPLRParams<::Complex>, step: ::Real, l: usize,) -> Self { + pub fn dplr( + dplr: &DPLRParams<::Complex>, + step: ::Real, + l: usize, + ) -> Self { let kernal = kernel_dplr::(dplr, step, l); Self::new(kernal) } diff --git a/ml/s4/src/hippo/dplr.rs b/ml/s4/src/hippo/dplr.rs index 43b07f27..5ca05234 100644 --- a/ml/s4/src/hippo/dplr.rs +++ b/ml/s4/src/hippo/dplr.rs @@ -129,21 +129,39 @@ where // } // } -impl From<(Array1>, Array1>, Array1>, Array2>)> for DPLR +impl + From<( + Array1>, + Array1>, + Array1>, + Array2>, + )> for DPLR where T: Clone + Num, { - fn from((lambda, p, b, v): (Array1>, Array1>, Array1>, Array2>)) -> Self { - + fn from( + (lambda, p, b, v): ( + Array1>, + Array1>, + Array1>, + Array2>, + ), + ) -> Self { DPLR { lambda, p, b, v } } } -impl From> for (Array1>, Array1>, Array1>, Array2>) +impl From> + for ( + Array1>, + Array1>, + Array1>, + Array2>, + ) where T: Clone + Num, { fn from(dplr: DPLR) -> Self { (dplr.lambda, dplr.p, dplr.b, dplr.v) } -} \ No newline at end of file +} diff --git a/ml/s4/src/ops/convolve.rs b/ml/s4/src/ops/convolve.rs index e6b0e84e..7e32d57f 100644 --- a/ml/s4/src/ops/convolve.rs +++ b/ml/s4/src/ops/convolve.rs @@ -4,19 +4,17 @@ */ use crate::core::prelude::Power; use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Array1, Array2,}; +use ndarray::prelude::{Array, Array1, Array2}; use ndarray::ScalarOperand; use num::Num; -/// Generates a large convolution kernal +/// Generates a large convolution kernal pub fn k_conv(a: &Array2, b: &Array2, c: &Array2, l: usize) -> Array1 where T: Num + ScalarOperand, Array2: Dot, Output = Array2>, { - let f = | i: usize | { - c.dot(&a.pow(i).dot(b)) - }; + let f = |i: usize| c.dot(&a.pow(i).dot(b)); let mut store = Vec::new(); for i in 0..l { @@ -25,6 +23,4 @@ where Array::from_vec(store) } -pub struct Filter { - -} \ No newline at end of file +pub struct Filter {} diff --git a/ml/s4/src/ops/discretize.rs b/ml/s4/src/ops/discretize.rs index 79f57c2b..1485b4d8 100644 --- a/ml/s4/src/ops/discretize.rs +++ b/ml/s4/src/ops/discretize.rs @@ -37,7 +37,7 @@ pub fn discrete( step: S, ) -> anyhow::Result> where - S: Scalar + ScalarOperand + NumOps, + S: Scalar + ScalarOperand + NumOps, T: ComplexFloat + Lapack + NumOps, { let (n, ..) = a.dim(); @@ -47,36 +47,11 @@ where let bl = (&eye - a * ss).inv()?; let ab = bl.dot(&(&eye + a * ss)); - let bb = (bl * ss).dot(b); + let bb = (bl * step).dot(b); Ok((ab, bb, c.clone()).into()) } -pub fn disc( - a: &Array2, - b: &Array2, - c: &Array2, - step: S, -) -> anyhow::Result<(Array2, Array2, Array2)> -where - C: ComplexFloat + Lapack + NumOps, - S: Scalar + ScalarOperand + NumOps, - T: ComplexFloat + Lapack + NumOps + NumOps, -{ - let (n, ..) = a.dim(); - let ss = step / S::from(2).unwrap(); // half step - let eye = Array2::::eye(n); - - let bl = (&eye - a * ss).inv()?; - - let ab = bl.dot(&(&eye + a * ss)); - let bb = (bl * ss).dot(b); - - Ok((ab, bb, c.clone())) -} - - - pub fn discretize_dplr( lambda: &Array1, p: &Array1, @@ -87,14 +62,14 @@ pub fn discretize_dplr( l: usize, ) -> anyhow::Result> where - T: Float + Conjugate + Lapack + NumOps + Scalar + ScalarOperand, + T: Float + Conjugate + Lapack + NumOps + Scalar + ScalarOperand, S: ComplexFloat + Conjugate + Lapack + NumOps, { let n = lambda.dim(); // create an identity matrix; (n, n) let eye = Array2::::eye(n); // compute the step size - let ss = T::from(2).unwrap() * step.recip(); + let ss = T::from(2).unwrap() / step; // turn the parameters into two-dimensional matricies let b2 = b.clone().insert_axis(Axis(1)); diff --git a/ml/s4/src/ops/mod.rs b/ml/s4/src/ops/mod.rs index 93d4aa5e..513df961 100644 --- a/ml/s4/src/ops/mod.rs +++ b/ml/s4/src/ops/mod.rs @@ -22,11 +22,9 @@ mod tests { const RNGKEY: u64 = 1; const SAMPLES: usize = 16; - #[test] #[ignore = "TODO: fix this test"] fn test_conversion() { - let step = (SAMPLES as f64).recip(); let dplr = DPLR::::new(FEATURES); @@ -36,7 +34,8 @@ mod tests { let c = randc_normal(RNGKEY, FEATURES); let kernel = { - let params = DPLRParams::new(lambda.clone(), p.clone(), p.clone(), b.clone(), c.clone()); + let params = + DPLRParams::new(lambda.clone(), p.clone(), p.clone(), b.clone(), c.clone()); kernel_dplr::(¶ms, step, SAMPLES) }; diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index e00ae78e..a932605f 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -7,7 +7,7 @@ use ndarray::{IntoDimension, ScalarOperand}; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, Uniform}; use ndarray_rand::RandomExt; -use num::complex::{Complex, ComplexDistribution,}; +use num::complex::{Complex, ComplexDistribution}; use num::traits::Num; use std::ops::Neg; @@ -29,7 +29,6 @@ where Array::random(shape, Uniform::new(a, b)) * (b.ln() - a.ln()) + a.ln() } - /// Generate a random array of complex numbers with real and imaginary parts in the range [0, 1) pub fn randc(shape: impl IntoDimension) -> Array, D> where diff --git a/ml/s4/tests/dplr.rs b/ml/s4/tests/dplr.rs index dcd13b6f..4a2d8ae4 100644 --- a/ml/s4/tests/dplr.rs +++ b/ml/s4/tests/dplr.rs @@ -9,15 +9,13 @@ use ndarray::prelude::*; use ndarray_linalg::flatten; use num::complex::ComplexFloat; -use core::prelude::{AsComplex, Conjugate, Power,}; +use core::prelude::{AsComplex, Conjugate, Power}; use s4::cmp::kernel::{kernel_dplr, DPLRParams}; use s4::hippo::dplr::DPLR; use s4::ops::{discrete, k_conv}; const RNGKEY: u64 = 1; - - #[test] // #[ignore = "TODO: fix this test"] fn test_gen_dplr() { @@ -32,7 +30,6 @@ fn test_gen_dplr() { println!("{:?}", &p); - let b2 = b.clone().insert_axis(Axis(1)); let p2 = p.clone().insert_axis(Axis(1)); @@ -63,13 +60,7 @@ fn test_gen_dplr() { // let cc = (&eye - ab.pow(samples)).conj().t().dot(&flatten(cb)); // - let params = DPLRParams::new( - lambda, - p.clone(), - p.clone(), - b.clone(), - cc, - ); + let params = DPLRParams::new(lambda, p.clone(), p.clone(), b.clone(), cc); // let kernal = kernel_dplr::(¶ms, step, samples); println!("Kernal: {:?}", kernal.shape()); @@ -83,4 +74,3 @@ fn test_gen_dplr() { 1e-4 ); } - From 642796da6c2a0f2c62a0faa02b91c4a92b8e0781 Mon Sep 17 00:00:00 2001 From: FL03 Date: Mon, 8 Jan 2024 15:18:54 -0600 Subject: [PATCH 111/118] update Signed-off-by: FL03 --- ml/s4/src/cmp/kernel.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ml/s4/src/cmp/kernel.rs b/ml/s4/src/cmp/kernel.rs index b967a671..875fe7c4 100644 --- a/ml/s4/src/cmp/kernel.rs +++ b/ml/s4/src/cmp/kernel.rs @@ -52,7 +52,7 @@ where ComplexFloat::Real> + NumOps<::Real> + ScalarOperand, { let f = |i: usize| -> ::Complex { - let im = T::PI().mul_complex(Complex::i() * T::from(2).unwrap()); + let im = T::PI().mul_complex(Complex::i() * T::from(2).unwrap()); // .neg() T::from(i) .unwrap() .div_real(T::from(l).unwrap()) From 0eb8eeee4c1e77a1bba1ced402adb76d41bdc1b0 Mon Sep 17 00:00:00 2001 From: FL03 Date: Mon, 8 Jan 2024 15:19:48 -0600 Subject: [PATCH 112/118] update Signed-off-by: FL03 --- ml/s4/src/ops/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/ml/s4/src/ops/mod.rs b/ml/s4/src/ops/mod.rs index 513df961..6da71a03 100644 --- a/ml/s4/src/ops/mod.rs +++ b/ml/s4/src/ops/mod.rs @@ -23,7 +23,6 @@ mod tests { const SAMPLES: usize = 16; #[test] - #[ignore = "TODO: fix this test"] fn test_conversion() { let step = (SAMPLES as f64).recip(); From be103fb5df956c31e0c90c51d936b3c8b5629d60 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sat, 27 Jan 2024 09:33:33 -0600 Subject: [PATCH 113/118] update Signed-off-by: FL03 --- core/src/masks/mask.rs | 52 +++---- core/src/masks/mod.rs | 5 + core/src/ops/fft/mod.rs | 154 ++++++++++++++++----- core/src/ops/fft/modes.rs | 28 ++++ core/src/params/mod.rs | 6 +- core/src/params/variable.rs | 16 +++ core/src/utils.rs | 15 ++ ml/s4/src/cmp/kernel.rs | 75 +++++----- ml/s4/src/ops/convolve.rs | 53 ++++++- ml/s4/src/ops/discretize.rs | 58 +++----- ml/s4/src/ops/mod.rs | 3 +- ml/s4/src/params/dplr.rs | 42 ++++++ ml/s4/src/params/kinds.rs | 2 +- ml/s4/src/params/mod.rs | 5 +- ml/s4/src/params/store.rs | 69 ++++----- ml/s4/src/ssm/mod.rs | 4 +- ml/s4/src/ssm/model.rs | 151 ++++++++++---------- ml/s4/tests/dplr.rs | 32 +++-- ml/transformers/src/attention/multi/mod.rs | 2 +- ml/transformers/src/codec/encode/mod.rs | 2 +- 20 files changed, 496 insertions(+), 278 deletions(-) create mode 100644 core/src/params/variable.rs create mode 100644 ml/s4/src/params/dplr.rs diff --git a/core/src/masks/mask.rs b/core/src/masks/mask.rs index de49bcdd..37451de7 100644 --- a/core/src/masks/mask.rs +++ b/core/src/masks/mask.rs @@ -2,24 +2,27 @@ Appellation: mask Contrib: FL03 */ -use ndarray::prelude::{Array, Array2}; -use ndarray::Dimension; -use ndarray_rand::rand_distr::{uniform::SampleUniform, Uniform}; +use ndarray::prelude::{Array, Array2, Dimension}; +use ndarray::ScalarOperand; +use ndarray_rand::rand_distr::uniform::{SampleUniform, Uniform}; use ndarray_rand::RandomExt; -use num::Float; +use num::traits::{Float, NumOps}; use serde::{Deserialize, Serialize}; use smart_default::SmartDefault; use std::ops; use strum::EnumIs; #[derive(Clone, Debug, Deserialize, EnumIs, PartialEq, Serialize, SmartDefault)] -pub enum Mask { +pub enum Mask { Masked(Array2), #[default] Unmasked, } -impl Mask { +impl Mask +where + T: NumOps + ScalarOperand, +{ pub fn forward(&self, data: &Array2) -> Array2 { match self { Self::Masked(bias) => data + bias, @@ -28,11 +31,11 @@ impl Mask { } } -impl Mask +impl Mask where T: Float + SampleUniform, { - pub fn masked(size: usize) -> Self { + pub fn uniform(size: usize) -> Self { let ds = (T::from(size).unwrap()).sqrt(); let dist = Uniform::new(-ds, ds); let mask = Array2::::random((size, size), dist); @@ -40,7 +43,7 @@ where } } -impl From for Mask +impl From for Mask where T: Float + SampleUniform, { @@ -52,19 +55,13 @@ where } } -impl From> for Mask -where - T: Float, -{ +impl From> for Mask { fn from(bias: Array2) -> Self { Self::Masked(bias) } } -impl From>> for Mask -where - T: Float, -{ +impl From>> for Mask { fn from(bias: Option>) -> Self { match bias { Some(bias) => Self::Masked(bias), @@ -73,10 +70,7 @@ where } } -impl From> for Option> -where - T: Float, -{ +impl From> for Option> { fn from(bias: Mask) -> Self { match bias { Mask::Masked(bias) => Some(bias), @@ -85,8 +79,10 @@ where } } -impl ops::Add> for Mask +impl ops::Add> for Mask where + D: Dimension, + T: NumOps + ScalarOperand, Array: ops::Add, Output = Array>, { type Output = Array; @@ -100,8 +96,10 @@ where } } -impl ops::Add<&Array> for Mask +impl ops::Add<&Array> for Mask where + D: Dimension, + T: NumOps + ScalarOperand, Array: ops::Add, Output = Array>, { type Output = Array; @@ -115,8 +113,10 @@ where } } -impl ops::Add> for Array +impl ops::Add> for Array where + D: Dimension, + T: NumOps + ScalarOperand, Array: ops::Add, Output = Array>, { type Output = Array; @@ -130,8 +130,10 @@ where } } -impl ops::Add<&Mask> for Array +impl ops::Add<&Mask> for Array where + D: Dimension, + T: NumOps + ScalarOperand, Array: ops::Add, Output = Array>, { type Output = Array; diff --git a/core/src/masks/mod.rs b/core/src/masks/mod.rs index 6748f63b..db9ce4cd 100644 --- a/core/src/masks/mod.rs +++ b/core/src/masks/mod.rs @@ -7,6 +7,11 @@ pub use self::{mask::*, utils::*}; pub(crate) mod mask; +pub trait Masked { + fn mask(&self) -> &Mask; + fn mask_mut(&mut self) -> &mut Mask; +} + pub(crate) mod utils { use super::Mask; use ndarray::prelude::Array2; diff --git a/core/src/ops/fft/mod.rs b/core/src/ops/fft/mod.rs index e7d8dffb..cf0324b0 100644 --- a/core/src/ops/fft/mod.rs +++ b/core/src/ops/fft/mod.rs @@ -20,36 +20,55 @@ pub(crate) mod utils { use super::FftPlan; use crate::prelude::AsComplex; use num::complex::{Complex, ComplexFloat}; - use num::traits::{Float, FloatConst, NumAssignOps, NumOps}; + use num::traits::{Float, FloatConst, NumOps}; + + pub(crate) fn fft_angle(n: usize) -> T + where + T: Float + FloatConst, + { + T::TAU() / T::from(n).unwrap() + } + + /// Computes the Fast Fourier Transform of a general signal. + pub fn fft(input: impl AsRef<[T]>, input_permutation: impl AsRef<[usize]>) -> Vec> + where + T: Float + FloatConst, + Complex: ComplexFloat, + { + let input = input.as_ref(); + + let n = input.len(); - pub(crate) fn fast_fourier_transform_input_permutation(length: usize) -> Vec { let mut result = Vec::new(); - result.reserve_exact(length); - for i in 0..length { - result.push(i); + result.reserve_exact(n); + for position in input_permutation.as_ref() { + result.push(input[*position].as_re()); } - let mut reverse = 0_usize; - let mut position = 1_usize; - while position < length { - let mut bit = length >> 1; - while bit & reverse != 0 { - reverse ^= bit; - bit >>= 1; - } - reverse ^= bit; - // This is equivalent to adding 1 to a reversed number - if position < reverse { - // Only swap each element once - result.swap(position, reverse); + let mut segment_length = 1_usize; + while segment_length < n { + segment_length <<= 1; + let angle = fft_angle::(segment_length); + let w_len = Complex::new(angle.cos(), angle.sin()); + for segment_start in (0..n).step_by(segment_length) { + let mut w = Complex::new(T::one(), T::zero()); + for position in segment_start..(segment_start + segment_length / 2) { + let a = result[position]; + let b = result[position + segment_length / 2] * w; + result[position] = a + b; + result[position + segment_length / 2] = a - b; + w = w * w_len; + } } - position += 1; } result } - pub fn fft(input: impl AsRef<[T]>, input_permutation: impl AsRef<[usize]>) -> Vec> + /// Computes the Fast Fourier Transform of a real-valued signal. + /// TODO: Fix the function; real-valued fft only computes the positive frequency terms + pub fn rfft(input: impl AsRef<[T]>, input_permutation: impl AsRef<[usize]>) -> Vec> where - T: AsComplex + Float + FloatConst + NumOps + NumOps, Complex> + NumAssignOps, + T: Float + FloatConst, + Complex: ComplexFloat, { let input = input.as_ref(); @@ -63,7 +82,7 @@ pub(crate) mod utils { let mut segment_length = 1_usize; while segment_length < n { segment_length <<= 1; - let angle = T::TAU() / T::from(segment_length).unwrap(); + let angle = fft_angle::(segment_length); let w_len = Complex::new(angle.cos(), angle.sin()); for segment_start in (0..n).step_by(segment_length) { let mut w = Complex::new(T::one(), T::zero()); @@ -72,16 +91,16 @@ pub(crate) mod utils { let b = result[position + segment_length / 2] * w; result[position] = a + b; result[position + segment_length / 2] = a - b; - w *= w_len; + w = w * w_len; } } } result } - + /// Computes the Inverse Fast Fourier Transform of a signal. pub fn ifft(input: &[Complex], input_permutation: &FftPlan) -> Vec> where - T: Float + FloatConst + NumOps, Complex>, + T: Float + FloatConst, Complex: ComplexFloat, { let n = input.len(); @@ -93,7 +112,7 @@ pub(crate) mod utils { let mut segment_length = 1_usize; while segment_length < n { segment_length <<= 1; - let angle = T::TAU().neg() / T::from(segment_length).unwrap(); + let angle = fft_angle::(segment_length); let w_len = Complex::new(T::cos(angle), T::sin(angle)); for segment_start in (0..n).step_by(segment_length) { let mut w = Complex::new(T::one(), T::zero()); @@ -110,10 +129,41 @@ pub(crate) mod utils { result.iter().map(|x| x * scale).collect() } + pub fn irfft(input: &[Complex], input_permutation: &FftPlan) -> Vec + where + T: Float + FloatConst + NumOps, Complex>, + Complex: ComplexFloat, + { + let n = input.len(); + let mut result = Vec::new(); + result.reserve_exact(n); + for position in input_permutation.clone().into_iter() { + result.push(input[position]); + } + let mut segment_length = 1_usize; + while segment_length < n { + segment_length <<= 1; + let angle = fft_angle::(segment_length); + let w_len = Complex::new(T::cos(angle), T::sin(angle)); + for segment_start in (0..n).step_by(segment_length) { + let mut w = Complex::new(T::one(), T::zero()); + for position in segment_start..(segment_start + segment_length / 2) { + let a = result[position]; + let b = result[position + segment_length / 2] * w; + result[position] = a + b; + result[position + segment_length / 2] = a - b; + w = w * w_len; + } + } + } + let scale = T::from(n).unwrap().recip(); + result.iter().map(|x| x.re() * scale).collect() + } + /// Computes the Inverse Fast Fourier Transform of a real-valued signal. pub fn ifftr(input: &[S], input_permutation: &FftPlan) -> Vec where - T: Float + FloatConst + NumOps + NumOps, Complex>, - S: ComplexFloat + NumOps + NumOps>, + T: Float + FloatConst, + S: ComplexFloat + NumOps>, { let n = input.len(); let mut result = Vec::new(); @@ -124,7 +174,7 @@ pub(crate) mod utils { let mut segment_length = 1_usize; while segment_length < n { segment_length <<= 1; - let angle = T::TAU().neg() / T::from(segment_length).unwrap(); + let angle = fft_angle::(segment_length); let w_len = Complex::new(T::cos(angle), T::sin(angle)); for segment_start in (0..n).step_by(segment_length) { let mut w = S::one(); @@ -145,13 +195,32 @@ pub(crate) mod utils { #[cfg(test)] mod tests { use super::*; - use num::Signed; + use crate::prelude::almost_equal; + use num::complex::ComplexFloat; - fn almost_equal(a: T, b: T, epsilon: T) -> bool - where - T: PartialOrd + Signed, - { - (a - b).abs() < epsilon + pub(crate) fn fast_fourier_transform_input_permutation(length: usize) -> Vec { + let mut result = Vec::new(); + result.reserve_exact(length); + for i in 0..length { + result.push(i); + } + let mut reverse = 0_usize; + let mut position = 1_usize; + while position < length { + let mut bit = length >> 1; + while bit & reverse != 0 { + reverse ^= bit; + bit >>= 1; + } + reverse ^= bit; + // This is equivalent to adding 1 to a reversed number + if position < reverse { + // Only swap each element once + result.swap(position, reverse); + } + position += 1; + } + result } const EPSILON: f64 = 1e-6; @@ -172,7 +241,10 @@ mod tests { let polynomial = vec![1.0f64, 1.0, 0.0, 2.5]; let permutation = FftPlan::new(polynomial.len()); let fft = fft(&polynomial, &permutation); - let ifft = ifftr(&fft, &permutation); + let ifft = irfft(&fft, &permutation) + .into_iter() + .map(|i| i.re()) + .collect::>(); for (x, y) in ifft.iter().zip(polynomial.iter()) { assert!(almost_equal(*x, *y, EPSILON)); } @@ -185,7 +257,10 @@ mod tests { let permutation = FftPlan::new(polynomial.len()); let mut fft = fft(&polynomial, &permutation); fft.iter_mut().for_each(|num| *num *= *num); - let ifft = ifftr(&fft, &permutation); + let ifft = irfft(&fft, &permutation) + .into_iter() + .map(|i| i.re()) + .collect::>(); let expected = [1.0, 2.0, 1.0, 4.0, 4.0, 0.0, 4.0, 0.0, 0.0]; for (x, y) in ifft.iter().zip(expected.iter()) { assert!(almost_equal(*x, *y, EPSILON)); @@ -203,7 +278,10 @@ mod tests { let permutation = FftPlan::new(polynomial.len()); let mut fft = fft(&polynomial, &permutation); fft.iter_mut().for_each(|num| *num *= *num); - let ifft = ifftr(&fft, &permutation); + let ifft = irfft(&fft, &permutation) + .into_iter() + .map(|i| i.re()) + .collect::>(); let expected = (0..((n << 1) - 1)).map(|i| std::cmp::min(i + 1, (n << 1) - 1 - i) as f64); for (&x, y) in ifft.iter().zip(expected) { assert!(almost_equal(x, y, EPSILON)); diff --git a/core/src/ops/fft/modes.rs b/core/src/ops/fft/modes.rs index 270db8e4..b460bb88 100644 --- a/core/src/ops/fft/modes.rs +++ b/core/src/ops/fft/modes.rs @@ -56,3 +56,31 @@ impl From for usize { direction as usize } } + +#[derive( + Clone, + Copy, + Debug, + Default, + Deserialize, + Display, + EnumCount, + EnumIs, + EnumIter, + EnumString, + EnumVariantNames, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[repr(usize)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum FftMode { + Real, + #[default] + Standard, +} diff --git a/core/src/params/mod.rs b/core/src/params/mod.rs index d2c2d211..d77a1a0e 100644 --- a/core/src/params/mod.rs +++ b/core/src/params/mod.rs @@ -6,13 +6,15 @@ //! //! ## Overview //! -pub use self::{group::*, iter::*, kinds::*, param::*, store::*}; +pub use self::{iter::*, kinds::*, param::*, store::*, variable::*}; -pub(crate) mod group; pub(crate) mod iter; pub(crate) mod kinds; pub(crate) mod param; pub(crate) mod store; +pub(crate) mod variable; + +pub mod group; use ndarray::prelude::{Array, Dimension, Ix2}; use num::Float; diff --git a/core/src/params/variable.rs b/core/src/params/variable.rs new file mode 100644 index 00000000..a2394f90 --- /dev/null +++ b/core/src/params/variable.rs @@ -0,0 +1,16 @@ +/* + Appellation: variable + Contrib: FL03 +*/ +//! # Variables +//! +//! ## Overview +//! Variables extend the functionality of the 'Parameter' by enabling mutability. +//! + +pub struct Variable; + +pub enum P { + Param, + Variable(Box), +} diff --git a/core/src/utils.rs b/core/src/utils.rs index e1f3b345..c3860319 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -37,6 +37,21 @@ where assert!(err <= tol, "Error: {:?}", err); } +pub fn assert_ok(res: Result) -> T +where + E: std::fmt::Debug, +{ + assert!(res.is_ok(), "Error: {:?}", res.err()); + res.unwrap() +} + +pub fn almost_equal(a: T, b: T, epsilon: T) -> bool +where + T: PartialOrd + Signed, +{ + (b - a).abs() < epsilon +} + /// Creates an n-dimensional array from an iterator of n dimensional arrays. pub fn concat_iter(axis: usize, iter: impl IntoIterator>) -> Array where diff --git a/ml/s4/src/cmp/kernel.rs b/ml/s4/src/cmp/kernel.rs index 875fe7c4..dac60cbb 100644 --- a/ml/s4/src/cmp/kernel.rs +++ b/ml/s4/src/cmp/kernel.rs @@ -4,45 +4,13 @@ */ use crate::core::ops::fft::*; use crate::core::prelude::Conjugate; +use crate::params::DPLRParams; use crate::prelude::cauchy; use ndarray::prelude::{Array, Array1}; use ndarray::ScalarOperand; use ndarray_linalg::Scalar; use num::complex::{Complex, ComplexFloat}; use num::traits::{Float, FloatConst, NumOps}; -use rustfft::FftPlanner; -use std::ops::Neg; - -pub struct DPLRParams { - pub lambda: Array1, - pub p: Array1, - pub q: Array1, - pub b: Array1, - pub c: Array1, -} - -impl DPLRParams { - pub fn new(lambda: Array1, p: Array1, q: Array1, b: Array1, c: Array1) -> Self { - Self { lambda, p, q, b, c } - } -} - -// impl DPLRParams -// where -// T: ComplexFloat, -// ::Real: NumOps + NumOps::Real>, Complex<::Real>>, -// Complex<::Real>: NumOps + NumOps<::Real, Complex<::Real>> -// { -// pub fn kernel(&self, step: T, l: usize) -> Array1<::Real> { -// let lt = T::from(l).unwrap(); -// let omega_l = { -// let f = | i: usize | -> Complex<::Real> { -// Complex::<::Real>::i().neg() * ::Real::from(i).unwrap() * ::Real::PI() / lt -// }; -// Array::from_iter((0..l).map(f)) -// }; -// } -// } pub fn omega_l(l: usize) -> Array1<::Complex> where @@ -62,6 +30,43 @@ where Array::from_iter((0..l).map(f)) } +pub struct Omega +where + T: Scalar, +{ + omega: Array1<::Complex>, +} + +impl Omega +where + T: Scalar>, + ::Real: FloatConst + NumOps<::Complex, ::Complex>, + ::Complex: + ComplexFloat::Real> + NumOps<::Real> + ScalarOperand, +{ + pub fn new(l: usize) -> Self { + let f = |i: usize| -> ::Complex { + let im = T::PI().mul_complex(Complex::i() * T::from(2).unwrap()); // .neg() + T::from(i) + .unwrap() + .div_real(T::from(l).unwrap()) + .mul_complex(im) + .exp() + }; + let omega = Array::from_iter((0..l).map(f)); + Self { omega } + } +} + +impl Omega +where + T: Scalar, +{ + pub fn omega(&self) -> &Array1<::Complex> { + &self.omega + } +} + pub fn kernel_dplr( dplr: &DPLRParams<::Complex>, step: ::Real, @@ -96,8 +101,8 @@ where let buffer = at_roots.into_raw_vec(); let permute = FftPlan::new(l); - let res = ifftr(buffer.as_slice(), &permute); - Array::from_vec(res) + let res = irfft(buffer.as_slice(), &permute); + Array::from_vec(res).mapv(|i| i.re()) } pub struct Kernel { diff --git a/ml/s4/src/ops/convolve.rs b/ml/s4/src/ops/convolve.rs index 7e32d57f..2fdf7b28 100644 --- a/ml/s4/src/ops/convolve.rs +++ b/ml/s4/src/ops/convolve.rs @@ -2,11 +2,13 @@ Appellation: convolve Contrib: FL03 */ +use crate::core::ops::fft::{fft, ifftr, FftPlan}; use crate::core::prelude::Power; use ndarray::linalg::Dot; use ndarray::prelude::{Array, Array1, Array2}; use ndarray::ScalarOperand; -use num::Num; +use num::complex::{Complex, ComplexFloat}; +use num::traits::{Float, FloatConst, Num, NumOps}; /// Generates a large convolution kernal pub fn k_conv(a: &Array2, b: &Array2, c: &Array2, l: usize) -> Array1 @@ -23,4 +25,53 @@ where Array::from_vec(store) } +pub fn casual_convolution(u: &Array1, k: &Array1) -> Array1 +where + T: Float + FloatConst + NumOps, Complex>, + Complex: ComplexFloat, +{ + assert!(u.shape()[0] == k.shape()[0]); + let l = u.shape()[0]; + let plan = FftPlan::new(l); + let ud = fft(u.clone().into_raw_vec(), plan.clone()); + let kd = fft(k.clone().into_raw_vec(), plan.clone()); + + let ud = Array::from_vec(ud); + let kd = Array::from_vec(kd); + + let res = ud * kd; + let res = ifftr(res.into_raw_vec().as_slice(), &plan); + Array::from_vec(res) +} + pub struct Filter {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::prelude::arange; + use crate::core::ops::fft::*; + + use lazy_static::lazy_static; + use ndarray::prelude::*; + + const _FEATURES: usize = 4; + const SAMPLES: usize = 8; + + lazy_static! { + static ref EXP: Array1 = + array![-7.10542736e-15, 0.0, 1.0, 4.0, 10.0, 20.0, 35.0, 56.0]; + } + + // #[ignore] + #[test] + fn test_casual_convolution() { + let u = arange(0.0, SAMPLES as f64, 1.0); + let plan = FftPlan::new(SAMPLES); + println!("{:?}", rfft(u.clone().into_raw_vec(), plan)); + let k = arange(0.0, SAMPLES as f64, 1.0); + let res = casual_convolution(&u, &k); + println!("{:?}", res); + assert_eq!(res, *EXP); + } +} diff --git a/ml/s4/src/ops/discretize.rs b/ml/s4/src/ops/discretize.rs index 1485b4d8..cfba22c2 100644 --- a/ml/s4/src/ops/discretize.rs +++ b/ml/s4/src/ops/discretize.rs @@ -7,30 +7,9 @@ use crate::core::prelude::{Conjugate, Power}; use ndarray::{Array, Array1, Array2, Axis, ScalarOperand}; use ndarray_linalg::{Inverse, Lapack, Scalar}; use num::complex::ComplexFloat; -use num::traits::{Float, NumOps}; +use num::traits::{Float, Num, NumOps}; -pub fn discretize( - a: &Array2, - b: &Array2, - c: &Array2, - step: T, -) -> anyhow::Result> -where - T: Lapack + Scalar + ScalarOperand, -{ - let (n, ..) = a.dim(); - let ss = step / T::from(2).unwrap(); // half step - let eye = Array2::::eye(n); - - let be = (&eye - a * ss).inv().expect("Could not invert matrix"); - - let ab = be.dot(&(&eye + a * ss)); - let bb = (be * ss).dot(b); - - Ok((ab, bb, c.clone()).into()) -} - -pub fn discrete( +pub fn discretize( a: &Array2, b: &Array2, c: &Array2, @@ -41,12 +20,12 @@ where T: ComplexFloat + Lapack + NumOps, { let (n, ..) = a.dim(); - let ss = step / S::from(2).unwrap(); // half step + let hs = step / S::from(2).unwrap(); // half step let eye = Array2::::eye(n); - let bl = (&eye - a * ss).inv()?; + let bl = (&eye - a * hs).inv()?; - let ab = bl.dot(&(&eye + a * ss)); + let ab = bl.dot(&(&eye + a * hs)); let bb = (bl * step).dot(b); Ok((ab, bb, c.clone()).into()) @@ -69,7 +48,7 @@ where // create an identity matrix; (n, n) let eye = Array2::::eye(n); // compute the step size - let ss = T::from(2).unwrap() / step; + let hs = T::from(2).unwrap() / step; // turn the parameters into two-dimensional matricies let b2 = b.clone().insert_axis(Axis(1)); @@ -79,12 +58,12 @@ where // compute the conjugate transpose of q let qct = q.clone().conj().t().to_owned().insert_axis(Axis(0)); // create a diagonal matrix D from the scaled eigenvalues: Dim(n, n) :: 1 / (step_size - value) - let d = Array::from_diag(&lambda.mapv(|i| (ss - i).recip())); + let d = Array::from_diag(&lambda.mapv(|i| (hs - i).recip())); // create a diagonal matrix from the eigenvalues let a = Array::from_diag(&lambda) - &p2.dot(&q.clone().insert_axis(Axis(1)).conj().t()); // compute A0 - let a0 = &eye * ss + &a; + let a0 = &eye * hs + &a; // compute A1 let a1 = { let tmp = qct.dot(&d.dot(&p2)).mapv(|i| (T::one() + i).recip()); @@ -95,7 +74,7 @@ where // compute b-bar let bb = a1.dot(&b2) * T::from(2).unwrap(); // compute c-bar - let cb = c2.dot(&(&eye - ab.clone().pow(l)).inv()?.conj()); + let cb = c2.dot(&(&eye - &ab.pow(l)).inv()?.conj()); // return the discretized system Ok((ab, bb, cb.conj()).into()) } @@ -123,20 +102,21 @@ impl Discrete { pub fn from_features(features: usize) -> Self where - T: Float, + T: Default, { - let a = Array2::::zeros((features, features)); - let b = Array2::::zeros((features, 1)); - let c = Array2::::zeros((1, features)); + let a = Array2::::default((features, features)); + let b = Array2::::default((features, 1)); + let c = Array2::::default((1, features)); Self::new(a, b, c) } } -impl Discrete -where - T: Lapack + Scalar + ScalarOperand, -{ - pub fn discretize(&self, step: T) -> anyhow::Result { +impl Discrete { + pub fn discretize(&self, step: S) -> anyhow::Result + where + S: Scalar + ScalarOperand + NumOps, + T: ComplexFloat + Lapack + NumOps, + { discretize(&self.a, &self.b, &self.c, step) } } diff --git a/ml/s4/src/ops/mod.rs b/ml/s4/src/ops/mod.rs index 6da71a03..bf71ae94 100644 --- a/ml/s4/src/ops/mod.rs +++ b/ml/s4/src/ops/mod.rs @@ -15,8 +15,9 @@ mod tests { use crate::core::prelude::{assert_atol, randc_normal}; use num::complex::ComplexFloat; - use crate::cmp::kernel::{kernel_dplr, DPLRParams}; + use crate::cmp::kernel::kernel_dplr; use crate::hippo::dplr::DPLR; + use crate::params::DPLRParams; const FEATURES: usize = 8; const RNGKEY: u64 = 1; diff --git a/ml/s4/src/params/dplr.rs b/ml/s4/src/params/dplr.rs new file mode 100644 index 00000000..3f8c8db2 --- /dev/null +++ b/ml/s4/src/params/dplr.rs @@ -0,0 +1,42 @@ +/* + Appellation: kernel + Contrib: FL03 +*/ +use ndarray::prelude::Array1; + +pub struct DPLRParams { + pub lambda: Array1, + pub p: Array1, + pub q: Array1, + pub b: Array1, + pub c: Array1, +} + +impl DPLRParams { + pub fn new(lambda: Array1, p: Array1, q: Array1, b: Array1, c: Array1) -> Self { + Self { lambda, p, q, b, c } + } +} + +// impl DPLRParams +// where +// T: ComplexFloat, +// ::Real: NumOps + NumOps::Real>, Complex<::Real>>, +// Complex<::Real>: NumOps + NumOps<::Real, Complex<::Real>> +// { +// pub fn kernel(&self, step: T, l: usize) -> Array1<::Real> { +// let lt = T::from(l).unwrap(); +// let omega_l = { +// let f = | i: usize | -> Complex<::Real> { +// Complex::<::Real>::i().neg() * ::Real::from(i).unwrap() * ::Real::PI() / lt +// }; +// Array::from_iter((0..l).map(f)) +// }; +// } +// } + +impl From<(Array1, Array1, Array1, Array1, Array1)> for DPLRParams { + fn from((lambda, p, q, b, c): (Array1, Array1, Array1, Array1, Array1)) -> Self { + Self::new(lambda, p, q, b, c) + } +} diff --git a/ml/s4/src/params/kinds.rs b/ml/s4/src/params/kinds.rs index 490e71d8..7b4820b1 100644 --- a/ml/s4/src/params/kinds.rs +++ b/ml/s4/src/params/kinds.rs @@ -86,7 +86,7 @@ impl From for SSMParams { #[repr(usize)] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] -pub enum DPLRParams { +pub enum DPLRParam { #[default] Lambda = 0, P = 1, diff --git a/ml/s4/src/params/mod.rs b/ml/s4/src/params/mod.rs index 3cda0748..13ed430d 100644 --- a/ml/s4/src/params/mod.rs +++ b/ml/s4/src/params/mod.rs @@ -1,9 +1,10 @@ /* - Appellation: store + Appellation: params Contrib: FL03 */ -pub use self::{kinds::*, store::*}; +pub use self::{dplr::*, kinds::*, store::*}; +pub(crate) mod dplr; pub(crate) mod kinds; pub(crate) mod store; diff --git a/ml/s4/src/params/store.rs b/ml/s4/src/params/store.rs index 8ba3e78c..5678191b 100644 --- a/ml/s4/src/params/store.rs +++ b/ml/s4/src/params/store.rs @@ -27,21 +27,6 @@ impl SSMStore where T: Clone + Num, { - pub fn new(a: Array2, b: Array2, c: Array2, d: Array2) -> Self { - Self { a, b, c, d } - } - - pub fn from_features(features: usize) -> Self - where - T: Default, - { - let a = Array2::::default((features, features)); - let b = Array2::::default((features, 1)); - let c = Array2::::default((1, features)); - let d = Array2::::default((1, 1)); - Self::new(a, b, c, d) - } - pub fn ones(features: usize) -> Self { let a = Array2::::ones((features, features)); let b = Array2::::ones((features, 1)); @@ -57,6 +42,23 @@ where let d = Array2::::zeros((1, 1)); Self::new(a, b, c, d) } +} + +impl SSMStore { + pub fn new(a: Array2, b: Array2, c: Array2, d: Array2) -> Self { + Self { a, b, c, d } + } + + pub fn from_features(features: usize) -> Self + where + T: Default, + { + let a = Array2::::default((features, features)); + let b = Array2::::default((features, 1)); + let c = Array2::::default((1, features)); + let d = Array2::::default((1, 1)); + Self::new(a, b, c, d) + } pub fn a(&self) -> &Array2 { &self.a @@ -132,10 +134,7 @@ where } } -impl ops::Index for SSMStore -where - T: Float, -{ +impl ops::Index for SSMStore { type Output = Array2; fn index(&self, index: SSMParams) -> &Self::Output { @@ -149,10 +148,7 @@ where } } -impl ops::IndexMut for SSMStore -where - T: Float, -{ +impl ops::IndexMut for SSMStore { fn index_mut(&mut self, index: SSMParams) -> &mut Self::Output { use SSMParams::*; match index { @@ -164,37 +160,25 @@ where } } -impl From> for (Array2, Array2, Array2, Array2) -where - T: Float, -{ +impl From> for (Array2, Array2, Array2, Array2) { fn from(store: SSMStore) -> Self { (store.a, store.b, store.c, store.d) } } -impl<'a, T> From<&'a SSMStore> for (&'a Array2, &'a Array2, &'a Array2, &'a Array2) -where - T: Float, -{ +impl<'a, T> From<&'a SSMStore> for (&'a Array2, &'a Array2, &'a Array2, &'a Array2) { fn from(store: &'a SSMStore) -> Self { (&store.a, &store.b, &store.c, &store.d) } } -impl From<(Array2, Array2, Array2, Array2)> for SSMStore -where - T: Float, -{ +impl From<(Array2, Array2, Array2, Array2)> for SSMStore { fn from((a, b, c, d): (Array2, Array2, Array2, Array2)) -> Self { Self::new(a, b, c, d) } } -impl From> for HashMap> -where - T: Float, -{ +impl From> for HashMap> { fn from(store: SSMStore) -> Self { HashMap::from_iter(store.into_iter()) } @@ -202,7 +186,7 @@ where impl FromIterator<(SSMParams, Array2)> for SSMStore where - T: Default + Float, + T: Clone + Default, { fn from_iter)>>(iter: I) -> Self { let tmp = HashMap::>::from_iter(iter); @@ -230,10 +214,7 @@ where } } -impl IntoIterator for SSMStore -where - T: Float, -{ +impl IntoIterator for SSMStore { type Item = (SSMParams, Array2); type IntoIter = std::collections::hash_map::IntoIter>; diff --git a/ml/s4/src/ssm/mod.rs b/ml/s4/src/ssm/mod.rs index 7a2dec3b..c94a7353 100644 --- a/ml/s4/src/ssm/mod.rs +++ b/ml/s4/src/ssm/mod.rs @@ -25,7 +25,7 @@ mod tests { let step = 0.001; let config = SSMConfig::new(true, 9, 2); - let model = SSM::::create(config).setup(); - assert!(model.discretize(step).is_ok()); + // let model = SSM::::create(config).setup(); + // assert!(model.discretize(step).is_ok()); } } diff --git a/ml/s4/src/ssm/model.rs b/ml/s4/src/ssm/model.rs index 496a3a96..84108292 100644 --- a/ml/s4/src/ssm/model.rs +++ b/ml/s4/src/ssm/model.rs @@ -8,15 +8,14 @@ use crate::ops::Discrete; use crate::params::{SSMParams::*, SSMStore}; use crate::prelude::{discretize, k_conv}; use ndarray::prelude::{Array1, Array2, Axis, NdFloat}; +use ndarray::ScalarOperand; use ndarray_conv::{Conv2DFftExt, PaddingMode, PaddingSize}; use ndarray_linalg::{Lapack, Scalar}; -use num::Float; +use num::complex::ComplexFloat; +use num::traits::{Float, FloatConst, Num, NumOps}; use rustfft::FftNum; -pub struct SSM -where - T: Float, -{ +pub struct SSM { cache: Array1, config: SSMConfig, kernel: Array1, @@ -24,18 +23,15 @@ where ssm: Discrete, } -impl SSM -where - T: Float, -{ +impl SSM { pub fn create(config: SSMConfig) -> Self where - T: Default, + T: Clone + Default, { let features = config.features(); - let cache = Array1::::zeros(features); - let kernel = Array1::::zeros(features); + let cache = Array1::::default(features); + let kernel = Array1::::default(features); let params = SSMStore::from_features(features); Self { cache, @@ -54,6 +50,15 @@ where &mut self.config } + pub fn discretize(&self, step: S) -> anyhow::Result> + where + S: Scalar + ScalarOperand + NumOps, + T: ComplexFloat + Lapack + NumOps, + { + let discrete = discretize(&self.params[A], &self.params[B], &self.params[C], step)?; + Ok(discrete.into()) + } + pub fn kernel(&self) -> &Array1 { &self.kernel } @@ -71,66 +76,62 @@ where } } -impl SSM -where - T: Lapack + NdFloat + Scalar, -{ - pub fn setup(mut self) -> Self { - self.kernel = self.gen_filter(); - - self.ssm = self.discretize(self.config().step_size()).expect(""); - self - } - - pub fn scan( - &self, - u: &Array2, - x0: &Array1, - ) -> Result, ndarray_linalg::error::LinalgError> { - self.params.scan(u, x0) - } - - pub fn conv(&self, u: &Array2) -> anyhow::Result> - where - T: FftNum, - { - let mode = PaddingMode::<2, T>::Const(T::zero()); - let size = PaddingSize::Full; - if let Some(res) = u.conv_2d_fft(&self.kernel.clone().insert_axis(Axis(1)), size, mode) { - Ok(res) - } else { - Err(anyhow::anyhow!("convolution failed")) - } - } - - pub fn discretize(&self, step: T) -> anyhow::Result> { - let discrete = discretize(&self.params[A], &self.params[B], &self.params[C], step)?; - Ok(discrete.into()) - } - - pub fn gen_filter(&self) -> Array1 { - k_conv( - &self.params[A], - &self.params[B], - &self.params[C], - self.config().samples(), - ) - } -} - -impl Forward> for SSM -where - T: FftNum + Lapack + NdFloat + Scalar, -{ - type Output = anyhow::Result>; - - fn forward(&self, args: &Array2) -> Self::Output { - let res = if !self.config().decode() { - self.conv(args)? - } else { - self.scan(args, &self.cache)? - }; - let pred = res + args * &self.params[D]; - Ok(pred) - } -} +// impl SSM +// where +// T: ComplexFloat + Lapack + NumOps<::Real> + Scalar, +// ::Real: ScalarOperand + NumOps + NumOps, +// { +// pub fn setup(mut self) -> Self { +// self.kernel = self.gen_filter(); + +// self.ssm = self.discretize(self.config().step_size()).expect(""); +// self +// } + +// pub fn scan( +// &self, +// u: &Array2, +// x0: &Array1, +// ) -> Result, ndarray_linalg::error::LinalgError> { +// self.params.scan(u, x0) +// } + +// pub fn conv(&self, u: &Array2) -> anyhow::Result> +// where +// T: FftNum, +// { +// let mode = PaddingMode::<2, T>::Const(T::zero()); +// let size = PaddingSize::Full; +// if let Some(res) = u.conv_2d_fft(&self.kernel.clone().insert_axis(Axis(1)), size, mode) { +// Ok(res) +// } else { +// Err(anyhow::anyhow!("convolution failed")) +// } +// } + +// pub fn gen_filter(&self) -> Array1 { +// k_conv( +// &self.params[A], +// &self.params[B], +// &self.params[C], +// self.config().samples(), +// ) +// } +// } + +// impl Forward> for SSM +// where +// T: FftNum + Lapack + NdFloat + Scalar, +// { +// type Output = anyhow::Result>; + +// fn forward(&self, args: &Array2) -> Self::Output { +// let res = if !self.config().decode() { +// self.conv(args)? +// } else { +// self.scan(args, &self.cache)? +// }; +// let pred = res + args * &self.params[D]; +// Ok(pred) +// } +// } diff --git a/ml/s4/tests/dplr.rs b/ml/s4/tests/dplr.rs index 4a2d8ae4..f83a0fe1 100644 --- a/ml/s4/tests/dplr.rs +++ b/ml/s4/tests/dplr.rs @@ -5,27 +5,37 @@ extern crate concision_s4; use concision_core as core; use concision_s4 as s4; +use lazy_static::lazy_static; use ndarray::prelude::*; use ndarray_linalg::flatten; -use num::complex::ComplexFloat; +use num::complex::{Complex, ComplexFloat}; -use core::prelude::{AsComplex, Conjugate, Power}; -use s4::cmp::kernel::{kernel_dplr, DPLRParams}; +use core::prelude::{seeded_uniform, AsComplex, Conjugate, Power}; +use s4::cmp::kernel::kernel_dplr; use s4::hippo::dplr::DPLR; -use s4::ops::{discrete, k_conv}; +use s4::ops::{discretize, k_conv}; +use s4::params::DPLRParams; +const FEATURES: usize = 4; const RNGKEY: u64 = 1; +const SAMPLES: usize = 16; + +lazy_static! { + static ref SEEDED_C: Array2 = seeded_uniform(RNGKEY, 0.0, 1.0, (1, FEATURES)); + static ref SAMPLE_C: Array2 = array![[0.02185547, 0.20907068, 0.23742378, 0.3723395]]; + static ref SAMPLE_IM: Array2> = SAMPLE_C.clone().mapv(AsComplex::as_re); +} #[test] // #[ignore = "TODO: fix this test"] fn test_gen_dplr() { let (features, samples) = (4, 16); - let eye = Array2::::eye(features); + let eye = Array2::::eye(FEATURES); - let step = (samples as f64).recip(); + let step = (SAMPLES as f64).recip(); - let dplr = DPLR::::new(features); + let dplr = DPLR::::new(FEATURES); let (lambda, p, b, _v) = dplr.into(); println!("{:?}", &p); @@ -49,20 +59,20 @@ fn test_gen_dplr() { // TODO: figure out why several of the signs are wrong let discrete = { - let tmp = discrete(&a, &b2, &c, step); + let tmp = discretize(&a, &b2, &c, step); assert!(tmp.is_ok(), "discretize failed: {:?}", tmp.err().unwrap()); tmp.unwrap() }; let (ab, bb, cb) = discrete.into(); // - let ak = k_conv(&ab, &bb, &cb.conj(), samples); + let ak = k_conv(&ab, &bb, &cb.conj(), SAMPLES); // - let cc = (&eye - ab.pow(samples)).conj().t().dot(&flatten(cb)); + let cc = (&eye - ab.pow(SAMPLES)).conj().t().dot(&flatten(cb)); // let params = DPLRParams::new(lambda, p.clone(), p.clone(), b.clone(), cc); // - let kernal = kernel_dplr::(¶ms, step, samples); + let kernal = kernel_dplr::(¶ms, step, SAMPLES); println!("Kernal: {:?}", kernal.shape()); let a_real = ak.mapv(|i| i.re()); diff --git a/ml/transformers/src/attention/multi/mod.rs b/ml/transformers/src/attention/multi/mod.rs index 420df30a..cf68c00e 100644 --- a/ml/transformers/src/attention/multi/mod.rs +++ b/ml/transformers/src/attention/multi/mod.rs @@ -89,7 +89,7 @@ mod tests { let (heads, seq, model) = (8, 10, 512); let data = Array2::::zeros((seq, model)); - let mask = Mask::::masked(seq).into(); + let mask = Mask::::uniform(seq).into(); let attention = MultiHeadAttention::new(heads, model); let score = attention .attention(&data, &mask) diff --git a/ml/transformers/src/codec/encode/mod.rs b/ml/transformers/src/codec/encode/mod.rs index 244d4f07..49e8003f 100644 --- a/ml/transformers/src/codec/encode/mod.rs +++ b/ml/transformers/src/codec/encode/mod.rs @@ -23,7 +23,7 @@ mod tests { fn test_encoder() { let (heads, seq, model) = (8, 10, 512); let _data = Array2::::zeros((seq, model)); - let _mask = Mask::::masked(seq); + let _mask = Mask::::uniform(seq); let params = EncoderParams::new(heads, model); let encoder = Encoder::new(params); From 0e4e0ad9c50fc77129c746879b693bddbf17946f Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 28 Jan 2024 11:59:55 -0600 Subject: [PATCH 114/118] update Signed-off-by: FL03 --- Cargo.toml | 4 +- core/Cargo.toml | 1 + core/src/errors/kinds.rs | 6 +- core/src/ops/fft/algorithms/dft.rs | 9 + core/src/ops/fft/algorithms/mod.rs | 9 + core/src/ops/fft/mod.rs | 275 ++++++++++++-------- core/src/ops/fft/modes.rs | 12 +- core/src/ops/fft/plan.rs | 26 +- core/src/params/kinds.rs | 5 +- core/src/specs/arrays.rs | 24 ++ core/src/specs/math/mod.rs | 19 +- core/src/specs/math/numerical.rs | 5 +- core/src/specs/math/ops.rs | 14 +- core/src/states/weighted.rs | 6 +- core/src/utils.rs | 236 ++++++++++------- data/src/flows/direction.rs | 4 +- data/src/misc/dtype.rs | 16 +- data/src/tensors/mode.rs | 4 +- ml/linear/src/cmp/params/kinds.rs | 5 +- ml/neural/src/errors/error.rs | 14 +- ml/neural/src/func/loss/kinds.rs | 5 +- ml/neural/src/func/prop/modes.rs | 4 +- ml/neural/src/layers/cmp/kinds.rs | 4 +- ml/neural/src/models/modes.rs | 4 +- ml/neural/src/nn/kinds.rs | 10 +- ml/neural/src/params/shapes.rs | 6 +- ml/s4/Cargo.toml | 1 + ml/s4/src/cmp/kernel.rs | 35 +-- ml/s4/src/hippo/kinds.rs | 6 +- ml/s4/src/ops/convolve.rs | 57 ++-- ml/s4/src/ops/discretize.rs | 2 +- ml/s4/src/params/kinds.rs | 6 +- ml/s4/src/utils.rs | 38 +++ ml/s4/tests/dplr.rs | 1 - ml/transformers/src/attention/params/qkv.rs | 5 +- 35 files changed, 551 insertions(+), 327 deletions(-) create mode 100644 core/src/ops/fft/algorithms/dft.rs create mode 100644 core/src/ops/fft/algorithms/mod.rs diff --git a/Cargo.toml b/Cargo.toml index 13eae1e5..8738bd12 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,8 +17,6 @@ anyhow = "1" approx = "0.5" itertools = { features = [], version = "0.12" } lazy_static = "1" -# ndarray = { features = ["serde-1"], version = "0.15" } -# ndarray-linalg = { features = [], version = "0.16" } ndarray-rand = { features = [], version = "0.14" } ndarray-stats = { features = [], version = "0.5" } num = { features = ["serde"], version = "0.4" } @@ -26,7 +24,7 @@ num = { features = ["serde"], version = "0.4" } serde = { features = ["derive"], version = "1" } serde_json = "1" smart-default = "0.7" -strum = { features = ["derive"], version = "0.25" } +strum = { features = ["derive"], version = "0.26" } [workspace] default-members = [ diff --git a/core/Cargo.toml b/core/Cargo.toml index f792da56..f170e3b9 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -39,6 +39,7 @@ smart-default.workspace = true strum.workspace = true [dev-dependencies] +lazy_static.workspace = true [package.metadata.docs.rs] all-features = true diff --git a/core/src/errors/kinds.rs b/core/src/errors/kinds.rs index 8bfbe4a5..442e0a87 100644 --- a/core/src/errors/kinds.rs +++ b/core/src/errors/kinds.rs @@ -4,7 +4,7 @@ */ use serde::{Deserialize, Serialize}; use smart_default::SmartDefault; -use strum::{Display, EnumCount, EnumIs, EnumIter, EnumVariantNames}; +use strum::{Display, EnumCount, EnumIs, EnumIter, VariantNames}; #[derive( Clone, @@ -14,7 +14,6 @@ use strum::{Display, EnumCount, EnumIs, EnumIter, EnumVariantNames}; EnumCount, EnumIs, EnumIter, - EnumVariantNames, Eq, Hash, Ord, @@ -22,6 +21,7 @@ use strum::{Display, EnumCount, EnumIs, EnumIter, EnumVariantNames}; PartialOrd, Serialize, SmartDefault, + VariantNames, )] #[non_exhaustive] #[serde(rename_all = "lowercase")] @@ -53,7 +53,6 @@ pub enum Errors { EnumCount, EnumIs, EnumIter, - EnumVariantNames, Eq, Hash, Ord, @@ -61,6 +60,7 @@ pub enum Errors { PartialOrd, Serialize, SmartDefault, + VariantNames, )] #[non_exhaustive] #[serde(rename_all = "lowercase")] diff --git a/core/src/ops/fft/algorithms/dft.rs b/core/src/ops/fft/algorithms/dft.rs new file mode 100644 index 00000000..8046e515 --- /dev/null +++ b/core/src/ops/fft/algorithms/dft.rs @@ -0,0 +1,9 @@ +/* + Appellation: dft + Contrib: FL03 +*/ +//! # Discrete Fourier Transform +//! +//! + +pub struct Dft; \ No newline at end of file diff --git a/core/src/ops/fft/algorithms/mod.rs b/core/src/ops/fft/algorithms/mod.rs new file mode 100644 index 00000000..ddf471f3 --- /dev/null +++ b/core/src/ops/fft/algorithms/mod.rs @@ -0,0 +1,9 @@ + +pub use self::dft::*; + +pub(crate) mod dft; + +#[cfg(test)] +mod tests { + +} \ No newline at end of file diff --git a/core/src/ops/fft/mod.rs b/core/src/ops/fft/mod.rs index cf0324b0..1b1021c4 100644 --- a/core/src/ops/fft/mod.rs +++ b/core/src/ops/fft/mod.rs @@ -11,179 +11,184 @@ pub(crate) mod fft; pub(crate) mod modes; pub(crate) mod plan; -pub trait Fft { - fn fft(&self) -> Vec; - fn ifft(&self) -> Vec; +pub mod algorithms; + +pub trait Fft { + fn fft(&self) -> Vec; + fn ifft(&self) -> Vec; } + pub(crate) mod utils { use super::FftPlan; use crate::prelude::AsComplex; use num::complex::{Complex, ComplexFloat}; - use num::traits::{Float, FloatConst, NumOps}; + use num::traits::{Float, FloatConst, Num, NumAssignOps, NumCast, NumOps }; + + // pub(crate) fn rsize(n: usize) -> usize { + // (n / 2).floor() + 1 + // } pub(crate) fn fft_angle(n: usize) -> T where - T: Float + FloatConst, + T: FloatConst + NumCast + NumOps, { T::TAU() / T::from(n).unwrap() } - /// Computes the Fast Fourier Transform of a general signal. - pub fn fft(input: impl AsRef<[T]>, input_permutation: impl AsRef<[usize]>) -> Vec> + pub(crate) fn floor(lhs: T, rhs: T) -> T where T: Copy + Num { + (lhs - lhs % rhs) / rhs + } + + pub(crate) fn unfloor(lhs: T, rhs: T) -> T where T: Copy + Num { + (lhs * rhs) - lhs % rhs + } + + /// Computes the Fast Fourier Transform of a one-dimensional, complex-valued signal. + pub fn fft(input: impl AsRef<[S]>, permute: &FftPlan) -> Vec> where + S: ComplexFloat, T: Float + FloatConst, - Complex: ComplexFloat, + Complex: ComplexFloat + NumOps + NumOps, { + // let input = input.as_ref(); - + // let n = input.len(); - - let mut result = Vec::new(); - result.reserve_exact(n); - for position in input_permutation.as_ref() { - result.push(input[*position].as_re()); + // initialize the result vector + let mut result = Vec::with_capacity(n); + // store the input values in the result vector according to the permutation + for position in permute.clone().into_iter() { + let arg = input[position]; + result.push(Complex::new(arg.re(), arg.im())); } - let mut segment_length = 1_usize; - while segment_length < n { - segment_length <<= 1; - let angle = fft_angle::(segment_length); - let w_len = Complex::new(angle.cos(), angle.sin()); - for segment_start in (0..n).step_by(segment_length) { + let mut segment: usize = 1; + while segment < n { + segment <<= 1; + // compute the angle of the complex number + let angle = fft_angle::(segment); + // compute the radius of the complex number (length) + let radius = Complex::new(angle.cos(), angle.sin()); + // iterate over the signal in segments of length `segment` + for start in (0..n).step_by(segment) { let mut w = Complex::new(T::one(), T::zero()); - for position in segment_start..(segment_start + segment_length / 2) { + for position in start..(start + segment / 2) { let a = result[position]; - let b = result[position + segment_length / 2] * w; + let b = result[position + segment / 2] * w; result[position] = a + b; - result[position + segment_length / 2] = a - b; - w = w * w_len; + result[position + segment / 2] = a - b; + w = w * radius; } } } result } - /// Computes the Fast Fourier Transform of a real-valued signal. - /// TODO: Fix the function; real-valued fft only computes the positive frequency terms + /// Computes the Fast Fourier Transform of an one-dimensional, real-valued signal. + /// TODO: Optimize the function to avoid unnecessary computation. pub fn rfft(input: impl AsRef<[T]>, input_permutation: impl AsRef<[usize]>) -> Vec> where T: Float + FloatConst, - Complex: ComplexFloat, + Complex: ComplexFloat + NumAssignOps, { + // create a reference to the input let input = input.as_ref(); - + // fetch the length of the input let n = input.len(); - - let mut result = Vec::new(); - result.reserve_exact(n); + // compute the size of the result vector + let size = (n - (n % 2)) / 2 + 1; + // initialize the output vector + let mut store = Vec::with_capacity(size); + // store the input values in the result vector according to the permutation for position in input_permutation.as_ref() { - result.push(input[*position].as_re()); + store.push(input[*position].as_re()); } - let mut segment_length = 1_usize; - while segment_length < n { - segment_length <<= 1; - let angle = fft_angle::(segment_length); - let w_len = Complex::new(angle.cos(), angle.sin()); - for segment_start in (0..n).step_by(segment_length) { + let mut segment: usize = 1; + while segment < n { + segment <<= 1; + // compute the angle of the complex number + let angle = fft_angle::(segment); + // compute the radius of the complex number (length) + let radius = Complex::new(angle.cos(), angle.sin()); + for start in (0..n).step_by(segment) { let mut w = Complex::new(T::one(), T::zero()); - for position in segment_start..(segment_start + segment_length / 2) { - let a = result[position]; - let b = result[position + segment_length / 2] * w; - result[position] = a + b; - result[position + segment_length / 2] = a - b; - w = w * w_len; + for position in start..(start + segment / 2) { + let a = store[position]; + let b = store[position + segment / 2] * w; + store[position] = a + b; + store[position + segment / 2] = a - b; + w *= radius; } } } - result + store.iter().cloned().filter(|x| x.im() >= T::zero()).collect() } - /// Computes the Inverse Fast Fourier Transform of a signal. - pub fn ifft(input: &[Complex], input_permutation: &FftPlan) -> Vec> + /// Computes the Inverse Fast Fourier Transform of an one-dimensional, complex-valued signal. + pub fn ifft(input: &[S], input_permutation: &FftPlan) -> Vec> where + S: ComplexFloat, T: Float + FloatConst, - Complex: ComplexFloat, - { - let n = input.len(); - let mut result = Vec::new(); - result.reserve_exact(n); - for position in input_permutation.clone().into_iter() { - result.push(input[position]); - } - let mut segment_length = 1_usize; - while segment_length < n { - segment_length <<= 1; - let angle = fft_angle::(segment_length); - let w_len = Complex::new(T::cos(angle), T::sin(angle)); - for segment_start in (0..n).step_by(segment_length) { - let mut w = Complex::new(T::one(), T::zero()); - for position in segment_start..(segment_start + segment_length / 2) { - let a = result[position]; - let b = result[position + segment_length / 2] * w; - result[position] = a + b; - result[position + segment_length / 2] = a - b; - w = w * w_len; - } - } - } - let scale = T::from(n).unwrap().recip(); - result.iter().map(|x| x * scale).collect() - } - - pub fn irfft(input: &[Complex], input_permutation: &FftPlan) -> Vec - where - T: Float + FloatConst + NumOps, Complex>, - Complex: ComplexFloat, + Complex: ComplexFloat + NumOps + NumOps, { let n = input.len(); - let mut result = Vec::new(); - result.reserve_exact(n); + let mut result = Vec::with_capacity(n); for position in input_permutation.clone().into_iter() { - result.push(input[position]); + let arg = input[position]; + result.push(Complex::new(arg.re(), arg.im())); } - let mut segment_length = 1_usize; - while segment_length < n { - segment_length <<= 1; - let angle = fft_angle::(segment_length); - let w_len = Complex::new(T::cos(angle), T::sin(angle)); - for segment_start in (0..n).step_by(segment_length) { + let mut length: usize = 1; + while length < n { + length <<= 1; + let angle = fft_angle::(length).neg(); + let radius = Complex::new(T::cos(angle), T::sin(angle)); // w_len + for start in (0..n).step_by(length) { let mut w = Complex::new(T::one(), T::zero()); - for position in segment_start..(segment_start + segment_length / 2) { + for position in start..(start + length / 2) { let a = result[position]; - let b = result[position + segment_length / 2] * w; + let b = result[position + length / 2] * w; result[position] = a + b; - result[position + segment_length / 2] = a - b; - w = w * w_len; + result[position + length / 2] = a - b; + w = w * radius; } } } let scale = T::from(n).unwrap().recip(); - result.iter().map(|x| x.re() * scale).collect() + result.iter().map(|x| *x * scale).collect() } - /// Computes the Inverse Fast Fourier Transform of a real-valued signal. - pub fn ifftr(input: &[S], input_permutation: &FftPlan) -> Vec + /// Computes the Inverse Fast Fourier Transform of an one-dimensional, real-valued signal. + /// TODO: Fix the function; currently fails to compute the correct result + pub fn irfft(input: &[Complex], plan: &FftPlan) -> Vec where T: Float + FloatConst, - S: ComplexFloat + NumOps>, + Complex: ComplexFloat + NumAssignOps, { let n = input.len(); - let mut result = Vec::new(); - result.reserve_exact(n); - for position in input_permutation.clone().into_iter() { + let mut result = vec![Complex::new(T::zero(), T::zero()); n]; + + for position in plan.clone().into_iter() { result.push(input[position]); } - let mut segment_length = 1_usize; - while segment_length < n { - segment_length <<= 1; - let angle = fft_angle::(segment_length); - let w_len = Complex::new(T::cos(angle), T::sin(angle)); - for segment_start in (0..n).step_by(segment_length) { - let mut w = S::one(); - for position in segment_start..(segment_start + segment_length / 2) { + // for res in result.clone() { + // if res.im() > T::zero() { + // result.push(res.conj()); + // } + // } + // segment length + let mut segment: usize = 1; + while segment < n { + segment <<= 1; + // compute the angle of the complex number + let angle = fft_angle::(segment).neg(); + // compute the radius of the complex number (length) + let radius = Complex::new(T::cos(angle), T::sin(angle)); + for start in (0..n).step_by(segment) { + let mut w = Complex::new(T::one(), T::zero()); + for position in start..(start + segment / 2) { let a = result[position]; - let b = result[position + segment_length / 2] * w; + let b = result[position + segment / 2] * w; result[position] = a + b; - result[position + segment_length / 2] = a - b; - w = w * w_len; + result[position + segment / 2] = a - b; + w *= radius; } } } @@ -196,9 +201,11 @@ pub(crate) mod utils { mod tests { use super::*; use crate::prelude::almost_equal; - use num::complex::ComplexFloat; + use lazy_static::lazy_static; + use num::complex::{Complex, ComplexFloat}; + - pub(crate) fn fast_fourier_transform_input_permutation(length: usize) -> Vec { + pub(crate) fn fft_permutation(length: usize) -> Vec { let mut result = Vec::new(); result.reserve_exact(length); for i in 0..length { @@ -225,6 +232,16 @@ mod tests { const EPSILON: f64 = 1e-6; + lazy_static! { + static ref EXPECTED_RFFT: Vec> = vec![ + Complex { re: 28.0, im: 0.0 }, + Complex { re: -4.0, im: 0.0 }, + Complex { re: -4.0, im: 1.6568542494923806 }, + Complex { re: -4.0, im: 4.000000000000001 }, + Complex { re: -3.999999999999999, im: 9.656854249492381 } + ]; + } + #[test] fn test_plan() { let samples = 16; @@ -232,16 +249,44 @@ mod tests { let plan = FftPlan::new(samples); assert_eq!( plan.plan(), - fast_fourier_transform_input_permutation(16).as_slice() + fft_permutation(16).as_slice() ); } + #[test] + fn test_rfft() { + let polynomial = (0..8).map(|i| i as f64).collect::>(); + let plan = FftPlan::new(polynomial.len()); + println!("Function Values: {:?}", &polynomial); + println!("Plan: {:?}", &plan); + let fft = rfft(&polynomial, &plan); + let mut tmp = fft.iter().cloned().filter(|i| i.im() > 0.0).map(|i| i.conj()).collect::>(); + tmp.sort_by(|a, b| a.im().partial_cmp(&b.im()).unwrap()); + println!("FFT: {:?}", &tmp); + let mut res = fft.clone(); + res.sort_by(|a, b| a.re().partial_cmp(&b.re()).unwrap()); + res.sort_by(|a, b| a.im().partial_cmp(&b.im()).unwrap()); + println!("R: {:?}", &res); + res.extend(tmp); + assert!(fft.len() == EXPECTED_RFFT.len()); + for (x, y) in fft.iter().zip(EXPECTED_RFFT.iter()) { + assert!(almost_equal(x.re(), y.re(), EPSILON)); + assert!(almost_equal(x.im(), y.im(), EPSILON)); + } + // let plan = FftPlan::new(fft.len()); + let ifft = irfft(&res, &plan); + println!("Inverse: {:?}", &ifft); + for (x, y) in ifft.iter().zip(polynomial.iter()) { + assert!(almost_equal(*x, *y, EPSILON)); + } + } + #[test] fn small_polynomial_returns_self() { let polynomial = vec![1.0f64, 1.0, 0.0, 2.5]; let permutation = FftPlan::new(polynomial.len()); let fft = fft(&polynomial, &permutation); - let ifft = irfft(&fft, &permutation) + let ifft = ifft(&fft, &permutation) .into_iter() .map(|i| i.re()) .collect::>(); @@ -257,7 +302,7 @@ mod tests { let permutation = FftPlan::new(polynomial.len()); let mut fft = fft(&polynomial, &permutation); fft.iter_mut().for_each(|num| *num *= *num); - let ifft = irfft(&fft, &permutation) + let ifft = ifft(&fft, &permutation) .into_iter() .map(|i| i.re()) .collect::>(); diff --git a/core/src/ops/fft/modes.rs b/core/src/ops/fft/modes.rs index b460bb88..fb4a5f29 100644 --- a/core/src/ops/fft/modes.rs +++ b/core/src/ops/fft/modes.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use serde::{Deserialize, Serialize}; -use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; +use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, VariantArray, VariantNames,}; #[derive( Clone, @@ -16,13 +16,14 @@ use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantArray, + VariantNames )] #[repr(usize)] #[serde(rename_all = "lowercase")] @@ -68,19 +69,20 @@ impl From for usize { EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantArray, + VariantNames )] #[repr(usize)] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] pub enum FftMode { - Real, #[default] - Standard, + Complex, + Real, } diff --git a/core/src/ops/fft/plan.rs b/core/src/ops/fft/plan.rs index 2cbf9b4b..58a9d293 100644 --- a/core/src/ops/fft/plan.rs +++ b/core/src/ops/fft/plan.rs @@ -2,6 +2,7 @@ Appellation: plan Contrib: FL03 */ +use super::FftDirection; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] @@ -11,27 +12,26 @@ pub struct FftPlan { impl FftPlan { pub fn new(n: usize) -> Self { - let mut permute = Vec::new(); - permute.reserve_exact(n); - permute.extend(0..n); + let mut plan = Vec::with_capacity(n); + plan.extend(0..n); - let mut reverse = 0; - let mut position = 1; - while position < n { + let mut rev = 0; // reverse + let mut pos = 1; // position + while pos < n { let mut bit = n >> 1; - while bit & reverse != 0 { - reverse ^= bit; + while bit & rev != 0 { + rev ^= bit; bit >>= 1; } - reverse ^= bit; + rev ^= bit; // This is equivalent to adding 1 to a reversed number - if position < reverse { + if pos < rev { // Only swap each element once - permute.swap(position, reverse); + plan.swap(pos, rev); } - position += 1; + pos += 1; } - Self { plan: permute } + Self { plan } } pub fn plan(&self) -> &[usize] { diff --git a/core/src/params/kinds.rs b/core/src/params/kinds.rs index 6c8a7f84..d8750d24 100644 --- a/core/src/params/kinds.rs +++ b/core/src/params/kinds.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use serde::{Deserialize, Serialize}; -use strum::{EnumIs, EnumIter, EnumString, EnumVariantNames}; +use strum::{EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; pub trait ParamType: ToString { fn kind(&self) -> String; @@ -23,16 +23,17 @@ where Debug, Default, Deserialize, + EnumCount, EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames )] #[non_exhaustive] #[repr(usize)] diff --git a/core/src/specs/arrays.rs b/core/src/specs/arrays.rs index ec1e9229..76dc0e97 100644 --- a/core/src/specs/arrays.rs +++ b/core/src/specs/arrays.rs @@ -13,6 +13,30 @@ use num::traits::real::Real; use num::traits::{Float, Num, NumAssignOps, ToPrimitive}; use std::ops; +pub trait Pad { + fn pad(&self, pad: usize) -> Self; + + fn pad_with(&self, pad: usize, value: T) -> Self; +} + + +// impl Pad for Array +// where +// T: Clone + Num, +// D: Dimension, +// { +// fn pad(&self, pad: usize) -> Self { +// self.pad_with(pad, T::zero()) +// } + +// fn pad_with(&self, pad: usize, value: T) -> Self { +// let mut pad = vec![value; pad]; +// pad.extend_from_slice(self); +// pad.extend_from_slice(&vec![value; pad.len()]); +// Array::from_vec(pad) +// } +// } + pub trait Affine: Sized { type Error; diff --git a/core/src/specs/math/mod.rs b/core/src/specs/math/mod.rs index 2d0c72b9..87fecfe1 100644 --- a/core/src/specs/math/mod.rs +++ b/core/src/specs/math/mod.rs @@ -9,7 +9,7 @@ pub(crate) mod ops; pub(crate) mod scalar; use num::complex::Complex; -use num::traits::{Float, Num}; +use num::traits::{Float, Num,}; pub trait AsComplex: Sized { fn as_complex(self, real: bool) -> Complex; @@ -35,6 +35,23 @@ where } } +pub trait FloorDiv { + type Output; + + fn floor_div(self, rhs: Rhs) -> Self::Output; +} + +impl FloorDiv for T +where + T: Copy + Num, +{ + type Output = T; + + fn floor_div(self, rhs: Self) -> Self::Output { + crate::floor_div(self, rhs) + } +} + pub trait RoundTo { fn round_to(&self, places: usize) -> Self; } diff --git a/core/src/specs/math/numerical.rs b/core/src/specs/math/numerical.rs index 7e5b68c2..b41e57fe 100644 --- a/core/src/specs/math/numerical.rs +++ b/core/src/specs/math/numerical.rs @@ -3,8 +3,9 @@ Contrib: FL03 */ use num::complex::Complex; -use num::traits::{Num, NumAssignOps, NumOps, Signed}; +use num::traits::{Num, NumAssignOps, NumOps, One, Signed}; // use num::traits::real::Real; +use std::ops::Div; pub trait Algebraic: NumOps + Sized {} @@ -103,7 +104,7 @@ pub trait Reciprocal { impl Reciprocal for T where - T: Num + NumOps, + T: Div + One, { fn recip(self) -> Self { Self::one() / self diff --git a/core/src/specs/math/ops.rs b/core/src/specs/math/ops.rs index 672e8cac..5ed5b36d 100644 --- a/core/src/specs/math/ops.rs +++ b/core/src/specs/math/ops.rs @@ -52,25 +52,23 @@ where } } -pub trait Arithmetic +pub trait Arithmetic where - Self: ops::Add + ops::Div + ops::Mul + ops::Sub, + Self: ops::Add + ops::Div + ops::Mul + ops::Sub, { - type Output; } -impl Arithmetic for A +impl Arithmetic for A where A: ops::Add + ops::Div + ops::Mul + ops::Sub, { - type Output = T; } pub trait MatrixOps: - Arithmetic, Output = Array> + Sized + Arithmetic, Array> + Sized where A: Dimension, B: Dimension, @@ -83,7 +81,7 @@ where B: Dimension, D: Dimension, T: Arithmetic, - Self: Arithmetic, Output = Array>, + Self: Arithmetic, Array>, { } @@ -93,7 +91,7 @@ where B: Dimension, D: Dimension, T: Arithmetic, - Self: Arithmetic, Output = Array>, + Self: Arithmetic, Array>, { } diff --git a/core/src/states/weighted.rs b/core/src/states/weighted.rs index aee6a61c..a4d2a0c5 100644 --- a/core/src/states/weighted.rs +++ b/core/src/states/weighted.rs @@ -4,16 +4,17 @@ */ use serde::{Deserialize, Serialize}; use smart_default::SmartDefault; -use strum::{Display, EnumIs, EnumIter, EnumVariantNames}; +use strum::{Display, EnumCount, EnumIs, EnumIter, VariantNames}; #[derive( Clone, + Copy, Debug, Deserialize, Display, + EnumCount, EnumIs, EnumIter, - EnumVariantNames, Eq, Hash, Ord, @@ -21,6 +22,7 @@ use strum::{Display, EnumIs, EnumIter, EnumVariantNames}; PartialOrd, Serialize, SmartDefault, + VariantNames, )] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] diff --git a/core/src/utils.rs b/core/src/utils.rs index c3860319..c2a6084f 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -2,18 +2,32 @@ Appellation: utils Contrib: FL03 */ +pub use self::{arrays::*, assertions::*}; use ndarray::linalg::Dot; use ndarray::prelude::*; -use ndarray::{concatenate, IntoDimension, RemoveAxis, ScalarOperand, ShapeError}; +use ndarray::{IntoDimension, ShapeError}; use ndarray_rand::rand::rngs::StdRng; use ndarray_rand::rand::SeedableRng; use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use ndarray_rand::RandomExt; -use num::cast::AsPrimitive; use rand::distributions::uniform::{SampleUniform, Uniform}; -// use num::complex::{Complex, ComplexDistribution}; -use num::{Complex, Float, FromPrimitive, Num, NumCast, Signed, Zero}; +use num::complex::Complex; +use num::traits::{AsPrimitive, Float, Num, NumCast,}; + +pub fn pad(a: impl IntoIterator, pad: usize, value: Option) -> Vec +where + T: Clone + Default, +{ + let pad = vec![value.unwrap_or_default(); pad]; + let mut res = Vec::from_iter(a); + res.extend(pad); + res +} +/// +pub fn floor_div(numerator: T, denom: T) -> T where T: Copy + Num { + (numerator - (numerator % denom)) / denom +} pub fn arange(a: T, b: T, h: T) -> Array1 where @@ -28,43 +42,8 @@ where res } -pub fn assert_atol(a: &Array, b: &Array, tol: T) -where - D: Dimension, - T: FromPrimitive + PartialOrd + ScalarOperand + Signed + std::fmt::Debug, -{ - let err = (b - a).mapv(|i| i.abs()).mean().unwrap(); - assert!(err <= tol, "Error: {:?}", err); -} - -pub fn assert_ok(res: Result) -> T -where - E: std::fmt::Debug, -{ - assert!(res.is_ok(), "Error: {:?}", res.err()); - res.unwrap() -} -pub fn almost_equal(a: T, b: T, epsilon: T) -> bool -where - T: PartialOrd + Signed, -{ - (b - a).abs() < epsilon -} -/// Creates an n-dimensional array from an iterator of n dimensional arrays. -pub fn concat_iter(axis: usize, iter: impl IntoIterator>) -> Array -where - D: RemoveAxis, - T: Clone, -{ - let mut arr = iter.into_iter().collect::>(); - let mut out = arr.pop().unwrap(); - for i in arr { - out = concatenate!(Axis(axis), out, i); - } - out -} pub fn genspace(features: usize) -> Array1 { Array1::from_iter((0..features).map(|x| T::from(x).unwrap())) @@ -136,21 +115,6 @@ pub fn round_to(val: T, decimals: usize) -> T { (val * factor).round() / factor } -/// Creates a larger array from an iterator of smaller arrays. -pub fn stack_iter(iter: impl IntoIterator>) -> Array2 -where - T: Clone + Num, -{ - let mut iter = iter.into_iter(); - let first = iter.next().unwrap(); - let shape = [iter.size_hint().0 + 1, first.len()]; - let mut res = Array2::::zeros(shape); - res.slice_mut(s![0, ..]).assign(&first); - for (i, s) in iter.enumerate() { - res.slice_mut(s![i + 1, ..]).assign(&s); - } - res -} /// Creates a random array from a uniform distribution using a given key pub fn seeded_uniform( key: u64, @@ -200,53 +164,131 @@ where { Array::random(shape, StandardNormal) } -/// Returns the upper triangular portion of a matrix. -pub fn triu(a: &Array2) -> Array2 -where - T: Clone + Zero, -{ - let mut out = a.clone(); - for i in 0..a.shape()[0] { - for j in 0..i { - out[[i, j]] = T::zero(); - } + + + + +pub(crate) mod assertions { + use ndarray::prelude::{Array, Dimension}; + use ndarray::ScalarOperand; + use num::traits::{FromPrimitive, Signed}; + use std::fmt::Debug; + /// + pub fn assert_atol(a: &Array, b: &Array, tol: T) + where + D: Dimension, + T: Debug + FromPrimitive + PartialOrd + ScalarOperand + Signed, + { + let err = (b - a).mapv(|i| i.abs()).mean().unwrap(); + assert!(err <= tol, "Error: {:?}", err); } - out -} -/// Returns the lower triangular portion of a matrix. -pub fn tril(a: &Array2) -> Array2 -where - T: Clone + Zero, -{ - let mut out = a.clone(); - for i in 0..a.shape()[0] { - for j in i + 1..a.shape()[1] { - out[[i, j]] = T::zero(); - } + /// A function helper for testing that some result is ok + pub fn assert_ok(res: Result) -> T + where + E: Debug, + { + assert!(res.is_ok(), "Error: {:?}", res.err()); + res.unwrap() } - out -} - -pub fn hstack(iter: impl IntoIterator>) -> Array2 -where - T: Clone + Num, -{ - let iter = Vec::from_iter(iter); - let mut res = Array2::::zeros((iter.first().unwrap().len(), iter.len())); - for (i, s) in iter.iter().enumerate() { - res.slice_mut(s![.., i]).assign(s); + /// + pub fn assert_approx(a: T, b: T, epsilon: T) + where + T: Debug + PartialOrd + Signed, + { + let err = (b - a).abs(); + assert!(err < epsilon, "Error: {:?}", err) + } + /// + pub fn almost_equal(a: T, b: T, epsilon: T) -> bool + where + T: PartialOrd + Signed, + { + (b - a).abs() < epsilon } - res } -pub fn vstack(iter: impl IntoIterator>) -> Array2 -where - T: Clone + Num, -{ - let iter = Vec::from_iter(iter); - let mut res = Array2::::zeros((iter.len(), iter.first().unwrap().len())); - for (i, s) in iter.iter().enumerate() { - res.slice_mut(s![i, ..]).assign(s); +pub(crate) mod arrays { + use ndarray::prelude::{s, Array, Array1, Array2, Axis}; + use ndarray::{RemoveAxis, concatenate}; + use num::traits::{Num, Zero}; + /// Creates an n-dimensional array from an iterator of n dimensional arrays. + pub fn concat_iter(axis: usize, iter: impl IntoIterator>) -> Array + where + D: RemoveAxis, + T: Clone, + { + let mut arr = iter.into_iter().collect::>(); + let mut out = arr.pop().unwrap(); + for i in arr { + out = concatenate!(Axis(axis), out, i); + } + out + } + /// Creates a larger array from an iterator of smaller arrays. + pub fn stack_iter(iter: impl IntoIterator>) -> Array2 + where + T: Clone + Num, + { + let mut iter = iter.into_iter(); + let first = iter.next().unwrap(); + let shape = [iter.size_hint().0 + 1, first.len()]; + let mut res = Array2::::zeros(shape); + res.slice_mut(s![0, ..]).assign(&first); + for (i, s) in iter.enumerate() { + res.slice_mut(s![i + 1, ..]).assign(&s); + } + res + } + /// + pub fn hstack(iter: impl IntoIterator>) -> Array2 + where + T: Clone + Num, + { + let iter = Vec::from_iter(iter); + let mut res = Array2::::zeros((iter.first().unwrap().len(), iter.len())); + for (i, s) in iter.iter().enumerate() { + res.slice_mut(s![.., i]).assign(s); + } + res + } + /// Returns the lower triangular portion of a matrix. + pub fn tril(a: &Array2) -> Array2 + where + T: Clone + Zero, + { + let mut out = a.clone(); + for i in 0..a.shape()[0] { + for j in i + 1..a.shape()[1] { + out[[i, j]] = T::zero(); + } + } + out + } + /// Returns the upper triangular portion of a matrix. + pub fn triu(a: &Array2) -> Array2 + where + T: Clone + Zero, + { + let mut out = a.clone(); + for i in 0..a.shape()[0] { + for j in 0..i { + out[[i, j]] = T::zero(); + } + } + out + } + /// + pub fn vstack(iter: impl IntoIterator>) -> Array2 + where + T: Clone + Num, + { + let iter = Vec::from_iter(iter); + let mut res = Array2::::zeros((iter.len(), iter.first().unwrap().len())); + for (i, s) in iter.iter().enumerate() { + res.slice_mut(s![i, ..]).assign(s); + } + res } - res } + +pub(crate) mod linalg {} \ No newline at end of file diff --git a/data/src/flows/direction.rs b/data/src/flows/direction.rs index 179fe6b9..f83e1b0d 100644 --- a/data/src/flows/direction.rs +++ b/data/src/flows/direction.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use serde::{Deserialize, Serialize}; -use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; +use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; #[derive( Clone, @@ -16,13 +16,13 @@ use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[repr(usize)] #[serde(rename_all = "lowercase")] diff --git a/data/src/misc/dtype.rs b/data/src/misc/dtype.rs index 5b8d1ad2..52b405c7 100644 --- a/data/src/misc/dtype.rs +++ b/data/src/misc/dtype.rs @@ -4,7 +4,7 @@ */ use serde::{Deserialize, Serialize}; use smart_default::SmartDefault; -use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; +use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; pub trait DataType { fn dtype(&self) -> DType; @@ -29,7 +29,6 @@ where EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, @@ -37,6 +36,7 @@ where PartialOrd, Serialize, SmartDefault, + VariantNames, )] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] @@ -151,13 +151,13 @@ impl From for DType { EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] @@ -188,27 +188,30 @@ impl From for DType { pub struct Int { size: IntSize, } + #[derive( Clone, Copy, Debug, + Default, Deserialize, Display, EnumCount, EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] pub enum IntSize { + #[default] S8 = 8, S16 = 16, S32 = 32, @@ -228,13 +231,13 @@ pub enum IntSize { EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] @@ -289,6 +292,7 @@ impl From for DType { DType::Integer(dtype) } } + #[derive( Clone, Copy, @@ -300,13 +304,13 @@ impl From for DType { EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] diff --git a/data/src/tensors/mode.rs b/data/src/tensors/mode.rs index ee153054..bac0be07 100644 --- a/data/src/tensors/mode.rs +++ b/data/src/tensors/mode.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use serde::{Deserialize, Serialize}; -use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; +use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; #[derive( Clone, @@ -16,13 +16,13 @@ use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[repr(usize)] #[serde(rename_all = "lowercase")] diff --git a/ml/linear/src/cmp/params/kinds.rs b/ml/linear/src/cmp/params/kinds.rs index 660fa41b..e618bc7a 100644 --- a/ml/linear/src/cmp/params/kinds.rs +++ b/ml/linear/src/cmp/params/kinds.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use serde::{Deserialize, Serialize}; -use strum::{EnumIs, EnumIter, EnumString, EnumVariantNames}; +use strum::{EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; pub trait ParamType: ToString { fn kind(&self) -> String; @@ -14,16 +14,17 @@ pub trait ParamType: ToString { Debug, Default, Deserialize, + EnumCount, EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[non_exhaustive] #[repr(usize)] diff --git a/ml/neural/src/errors/error.rs b/ml/neural/src/errors/error.rs index d51d1243..62abf063 100644 --- a/ml/neural/src/errors/error.rs +++ b/ml/neural/src/errors/error.rs @@ -4,16 +4,16 @@ */ use serde::{Deserialize, Serialize}; use smart_default::SmartDefault; -use strum::{Display, EnumIs, EnumIter, EnumVariantNames}; +use strum::{Display, EnumCount, EnumIs, EnumIter, VariantNames}; #[derive( Clone, Debug, Deserialize, Display, + EnumCount, EnumIs, EnumIter, - EnumVariantNames, Eq, Hash, Ord, @@ -21,6 +21,7 @@ use strum::{Display, EnumIs, EnumIter, EnumVariantNames}; PartialOrd, Serialize, SmartDefault, + VariantNames )] #[non_exhaustive] #[serde(rename_all = "lowercase")] @@ -65,9 +66,9 @@ impl From for MlError { Debug, Deserialize, Display, + EnumCount, EnumIs, EnumIter, - EnumVariantNames, Eq, Hash, Ord, @@ -75,6 +76,7 @@ impl From for MlError { PartialOrd, Serialize, SmartDefault, + VariantNames )] #[non_exhaustive] #[serde(rename_all = "lowercase")] @@ -89,9 +91,9 @@ pub enum PredictError { Debug, Deserialize, Display, + EnumCount, EnumIs, EnumIter, - EnumVariantNames, Eq, Hash, Ord, @@ -99,6 +101,7 @@ pub enum PredictError { PartialOrd, Serialize, SmartDefault, + VariantNames )] #[non_exhaustive] #[serde(rename_all = "lowercase")] @@ -115,9 +118,9 @@ pub enum ComputeError { Debug, Deserialize, Display, + EnumCount, EnumIs, EnumIter, - EnumVariantNames, Eq, Hash, Ord, @@ -125,6 +128,7 @@ pub enum ComputeError { PartialOrd, Serialize, SmartDefault, + VariantNames )] #[non_exhaustive] #[serde(rename_all = "lowercase")] diff --git a/ml/neural/src/func/loss/kinds.rs b/ml/neural/src/func/loss/kinds.rs index 676dd32b..92bddbc8 100644 --- a/ml/neural/src/func/loss/kinds.rs +++ b/ml/neural/src/func/loss/kinds.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; +use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; #[derive( Clone, @@ -12,16 +12,17 @@ use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; Default, Deserialize, Display, + EnumCount, EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[repr(usize)] #[serde(rename_all = "lowercase")] diff --git a/ml/neural/src/func/prop/modes.rs b/ml/neural/src/func/prop/modes.rs index 3860b43e..cb58ad73 100644 --- a/ml/neural/src/func/prop/modes.rs +++ b/ml/neural/src/func/prop/modes.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; +use strum::{Display, EnumIs, EnumIter, EnumString, VariantNames}; #[derive( Clone, @@ -15,13 +15,13 @@ use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[repr(usize)] #[serde(rename_all = "lowercase")] diff --git a/ml/neural/src/layers/cmp/kinds.rs b/ml/neural/src/layers/cmp/kinds.rs index 703c429b..9fa587bc 100644 --- a/ml/neural/src/layers/cmp/kinds.rs +++ b/ml/neural/src/layers/cmp/kinds.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; +use strum::{Display, EnumIs, EnumIter, EnumString, VariantNames}; #[derive( Clone, @@ -15,13 +15,13 @@ use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[repr(usize)] #[serde(rename_all = "lowercase")] diff --git a/ml/neural/src/models/modes.rs b/ml/neural/src/models/modes.rs index 08192212..964e9ab7 100644 --- a/ml/neural/src/models/modes.rs +++ b/ml/neural/src/models/modes.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames, VariantNames}; +use strum::{Display, EnumIs, EnumIter, EnumString, VariantNames}; #[derive( Clone, @@ -15,13 +15,13 @@ use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames, VariantName EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[repr(usize)] #[serde(rename_all = "lowercase")] diff --git a/ml/neural/src/nn/kinds.rs b/ml/neural/src/nn/kinds.rs index 600f0c00..e79b5655 100644 --- a/ml/neural/src/nn/kinds.rs +++ b/ml/neural/src/nn/kinds.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; +use strum::{Display, EnumIs, EnumIter, EnumString, VariantNames}; #[derive( Clone, @@ -15,13 +15,13 @@ use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[repr(usize)] #[serde(rename_all = "snake_case")] @@ -66,13 +66,13 @@ impl NetworkKind { EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[repr(usize)] #[serde(rename_all = "lowercase")] @@ -108,13 +108,13 @@ impl Learning { EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[repr(usize)] #[serde(rename_all = "snake_case")] @@ -139,13 +139,13 @@ pub enum NetworkType { EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[repr(usize)] #[serde(rename_all = "snake_case")] diff --git a/ml/neural/src/params/shapes.rs b/ml/neural/src/params/shapes.rs index bd67e16c..e7634373 100644 --- a/ml/neural/src/params/shapes.rs +++ b/ml/neural/src/params/shapes.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; +use strum::{Display, EnumIs, EnumIter, EnumString, VariantNames}; pub trait LayerFeatures { fn inputs(&self) -> usize; @@ -33,13 +33,13 @@ impl LayerFeatures for ParameterShapes { Deserialize, EnumIs, EnumIter, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] pub enum ParameterShapes { Layer { inputs: usize, outputs: usize }, @@ -56,13 +56,13 @@ pub enum ParameterShapes { EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[repr(usize)] #[serde(rename_all = "lowercase")] diff --git a/ml/s4/Cargo.toml b/ml/s4/Cargo.toml index 5b47742e..45daa666 100644 --- a/ml/s4/Cargo.toml +++ b/ml/s4/Cargo.toml @@ -75,6 +75,7 @@ ndarray-rand.workspace = true ndarray-stats.workspace = true num.workspace = true rand = "0.8" +realfft = "3" rustfft = { features = [], version = "6" } serde.workspace = true serde_json.workspace = true diff --git a/ml/s4/src/cmp/kernel.rs b/ml/s4/src/cmp/kernel.rs index dac60cbb..d238483e 100644 --- a/ml/s4/src/cmp/kernel.rs +++ b/ml/s4/src/cmp/kernel.rs @@ -2,15 +2,16 @@ Appellation: kernel Contrib: FL03 */ -use crate::core::ops::fft::*; +use crate::core::ops::fft::{ifft, FftPlan}; use crate::core::prelude::Conjugate; use crate::params::DPLRParams; -use crate::prelude::cauchy; +use crate::prelude::{cauchy, }; use ndarray::prelude::{Array, Array1}; use ndarray::ScalarOperand; use ndarray_linalg::Scalar; use num::complex::{Complex, ComplexFloat}; use num::traits::{Float, FloatConst, NumOps}; +use rustfft::FftNum; pub fn omega_l(l: usize) -> Array1<::Complex> where @@ -73,35 +74,34 @@ pub fn kernel_dplr( l: usize, ) -> Array1<::Real> where - T: Conjugate + Float + Scalar>, + T: Conjugate + FftNum + Float + Scalar>, ::Real: FloatConst + NumOps<::Complex, ::Complex> + ScalarOperand, ::Complex: Conjugate + ScalarOperand, { - let one = T::one(); + // initialize some constants let two = T::from(2).unwrap(); // get the lambda matrix let lambda = dplr.lambda.clone(); - // generate omega - let omega_l: Array1<::Complex> = omega_l::(l); // collect the relevant terms for A let aterm = (dplr.c.conj(), dplr.q.conj()); // collect the relevant terms for B let bterm = (dplr.b.clone(), dplr.p.clone()); - let g = omega_l.mapv(|i| (one - i) * (one + i).recip()) * (two * step.recip()); - let c = omega_l.mapv(|i| two * (one + i).recip()); + // generate omega + let omega_l = omega_l::(l); + + let g = omega_l.mapv(|i| (T::one() - i) * (T::one() + i).recip()) * (two * step.recip()); + let c = omega_l.mapv(|i| two * (T::one() + i).recip()); // compute the cauchy matrix - let k00: Array1<::Complex> = cauchy(&(&aterm.0 * &bterm.0), &g, &lambda); - let k01: Array1<::Complex> = cauchy(&(&aterm.0 * &bterm.1), &g, &lambda); - let k10: Array1<::Complex> = cauchy(&(&aterm.1 * &bterm.0), &g, &lambda); - let k11: Array1<::Complex> = cauchy(&(&aterm.1 * &bterm.1), &g, &lambda); + let k00 = cauchy(&(&aterm.0 * &bterm.0), &g, &lambda); + let k01 = cauchy(&(&aterm.0 * &bterm.1), &g, &lambda); + let k10 = cauchy(&(&aterm.1 * &bterm.0), &g, &lambda); + let k11 = cauchy(&(&aterm.1 * &bterm.1), &g, &lambda); // compute the roots of unity - let at_roots = &c * (&k00 - k01 * &k11.mapv(|i| (i + one).recip()) * &k10); - - let buffer = at_roots.into_raw_vec(); - let permute = FftPlan::new(l); - let res = irfft(buffer.as_slice(), &permute); + let at_roots = &c * (&k00 - k01 * &k11.mapv(|i| (i + T::one()).recip()) * &k10); + let plan = FftPlan::new(l); + let res = ifft(at_roots.into_raw_vec().as_slice(), &plan); Array::from_vec(res).mapv(|i| i.re()) } @@ -123,6 +123,7 @@ impl Kernel where T: Scalar>, ::Real: Conjugate + + FftNum + Float + FloatConst + NumOps<::Complex, ::Complex> diff --git a/ml/s4/src/hippo/kinds.rs b/ml/s4/src/hippo/kinds.rs index decab8d7..be7dd63a 100644 --- a/ml/s4/src/hippo/kinds.rs +++ b/ml/s4/src/hippo/kinds.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use serde::{Deserialize, Serialize}; -use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; +use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; #[derive( Clone, @@ -16,13 +16,13 @@ use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[repr(usize)] #[serde(rename_all = "lowercase")] @@ -43,13 +43,13 @@ pub enum Rank { EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[repr(usize)] #[serde(rename_all = "lowercase")] diff --git a/ml/s4/src/ops/convolve.rs b/ml/s4/src/ops/convolve.rs index 2fdf7b28..c8c88ce2 100644 --- a/ml/s4/src/ops/convolve.rs +++ b/ml/s4/src/ops/convolve.rs @@ -2,13 +2,15 @@ Appellation: convolve Contrib: FL03 */ -use crate::core::ops::fft::{fft, ifftr, FftPlan}; -use crate::core::prelude::Power; +// use crate::core::ops::fft::{rfft, irfft, FftPlan}; +use crate::prelude::{irfft, rfft}; +use crate::core::prelude::{pad, Power}; use ndarray::linalg::Dot; use ndarray::prelude::{Array, Array1, Array2}; use ndarray::ScalarOperand; use num::complex::{Complex, ComplexFloat}; -use num::traits::{Float, FloatConst, Num, NumOps}; +use num::traits::{Float, FloatConst, Num, NumAssignOps,}; +use rustfft::FftNum; /// Generates a large convolution kernal pub fn k_conv(a: &Array2, b: &Array2, c: &Array2, l: usize) -> Array1 @@ -25,23 +27,45 @@ where Array::from_vec(store) } +// pub fn casual_convolution(u: &Array1, k: &Array1) -> Array1 +// where +// T: Float + FloatConst, +// Complex: ComplexFloat + NumAssignOps, +// { +// assert!(u.shape()[0] == k.shape()[0]); +// let l = u.shape()[0]; +// let plan = FftPlan::new(l); +// let ud = rfft::(u.clone().into_raw_vec(), &plan); +// let kd = rfft::(k.clone().into_raw_vec(), &plan); + +// let ud = Array::from_vec(ud); +// let kd = Array::from_vec(kd); + +// let tmp = ud * kd; +// let res = irfft(tmp.into_raw_vec().as_slice(), &plan); +// Array::from_vec(res) +// } + pub fn casual_convolution(u: &Array1, k: &Array1) -> Array1 where - T: Float + FloatConst + NumOps, Complex>, - Complex: ComplexFloat, + T: Default + FftNum + Float + FloatConst, + Complex: ComplexFloat + NumAssignOps, { assert!(u.shape()[0] == k.shape()[0]); let l = u.shape()[0]; - let plan = FftPlan::new(l); - let ud = fft(u.clone().into_raw_vec(), plan.clone()); - let kd = fft(k.clone().into_raw_vec(), plan.clone()); + let ud = { + let padded = pad(u.clone(), k.len(), Some(T::zero())); + Array::from_vec(rfft::(padded)) + }; - let ud = Array::from_vec(ud); - let kd = Array::from_vec(kd); + let kd = { + let padded = pad(k.clone(), l, Some(T::zero())); + Array::from_vec(rfft::(padded)) + }; - let res = ud * kd; - let res = ifftr(res.into_raw_vec().as_slice(), &plan); - Array::from_vec(res) + let tmp = &ud * kd; + let res = irfft(tmp, l); + Array::from_vec(res[0..l].to_vec()) } pub struct Filter {} @@ -60,16 +84,17 @@ mod tests { lazy_static! { static ref EXP: Array1 = - array![-7.10542736e-15, 0.0, 1.0, 4.0, 10.0, 20.0, 35.0, 56.0]; + array![-7.10542736e-15, 0.0, 1.0, 4.0, 1.0e1, 2.0e1, 3.5e1, 5.6e1]; } // #[ignore] #[test] fn test_casual_convolution() { let u = arange(0.0, SAMPLES as f64, 1.0); - let plan = FftPlan::new(SAMPLES); - println!("{:?}", rfft(u.clone().into_raw_vec(), plan)); let k = arange(0.0, SAMPLES as f64, 1.0); + + let plan = FftPlan::new(SAMPLES); + // println!("{:?}", rfft(u.clone().into_raw_vec(), plan)); let res = casual_convolution(&u, &k); println!("{:?}", res); assert_eq!(res, *EXP); diff --git a/ml/s4/src/ops/discretize.rs b/ml/s4/src/ops/discretize.rs index cfba22c2..5f7e9663 100644 --- a/ml/s4/src/ops/discretize.rs +++ b/ml/s4/src/ops/discretize.rs @@ -7,7 +7,7 @@ use crate::core::prelude::{Conjugate, Power}; use ndarray::{Array, Array1, Array2, Axis, ScalarOperand}; use ndarray_linalg::{Inverse, Lapack, Scalar}; use num::complex::ComplexFloat; -use num::traits::{Float, Num, NumOps}; +use num::traits::{Float, NumOps}; pub fn discretize( a: &Array2, diff --git a/ml/s4/src/params/kinds.rs b/ml/s4/src/params/kinds.rs index 7b4820b1..300a192c 100644 --- a/ml/s4/src/params/kinds.rs +++ b/ml/s4/src/params/kinds.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use serde::{Deserialize, Serialize}; -use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; +use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; #[derive( Clone, @@ -16,13 +16,13 @@ use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, EnumVariantNames}; EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[repr(usize)] #[serde(rename_all = "lowercase")] @@ -75,13 +75,13 @@ impl From for SSMParams { EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[repr(usize)] #[serde(rename_all = "lowercase")] diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index a932605f..69f96c8b 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -2,6 +2,8 @@ Appellation: utils Contrib: FL03 */ +pub use self::fft::*; + use ndarray::prelude::*; use ndarray::{IntoDimension, ScalarOperand}; use ndarray_rand::rand_distr::uniform::SampleUniform; @@ -63,3 +65,39 @@ where } res } + +pub(crate) mod fft { + use num::{Complex, NumCast}; + use realfft::RealFftPlanner; + use rustfft::FftNum; + + pub fn rfft(args: impl IntoIterator) -> Vec> where T: FftNum { + let mut buffer = Vec::from_iter(args); + // make a planner + let mut real_planner = RealFftPlanner::::new(); + // create a FFT + let r2c = real_planner.plan_fft_forward(buffer.len()); + // make a vector for storing the spectrum + let mut spectrum = r2c.make_output_vec(); + // forward transform the signal + r2c.process(&mut buffer, &mut spectrum).unwrap(); + spectrum + + } + + pub fn irfft(args: impl IntoIterator>, len: usize) -> Vec where T: FftNum + NumCast { + let mut buffer = Vec::from_iter(args); + // make a planner + let mut real_planner = RealFftPlanner::::new(); + // create a FFT + let r2c = real_planner.plan_fft_inverse(len); + // make a vector for storing the spectrum + let mut spectrum = r2c.make_output_vec(); + // forward transform the signal + r2c.process(&mut buffer, &mut spectrum).unwrap(); + let scale = T::one() / T::from(len).unwrap(); + spectrum.iter().cloned().map(|i| i * scale).collect() + + } + +} \ No newline at end of file diff --git a/ml/s4/tests/dplr.rs b/ml/s4/tests/dplr.rs index f83a0fe1..c8ddae7d 100644 --- a/ml/s4/tests/dplr.rs +++ b/ml/s4/tests/dplr.rs @@ -29,7 +29,6 @@ lazy_static! { #[test] // #[ignore = "TODO: fix this test"] fn test_gen_dplr() { - let (features, samples) = (4, 16); let eye = Array2::::eye(FEATURES); diff --git a/ml/transformers/src/attention/params/qkv.rs b/ml/transformers/src/attention/params/qkv.rs index 9969424b..1a538909 100644 --- a/ml/transformers/src/attention/params/qkv.rs +++ b/ml/transformers/src/attention/params/qkv.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; +use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; #[derive( Clone, @@ -12,16 +12,17 @@ use strum::{Display, EnumIs, EnumIter, EnumString, EnumVariantNames}; Default, Deserialize, Display, + EnumCount, EnumIs, EnumIter, EnumString, - EnumVariantNames, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, + VariantNames, )] #[repr(usize)] #[serde(rename_all = "lowercase")] From e3578db4721b9e752b6f7722a16b698ab7bfe371 Mon Sep 17 00:00:00 2001 From: FL03 Date: Sun, 28 Jan 2024 12:38:28 -0600 Subject: [PATCH 115/118] update Signed-off-by: FL03 --- core/src/ops/fft/algorithms/dft.rs | 6 ++-- core/src/ops/fft/algorithms/mod.rs | 5 +-- core/src/ops/fft/mod.rs | 56 +++++++++++++++++++++--------- core/src/ops/fft/modes.rs | 6 ++-- core/src/params/kinds.rs | 2 +- core/src/specs/arrays.rs | 1 - core/src/specs/math/mod.rs | 2 +- core/src/specs/math/ops.rs | 10 +++--- core/src/utils.rs | 24 ++++++------- ml/neural/src/errors/error.rs | 8 ++--- ml/s4/src/cmp/kernel.rs | 2 +- ml/s4/src/ops/convolve.rs | 29 ++++++++++------ ml/s4/src/utils.rs | 15 ++++---- ml/s4/tests/dplr.rs | 1 - 14 files changed, 99 insertions(+), 68 deletions(-) diff --git a/core/src/ops/fft/algorithms/dft.rs b/core/src/ops/fft/algorithms/dft.rs index 8046e515..17f986f4 100644 --- a/core/src/ops/fft/algorithms/dft.rs +++ b/core/src/ops/fft/algorithms/dft.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ //! # Discrete Fourier Transform -//! -//! +//! +//! -pub struct Dft; \ No newline at end of file +pub struct Dft; diff --git a/core/src/ops/fft/algorithms/mod.rs b/core/src/ops/fft/algorithms/mod.rs index ddf471f3..b639612e 100644 --- a/core/src/ops/fft/algorithms/mod.rs +++ b/core/src/ops/fft/algorithms/mod.rs @@ -1,9 +1,6 @@ - pub use self::dft::*; pub(crate) mod dft; #[cfg(test)] -mod tests { - -} \ No newline at end of file +mod tests {} diff --git a/core/src/ops/fft/mod.rs b/core/src/ops/fft/mod.rs index 1b1021c4..c90585d6 100644 --- a/core/src/ops/fft/mod.rs +++ b/core/src/ops/fft/mod.rs @@ -18,12 +18,11 @@ pub trait Fft { fn ifft(&self) -> Vec; } - pub(crate) mod utils { use super::FftPlan; use crate::prelude::AsComplex; use num::complex::{Complex, ComplexFloat}; - use num::traits::{Float, FloatConst, Num, NumAssignOps, NumCast, NumOps }; + use num::traits::{Float, FloatConst, Num, NumAssignOps, NumCast, NumOps}; // pub(crate) fn rsize(n: usize) -> usize { // (n / 2).floor() + 1 @@ -36,11 +35,17 @@ pub(crate) mod utils { T::TAU() / T::from(n).unwrap() } - pub(crate) fn floor(lhs: T, rhs: T) -> T where T: Copy + Num { + pub(crate) fn floor(lhs: T, rhs: T) -> T + where + T: Copy + Num, + { (lhs - lhs % rhs) / rhs } - pub(crate) fn unfloor(lhs: T, rhs: T) -> T where T: Copy + Num { + pub(crate) fn unfloor(lhs: T, rhs: T) -> T + where + T: Copy + Num, + { (lhs * rhs) - lhs % rhs } @@ -86,7 +91,10 @@ pub(crate) mod utils { /// Computes the Fast Fourier Transform of an one-dimensional, real-valued signal. /// TODO: Optimize the function to avoid unnecessary computation. - pub fn rfft(input: impl AsRef<[T]>, input_permutation: impl AsRef<[usize]>) -> Vec> + pub fn rfft( + input: impl AsRef<[T]>, + input_permutation: impl AsRef<[usize]>, + ) -> Vec> where T: Float + FloatConst, Complex: ComplexFloat + NumAssignOps, @@ -121,7 +129,11 @@ pub(crate) mod utils { } } } - store.iter().cloned().filter(|x| x.im() >= T::zero()).collect() + store + .iter() + .cloned() + .filter(|x| x.im() >= T::zero()) + .collect() } /// Computes the Inverse Fast Fourier Transform of an one-dimensional, complex-valued signal. pub fn ifft(input: &[S], input_permutation: &FftPlan) -> Vec> @@ -204,7 +216,6 @@ mod tests { use lazy_static::lazy_static; use num::complex::{Complex, ComplexFloat}; - pub(crate) fn fft_permutation(length: usize) -> Vec { let mut result = Vec::new(); result.reserve_exact(length); @@ -234,11 +245,20 @@ mod tests { lazy_static! { static ref EXPECTED_RFFT: Vec> = vec![ - Complex { re: 28.0, im: 0.0 }, - Complex { re: -4.0, im: 0.0 }, - Complex { re: -4.0, im: 1.6568542494923806 }, - Complex { re: -4.0, im: 4.000000000000001 }, - Complex { re: -3.999999999999999, im: 9.656854249492381 } + Complex { re: 28.0, im: 0.0 }, + Complex { re: -4.0, im: 0.0 }, + Complex { + re: -4.0, + im: 1.6568542494923806 + }, + Complex { + re: -4.0, + im: 4.000000000000001 + }, + Complex { + re: -3.999999999999999, + im: 9.656854249492381 + } ]; } @@ -247,10 +267,7 @@ mod tests { let samples = 16; let plan = FftPlan::new(samples); - assert_eq!( - plan.plan(), - fft_permutation(16).as_slice() - ); + assert_eq!(plan.plan(), fft_permutation(16).as_slice()); } #[test] @@ -260,7 +277,12 @@ mod tests { println!("Function Values: {:?}", &polynomial); println!("Plan: {:?}", &plan); let fft = rfft(&polynomial, &plan); - let mut tmp = fft.iter().cloned().filter(|i| i.im() > 0.0).map(|i| i.conj()).collect::>(); + let mut tmp = fft + .iter() + .cloned() + .filter(|i| i.im() > 0.0) + .map(|i| i.conj()) + .collect::>(); tmp.sort_by(|a, b| a.im().partial_cmp(&b.im()).unwrap()); println!("FFT: {:?}", &tmp); let mut res = fft.clone(); diff --git a/core/src/ops/fft/modes.rs b/core/src/ops/fft/modes.rs index fb4a5f29..c5cb75af 100644 --- a/core/src/ops/fft/modes.rs +++ b/core/src/ops/fft/modes.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use serde::{Deserialize, Serialize}; -use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, VariantArray, VariantNames,}; +use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, VariantArray, VariantNames}; #[derive( Clone, @@ -23,7 +23,7 @@ use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, VariantArray, Vari PartialOrd, Serialize, VariantArray, - VariantNames + VariantNames, )] #[repr(usize)] #[serde(rename_all = "lowercase")] @@ -76,7 +76,7 @@ impl From for usize { PartialOrd, Serialize, VariantArray, - VariantNames + VariantNames, )] #[repr(usize)] #[serde(rename_all = "lowercase")] diff --git a/core/src/params/kinds.rs b/core/src/params/kinds.rs index d8750d24..fbfd0262 100644 --- a/core/src/params/kinds.rs +++ b/core/src/params/kinds.rs @@ -33,7 +33,7 @@ where PartialEq, PartialOrd, Serialize, - VariantNames + VariantNames, )] #[non_exhaustive] #[repr(usize)] diff --git a/core/src/specs/arrays.rs b/core/src/specs/arrays.rs index 76dc0e97..3e2847e0 100644 --- a/core/src/specs/arrays.rs +++ b/core/src/specs/arrays.rs @@ -19,7 +19,6 @@ pub trait Pad { fn pad_with(&self, pad: usize, value: T) -> Self; } - // impl Pad for Array // where // T: Clone + Num, diff --git a/core/src/specs/math/mod.rs b/core/src/specs/math/mod.rs index 87fecfe1..878ce3c3 100644 --- a/core/src/specs/math/mod.rs +++ b/core/src/specs/math/mod.rs @@ -9,7 +9,7 @@ pub(crate) mod ops; pub(crate) mod scalar; use num::complex::Complex; -use num::traits::{Float, Num,}; +use num::traits::{Float, Num}; pub trait AsComplex: Sized { fn as_complex(self, real: bool) -> Complex; diff --git a/core/src/specs/math/ops.rs b/core/src/specs/math/ops.rs index 5ed5b36d..14d3d789 100644 --- a/core/src/specs/math/ops.rs +++ b/core/src/specs/math/ops.rs @@ -54,16 +54,18 @@ where pub trait Arithmetic where - Self: ops::Add + ops::Div + ops::Mul + ops::Sub, + Self: ops::Add + + ops::Div + + ops::Mul + + ops::Sub, { } -impl Arithmetic for A -where +impl Arithmetic for A where A: ops::Add + ops::Div + ops::Mul - + ops::Sub, + + ops::Sub { } diff --git a/core/src/utils.rs b/core/src/utils.rs index c2a6084f..c1088ae5 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -11,9 +11,9 @@ use ndarray_rand::rand::rngs::StdRng; use ndarray_rand::rand::SeedableRng; use ndarray_rand::rand_distr::{Distribution, StandardNormal}; use ndarray_rand::RandomExt; -use rand::distributions::uniform::{SampleUniform, Uniform}; use num::complex::Complex; -use num::traits::{AsPrimitive, Float, Num, NumCast,}; +use num::traits::{AsPrimitive, Float, Num, NumCast}; +use rand::distributions::uniform::{SampleUniform, Uniform}; pub fn pad(a: impl IntoIterator, pad: usize, value: Option) -> Vec where @@ -25,7 +25,10 @@ where res } /// -pub fn floor_div(numerator: T, denom: T) -> T where T: Copy + Num { +pub fn floor_div(numerator: T, denom: T) -> T +where + T: Copy + Num, +{ (numerator - (numerator % denom)) / denom } @@ -42,9 +45,6 @@ where res } - - - pub fn genspace(features: usize) -> Array1 { Array1::from_iter((0..features).map(|x| T::from(x).unwrap())) } @@ -165,9 +165,6 @@ where Array::random(shape, StandardNormal) } - - - pub(crate) mod assertions { use ndarray::prelude::{Array, Dimension}; use ndarray::ScalarOperand; @@ -209,10 +206,13 @@ pub(crate) mod assertions { pub(crate) mod arrays { use ndarray::prelude::{s, Array, Array1, Array2, Axis}; - use ndarray::{RemoveAxis, concatenate}; + use ndarray::{concatenate, RemoveAxis}; use num::traits::{Num, Zero}; /// Creates an n-dimensional array from an iterator of n dimensional arrays. - pub fn concat_iter(axis: usize, iter: impl IntoIterator>) -> Array + pub fn concat_iter( + axis: usize, + iter: impl IntoIterator>, + ) -> Array where D: RemoveAxis, T: Clone, @@ -291,4 +291,4 @@ pub(crate) mod arrays { } } -pub(crate) mod linalg {} \ No newline at end of file +pub(crate) mod linalg {} diff --git a/ml/neural/src/errors/error.rs b/ml/neural/src/errors/error.rs index 62abf063..1c00849b 100644 --- a/ml/neural/src/errors/error.rs +++ b/ml/neural/src/errors/error.rs @@ -21,7 +21,7 @@ use strum::{Display, EnumCount, EnumIs, EnumIter, VariantNames}; PartialOrd, Serialize, SmartDefault, - VariantNames + VariantNames, )] #[non_exhaustive] #[serde(rename_all = "lowercase")] @@ -76,7 +76,7 @@ impl From for MlError { PartialOrd, Serialize, SmartDefault, - VariantNames + VariantNames, )] #[non_exhaustive] #[serde(rename_all = "lowercase")] @@ -101,7 +101,7 @@ pub enum PredictError { PartialOrd, Serialize, SmartDefault, - VariantNames + VariantNames, )] #[non_exhaustive] #[serde(rename_all = "lowercase")] @@ -128,7 +128,7 @@ pub enum ComputeError { PartialOrd, Serialize, SmartDefault, - VariantNames + VariantNames, )] #[non_exhaustive] #[serde(rename_all = "lowercase")] diff --git a/ml/s4/src/cmp/kernel.rs b/ml/s4/src/cmp/kernel.rs index d238483e..d4cb1bcb 100644 --- a/ml/s4/src/cmp/kernel.rs +++ b/ml/s4/src/cmp/kernel.rs @@ -5,7 +5,7 @@ use crate::core::ops::fft::{ifft, FftPlan}; use crate::core::prelude::Conjugate; use crate::params::DPLRParams; -use crate::prelude::{cauchy, }; +use crate::prelude::cauchy; use ndarray::prelude::{Array, Array1}; use ndarray::ScalarOperand; use ndarray_linalg::Scalar; diff --git a/ml/s4/src/ops/convolve.rs b/ml/s4/src/ops/convolve.rs index c8c88ce2..654f197b 100644 --- a/ml/s4/src/ops/convolve.rs +++ b/ml/s4/src/ops/convolve.rs @@ -3,13 +3,13 @@ Contrib: FL03 */ // use crate::core::ops::fft::{rfft, irfft, FftPlan}; +use crate::core::prelude::{floor_div, pad, Power}; use crate::prelude::{irfft, rfft}; -use crate::core::prelude::{pad, Power}; use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Array1, Array2}; +use ndarray::prelude::{array, s, Array, Array1, Array2, Axis}; use ndarray::ScalarOperand; use num::complex::{Complex, ComplexFloat}; -use num::traits::{Float, FloatConst, Num, NumAssignOps,}; +use num::traits::{Float, FloatConst, Num, NumAssignOps}; use rustfft::FftNum; /// Generates a large convolution kernal @@ -53,6 +53,9 @@ where { assert!(u.shape()[0] == k.shape()[0]); let l = u.shape()[0]; + println!("{:?}", l); + let l2 = l * 2; + let inv_size = floor_div(l, 2) + 1; let ud = { let padded = pad(u.clone(), k.len(), Some(T::zero())); Array::from_vec(rfft::(padded)) @@ -63,8 +66,11 @@ where Array::from_vec(rfft::(padded)) }; - let tmp = &ud * kd; - let res = irfft(tmp, l); + let mut tmp = &ud * kd; + // let a = array![Complex::new(T::zero(), T::zero())]; + // tmp.append(Axis(0), a.view()).expect(""); + let res = irfft(tmp, l2); + // let res = irfft(tmp.slice(s![0..l]).to_vec(), l2); Array::from_vec(res[0..l].to_vec()) } @@ -73,8 +79,8 @@ pub struct Filter {} #[cfg(test)] mod tests { use super::*; - use crate::core::prelude::arange; use crate::core::ops::fft::*; + use crate::core::prelude::{arange, assert_approx}; use lazy_static::lazy_static; use ndarray::prelude::*; @@ -85,6 +91,8 @@ mod tests { lazy_static! { static ref EXP: Array1 = array![-7.10542736e-15, 0.0, 1.0, 4.0, 1.0e1, 2.0e1, 3.5e1, 5.6e1]; + static ref EXP2: Array1 = + array![0.0, -7.10542736e-15, 1.0, 4.0, 1.0e1, 2.0e1, 3.5e1, 5.6e1]; } // #[ignore] @@ -92,11 +100,12 @@ mod tests { fn test_casual_convolution() { let u = arange(0.0, SAMPLES as f64, 1.0); let k = arange(0.0, SAMPLES as f64, 1.0); - - let plan = FftPlan::new(SAMPLES); - // println!("{:?}", rfft(u.clone().into_raw_vec(), plan)); + + let _plan = FftPlan::new(SAMPLES); let res = casual_convolution(&u, &k); println!("{:?}", res); - assert_eq!(res, *EXP); + for (i, j) in res.into_iter().zip(EXP2.clone().into_iter()) { + assert_approx(i, j, 1e-8); + } } } diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index 69f96c8b..4c9055f8 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -71,7 +71,10 @@ pub(crate) mod fft { use realfft::RealFftPlanner; use rustfft::FftNum; - pub fn rfft(args: impl IntoIterator) -> Vec> where T: FftNum { + pub fn rfft(args: impl IntoIterator) -> Vec> + where + T: FftNum, + { let mut buffer = Vec::from_iter(args); // make a planner let mut real_planner = RealFftPlanner::::new(); @@ -82,10 +85,12 @@ pub(crate) mod fft { // forward transform the signal r2c.process(&mut buffer, &mut spectrum).unwrap(); spectrum - } - pub fn irfft(args: impl IntoIterator>, len: usize) -> Vec where T: FftNum + NumCast { + pub fn irfft(args: impl IntoIterator>, len: usize) -> Vec + where + T: FftNum + NumCast, + { let mut buffer = Vec::from_iter(args); // make a planner let mut real_planner = RealFftPlanner::::new(); @@ -97,7 +102,5 @@ pub(crate) mod fft { r2c.process(&mut buffer, &mut spectrum).unwrap(); let scale = T::one() / T::from(len).unwrap(); spectrum.iter().cloned().map(|i| i * scale).collect() - } - -} \ No newline at end of file +} diff --git a/ml/s4/tests/dplr.rs b/ml/s4/tests/dplr.rs index c8ddae7d..f6c9ddd4 100644 --- a/ml/s4/tests/dplr.rs +++ b/ml/s4/tests/dplr.rs @@ -29,7 +29,6 @@ lazy_static! { #[test] // #[ignore = "TODO: fix this test"] fn test_gen_dplr() { - let eye = Array2::::eye(FEATURES); let step = (SAMPLES as f64).recip(); From fb22cdc28cdb5b868fdb4844ae4dc4626d3de751 Mon Sep 17 00:00:00 2001 From: FL03 Date: Mon, 29 Jan 2024 10:52:16 -0600 Subject: [PATCH 116/118] update Signed-off-by: FL03 --- ml/s4/examples/sand.rs | 30 +++++++++------------------- ml/s4/src/ops/convolve.rs | 6 ++---- ml/s4/src/ops/mod.rs | 20 +++++++++++++------ ml/s4/src/ops/scan.rs | 42 ++++++++++++++++++++++++++++++++++++++- ml/s4/src/utils.rs | 8 ++++---- ml/s4/tests/utils.rs | 37 ++++++++++++++++++++++++---------- 6 files changed, 97 insertions(+), 46 deletions(-) diff --git a/ml/s4/examples/sand.rs b/ml/s4/examples/sand.rs index 69eef64c..b5eb5617 100644 --- a/ml/s4/examples/sand.rs +++ b/ml/s4/examples/sand.rs @@ -4,33 +4,21 @@ extern crate concision_s4; use concision_core as core; use concision_s4 as s4; -use core::prelude::{Arange, AsComplex}; use ndarray::prelude::*; -use rustfft::FftPlanner; -use s4::prelude::cauchy; -use s4::ssm::{SSMConfig, SSM}; +use s4::prelude::scan; fn main() -> anyhow::Result<()> { let (features, samples) = (4, 16); - let arr = Array::::arange(samples).mapv(AsComplex::as_re); - let mut planner = FftPlanner::::new(); - let fft = planner.plan_fft_inverse(samples); - let mut buffer = arr.to_vec(); - println!("Buffer: {:?}", &buffer); - fft.process(buffer.as_mut_slice()); + let u = Array::range(0.0, features as f64, 1.0).insert_axis(Axis(1)); + let x0 = Array1::::zeros(features); - let res = Array::from_vec(buffer); - - println!("FFT:\n\n{:?}\n", res); - - let config = SSMConfig::new(true, features, samples); - let _ssm = SSM::::create(config); - - let a = Array1::::arange(features * features).into_shape((features, features))? + 1.0; - let b = Array1::::arange(features).insert_axis(Axis(1)); - let c = Array1::::arange(features).insert_axis(Axis(0)) * 2.0; - println!("Cauchy:\n\n{:?}", cauchy(&a, &b, &c)); + // let step = | st, u | { + // let x1 = st; + // let yk = u; + // Some(x1) + // }; + // println!("{:?}", scan(step, u, x0.to_vec())); Ok(()) } diff --git a/ml/s4/src/ops/convolve.rs b/ml/s4/src/ops/convolve.rs index 654f197b..87f80a25 100644 --- a/ml/s4/src/ops/convolve.rs +++ b/ml/s4/src/ops/convolve.rs @@ -98,12 +98,10 @@ mod tests { // #[ignore] #[test] fn test_casual_convolution() { - let u = arange(0.0, SAMPLES as f64, 1.0); - let k = arange(0.0, SAMPLES as f64, 1.0); + let u = Array::range(0.0, SAMPLES as f64, 1.0); + let k = Array::range(0.0, SAMPLES as f64, 1.0); - let _plan = FftPlan::new(SAMPLES); let res = casual_convolution(&u, &k); - println!("{:?}", res); for (i, j) in res.into_iter().zip(EXP2.clone().into_iter()) { assert_approx(i, j, 1e-8); } diff --git a/ml/s4/src/ops/mod.rs b/ml/s4/src/ops/mod.rs index bf71ae94..b9c107aa 100644 --- a/ml/s4/src/ops/mod.rs +++ b/ml/s4/src/ops/mod.rs @@ -12,13 +12,14 @@ pub(crate) mod scan; #[cfg(test)] mod tests { use super::*; - use crate::core::prelude::{assert_atol, randc_normal}; - use num::complex::ComplexFloat; - use crate::cmp::kernel::kernel_dplr; + use crate::core::prelude::{assert_atol, randc_normal}; use crate::hippo::dplr::DPLR; use crate::params::DPLRParams; + use ndarray::prelude::*; + use num::complex::ComplexFloat; + const FEATURES: usize = 8; const RNGKEY: u64 = 1; const SAMPLES: usize = 16; @@ -26,19 +27,19 @@ mod tests { #[test] fn test_conversion() { let step = (SAMPLES as f64).recip(); - + // Initialize a new DPLR Matrix let dplr = DPLR::::new(FEATURES); let (lambda, p, b, _) = dplr.clone().into(); // let c = randcomplex(features); let c = randc_normal(RNGKEY, FEATURES); - + // CNN Form let kernel = { let params = DPLRParams::new(lambda.clone(), p.clone(), p.clone(), b.clone(), c.clone()); kernel_dplr::(¶ms, step, SAMPLES) }; - + // RNN Form let discrete = discretize_dplr(&lambda, &p, &p, &b, &c, step, SAMPLES).expect(""); let (ab, bb, cb) = discrete.into(); @@ -46,6 +47,13 @@ mod tests { let k2r = k2.mapv(|i| i.re()); assert_atol(&kernel, &k2r, 1e-4); + + // Apply the CNN + let u = Array::range(0.0, SAMPLES as f64, 1.0); + + let y1 = casual_convolution(&u, &kernel); + + // Apply the RNN } #[test] diff --git a/ml/s4/src/ops/scan.rs b/ml/s4/src/ops/scan.rs index f81367b2..756b6941 100644 --- a/ml/s4/src/ops/scan.rs +++ b/ml/s4/src/ops/scan.rs @@ -3,9 +3,49 @@ Contrib: FL03 */ use crate::params::SSMStore; - +use ndarray::prelude::{Array1, Array2, ArrayView1, NdFloat}; +use ndarray_linalg::error::LinalgError; +use ndarray_linalg::{vstack, Scalar}; +use num::complex::ComplexFloat; use num::Float; +pub fn scan_ssm( + a: &Array2, + b: &Array2, + c: &Array2, + u: &Array2, + x0: &Array1, +) -> Result, LinalgError> +where + T: ComplexFloat + NdFloat + Scalar, +{ + let step = |xs: &mut Array1, us: ArrayView1| { + let x1 = a.dot(xs) + b.dot(&us); + let y1 = c.dot(&x1); + Some(y1) + }; + let scan = u + .outer_iter() + .scan(x0.clone(), step) + .collect::>>(); + vstack(scan.as_slice()) +} + +pub fn scan(f: &mut F, init: S, xs: Vec) -> (S, Vec) +where + F: FnMut(&mut S, &T) -> S, + S: Clone, + T: Clone, +{ + let mut state = init; + let mut out = Vec::with_capacity(xs.len()); + for x in xs { + state = f(&mut state, &x); + out.push(state.clone()); + } + (state, out) +} + pub struct Scanner<'a, T = f64> where T: Float, diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index 4c9055f8..88bfbc3c 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -9,7 +9,7 @@ use ndarray::{IntoDimension, ScalarOperand}; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, Uniform}; use ndarray_rand::RandomExt; -use num::complex::{Complex, ComplexDistribution}; +use num::complex::{Complex, ComplexDistribution, ComplexFloat}; use num::traits::Num; use std::ops::Neg; @@ -50,16 +50,16 @@ pub fn scanner( x0: &Array1, ) -> Array2 where - T: NdFloat, + T: ComplexFloat + NdFloat, { let step = |xs: &mut Array1, us: ArrayView1| { - let x1 = a.dot(xs) + b.t().dot(&us); + let x1 = a.dot(xs) + b.dot(&us); let y1 = c.dot(&x1.t()); Some(y1) }; let scan = u.outer_iter().scan(x0.clone(), step).collect::>(); let shape = [scan.len(), scan[0].len()]; - let mut res = Array2::::zeros(shape.into_dimension()); + let mut res = Array2::zeros(shape.into_dimension()); for (i, s) in scan.iter().enumerate() { res.slice_mut(s![i, ..]).assign(s); } diff --git a/ml/s4/tests/utils.rs b/ml/s4/tests/utils.rs index 8876b845..9bb957ba 100644 --- a/ml/s4/tests/utils.rs +++ b/ml/s4/tests/utils.rs @@ -2,21 +2,38 @@ extern crate concision_s4; use concision_s4 as s4; +use s4::prelude::{scan_ssm, scanner, SSMStore}; + +use lazy_static::lazy_static; use ndarray::prelude::*; -use s4::prelude::{scanner, SSMStore}; +use num::Complex; + +const FEATURES: usize = 3; +const SAMPLES: usize = 16; + +lazy_static! { + static ref U: Array2 = Array::range(0.0, FEATURES as f64, 1.0).insert_axis(Axis(1)); + static ref X0: Array1 = Array1::zeros(FEATURES); + static ref X1: Array1> = Array1::zeros(FEATURES); + static ref A: Array2 = Array::range(0.0, (FEATURES * FEATURES) as f64, 1.0) + .into_shape((FEATURES, FEATURES)) + .unwrap(); + static ref B: Array2 = Array::range(0.0, FEATURES as f64, 1.0).insert_axis(Axis(1)); + static ref C: Array2 = Array::range(0.0, FEATURES as f64, 1.0).insert_axis(Axis(0)); +} #[test] fn test_scan() { - let features = 2; - - let u = Array2::ones((10, features)); - let x0 = Array1::ones(features); + let u = U.clone(); + let x0 = X0.clone(); - let ssm = SSMStore::::from_features(features); + let ssm = SSMStore::::from_features(FEATURES); let (a, b, c, _d) = ssm.clone().into(); - let scan1 = scanner(&a, &b, &c, &u, &x0); - - let scan2 = ssm.scan(&u, &x0).unwrap(); + let scan1 = scanner(&A, &B, &C, &u, &x0); + let scan2 = scan_ssm(&A, &B, &C, &u, &x0).expect(""); + println!("{:?}", &scan2); + // let scan2 = ssm.scan(&u, &x0).unwrap(); - assert_eq!(scan1, scan2); + assert_eq!(&scan1, &scan2); + assert!(false) } From 65e1009df351b8528425cd5c29e5e30172397776 Mon Sep 17 00:00:00 2001 From: FL03 Date: Mon, 29 Jan 2024 12:38:23 -0600 Subject: [PATCH 117/118] update Signed-off-by: FL03 --- ml/s4/src/ops/scan.rs | 9 +++++---- ml/s4/tests/utils.rs | 14 ++++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/ml/s4/src/ops/scan.rs b/ml/s4/src/ops/scan.rs index 756b6941..09d6d46a 100644 --- a/ml/s4/src/ops/scan.rs +++ b/ml/s4/src/ops/scan.rs @@ -22,25 +22,26 @@ where let step = |xs: &mut Array1, us: ArrayView1| { let x1 = a.dot(xs) + b.dot(&us); let y1 = c.dot(&x1); + *xs = x1; Some(y1) }; - let scan = u + let scan: Vec> = u .outer_iter() .scan(x0.clone(), step) - .collect::>>(); + .collect(); vstack(scan.as_slice()) } pub fn scan(f: &mut F, init: S, xs: Vec) -> (S, Vec) where - F: FnMut(&mut S, &T) -> S, + F: FnMut(&mut S, T) -> S, S: Clone, T: Clone, { let mut state = init; let mut out = Vec::with_capacity(xs.len()); for x in xs { - state = f(&mut state, &x); + state = f(&mut state, x); out.push(state.clone()); } (state, out) diff --git a/ml/s4/tests/utils.rs b/ml/s4/tests/utils.rs index 9bb957ba..c6954d55 100644 --- a/ml/s4/tests/utils.rs +++ b/ml/s4/tests/utils.rs @@ -20,6 +20,8 @@ lazy_static! { .unwrap(); static ref B: Array2 = Array::range(0.0, FEATURES as f64, 1.0).insert_axis(Axis(1)); static ref C: Array2 = Array::range(0.0, FEATURES as f64, 1.0).insert_axis(Axis(0)); + + static ref EXP: Array2 = array![[0.0], [5.0], [70.0]]; } #[test] @@ -29,11 +31,11 @@ fn test_scan() { let ssm = SSMStore::::from_features(FEATURES); let (a, b, c, _d) = ssm.clone().into(); - let scan1 = scanner(&A, &B, &C, &u, &x0); - let scan2 = scan_ssm(&A, &B, &C, &u, &x0).expect(""); - println!("{:?}", &scan2); - // let scan2 = ssm.scan(&u, &x0).unwrap(); + // let scan1 = scanner(&A, &B, &C, &u, &x0); + let scan = scan_ssm(&A, &B, &C, &u, &x0).expect(""); + println!("{:?}", &scan); + - assert_eq!(&scan1, &scan2); - assert!(false) + assert_eq!(&scan, *EXP); + // assert!(false) } From be3085398914bd24e2be4f919cde1e717b14f5e8 Mon Sep 17 00:00:00 2001 From: FL03 Date: Mon, 29 Jan 2024 14:11:36 -0600 Subject: [PATCH 118/118] update Signed-off-by: FL03 --- ml/s4/examples/sand.rs | 1 - ml/s4/src/ops/mod.rs | 38 +------------------------ ml/s4/src/ops/scan.rs | 50 +++++++++++++++++++++------------ ml/s4/src/params/store.rs | 5 ++-- ml/s4/src/utils.rs | 30 ++------------------ ml/s4/tests/conversion.rs | 59 +++++++++++++++++++++++++++++++++++++++ ml/s4/tests/utils.rs | 41 --------------------------- 7 files changed, 98 insertions(+), 126 deletions(-) create mode 100644 ml/s4/tests/conversion.rs delete mode 100644 ml/s4/tests/utils.rs diff --git a/ml/s4/examples/sand.rs b/ml/s4/examples/sand.rs index b5eb5617..bf8d58df 100644 --- a/ml/s4/examples/sand.rs +++ b/ml/s4/examples/sand.rs @@ -5,7 +5,6 @@ use concision_core as core; use concision_s4 as s4; use ndarray::prelude::*; -use s4::prelude::scan; fn main() -> anyhow::Result<()> { let (features, samples) = (4, 16); diff --git a/ml/s4/src/ops/mod.rs b/ml/s4/src/ops/mod.rs index b9c107aa..09a6e2eb 100644 --- a/ml/s4/src/ops/mod.rs +++ b/ml/s4/src/ops/mod.rs @@ -12,49 +12,13 @@ pub(crate) mod scan; #[cfg(test)] mod tests { use super::*; - use crate::cmp::kernel::kernel_dplr; - use crate::core::prelude::{assert_atol, randc_normal}; + use crate::core::prelude::randc_normal; use crate::hippo::dplr::DPLR; - use crate::params::DPLRParams; - - use ndarray::prelude::*; - use num::complex::ComplexFloat; const FEATURES: usize = 8; const RNGKEY: u64 = 1; const SAMPLES: usize = 16; - #[test] - fn test_conversion() { - let step = (SAMPLES as f64).recip(); - // Initialize a new DPLR Matrix - let dplr = DPLR::::new(FEATURES); - let (lambda, p, b, _) = dplr.clone().into(); - - // let c = randcomplex(features); - let c = randc_normal(RNGKEY, FEATURES); - // CNN Form - let kernel = { - let params = - DPLRParams::new(lambda.clone(), p.clone(), p.clone(), b.clone(), c.clone()); - kernel_dplr::(¶ms, step, SAMPLES) - }; - // RNN Form - let discrete = discretize_dplr(&lambda, &p, &p, &b, &c, step, SAMPLES).expect(""); - let (ab, bb, cb) = discrete.into(); - - let k2 = k_conv(&ab, &bb, &cb, SAMPLES); - let k2r = k2.mapv(|i| i.re()); - - assert_atol(&kernel, &k2r, 1e-4); - - // Apply the CNN - let u = Array::range(0.0, SAMPLES as f64, 1.0); - - let y1 = casual_convolution(&u, &kernel); - - // Apply the RNN - } #[test] fn test_discretize() { diff --git a/ml/s4/src/ops/scan.rs b/ml/s4/src/ops/scan.rs index 09d6d46a..601dc810 100644 --- a/ml/s4/src/ops/scan.rs +++ b/ml/s4/src/ops/scan.rs @@ -3,10 +3,9 @@ Contrib: FL03 */ use crate::params::SSMStore; -use ndarray::prelude::{Array1, Array2, ArrayView1, NdFloat}; +use ndarray::prelude::{Array1, Array2, ArrayView1,}; use ndarray_linalg::error::LinalgError; use ndarray_linalg::{vstack, Scalar}; -use num::complex::ComplexFloat; use num::Float; pub fn scan_ssm( @@ -17,7 +16,7 @@ pub fn scan_ssm( x0: &Array1, ) -> Result, LinalgError> where - T: ComplexFloat + NdFloat + Scalar, + T: Scalar, { let step = |xs: &mut Array1, us: ArrayView1| { let x1 = a.dot(xs) + b.dot(&us); @@ -32,21 +31,6 @@ where vstack(scan.as_slice()) } -pub fn scan(f: &mut F, init: S, xs: Vec) -> (S, Vec) -where - F: FnMut(&mut S, T) -> S, - S: Clone, - T: Clone, -{ - let mut state = init; - let mut out = Vec::with_capacity(xs.len()); - for x in xs { - state = f(&mut state, x); - out.push(state.clone()); - } - (state, out) -} - pub struct Scanner<'a, T = f64> where T: Float, @@ -70,3 +54,33 @@ where self.model } } + +#[cfg(test)] +mod tests { + use super::*; + + use ndarray::prelude::*; + + const FEATURES: usize = 3; + + + #[test] + fn test_scan() { + let exp = array![[0.0], [5.0], [70.0]]; + + let u = Array::range(0.0, FEATURES as f64, 1.0).insert_axis(Axis(1)); + let x0 = Array1::zeros(FEATURES); // Array1::>::zeros(FEATURES) + + let a = Array::range(0.0, (FEATURES * FEATURES) as f64, 1.0) + .into_shape((FEATURES, FEATURES)) + .unwrap(); + let b = Array::range(0.0, FEATURES as f64, 1.0).insert_axis(Axis(1)); + let c = Array::range(0.0, FEATURES as f64, 1.0).insert_axis(Axis(0)); + + + let scan = scan_ssm(&a, &b, &c, &u, &x0).expect(""); + + assert_eq!(&scan, &exp); + } + +} \ No newline at end of file diff --git a/ml/s4/src/params/store.rs b/ml/s4/src/params/store.rs index 5678191b..a4380f75 100644 --- a/ml/s4/src/params/store.rs +++ b/ml/s4/src/params/store.rs @@ -99,8 +99,9 @@ where { pub fn scan(&self, u: &Array2, x0: &Array1) -> Result, LinalgError> { let step = |xs: &mut Array1, us: ArrayView1| { - let x1 = self.a().dot(xs) + self.b().t().dot(&us); - let y1 = self.c().dot(&x1.t()); + let x1 = self.a().dot(xs) + self.b().dot(&us); + let y1 = self.c().dot(&x1); + *xs = x1; Some(y1) }; vstack( diff --git a/ml/s4/src/utils.rs b/ml/s4/src/utils.rs index 88bfbc3c..0930f85f 100644 --- a/ml/s4/src/utils.rs +++ b/ml/s4/src/utils.rs @@ -9,10 +9,11 @@ use ndarray::{IntoDimension, ScalarOperand}; use ndarray_rand::rand_distr::uniform::SampleUniform; use ndarray_rand::rand_distr::{Distribution, Uniform}; use ndarray_rand::RandomExt; -use num::complex::{Complex, ComplexDistribution, ComplexFloat}; +use num::complex::{Complex, ComplexDistribution,}; use num::traits::Num; use std::ops::Neg; +/// pub fn cauchy(a: &Array, b: &Array, c: &Array) -> Array where A: Dimension, @@ -22,7 +23,7 @@ where let cdot = |b: T| (a / (c * T::one().neg() + b)).sum(); b.mapv(cdot) } - +/// pub fn logstep(a: T, b: T, shape: impl IntoDimension) -> Array where D: Dimension, @@ -30,7 +31,6 @@ where { Array::random(shape, Uniform::new(a, b)) * (b.ln() - a.ln()) + a.ln() } - /// Generate a random array of complex numbers with real and imaginary parts in the range [0, 1) pub fn randc(shape: impl IntoDimension) -> Array, D> where @@ -42,30 +42,6 @@ where Array::random(shape, distr) } -pub fn scanner( - a: &Array2, - b: &Array2, - c: &Array2, - u: &Array2, - x0: &Array1, -) -> Array2 -where - T: ComplexFloat + NdFloat, -{ - let step = |xs: &mut Array1, us: ArrayView1| { - let x1 = a.dot(xs) + b.dot(&us); - let y1 = c.dot(&x1.t()); - Some(y1) - }; - let scan = u.outer_iter().scan(x0.clone(), step).collect::>(); - let shape = [scan.len(), scan[0].len()]; - let mut res = Array2::zeros(shape.into_dimension()); - for (i, s) in scan.iter().enumerate() { - res.slice_mut(s![i, ..]).assign(s); - } - res -} - pub(crate) mod fft { use num::{Complex, NumCast}; use realfft::RealFftPlanner; diff --git a/ml/s4/tests/conversion.rs b/ml/s4/tests/conversion.rs new file mode 100644 index 00000000..2900fef8 --- /dev/null +++ b/ml/s4/tests/conversion.rs @@ -0,0 +1,59 @@ +#[cfg(test)] +extern crate concision_s4; + +use concision_core as core; +use concision_s4 as s4; +use s4::ops::scan_ssm; + + +use core::prelude::{assert_atol, randc_normal}; +use s4::prelude::{casual_convolution, discretize_dplr, k_conv, DPLRParams}; +use s4::cmp::kernel::kernel_dplr; +use s4::hippo::dplr::DPLR; + +use ndarray::prelude::*; +use ndarray_linalg::flatten; +use num::complex::{Complex, ComplexFloat}; + +const EPSILON: f64 = 1e-4; +const FEATURES: usize = 8; +const RNGKEY: u64 = 1; +const SAMPLES: usize = 16; + +#[test] +fn test_conversion() { + let step = (SAMPLES as f64).recip(); + // Initialize a new DPLR Matrix + let dplr = DPLR::::new(FEATURES); + let (lambda, p, b, _) = dplr.clone().into(); + + // let c = randcomplex(features); + let c = randc_normal(RNGKEY, FEATURES); + // CNN Form + let kernel = { + let params = + DPLRParams::new(lambda.clone(), p.clone(), p.clone(), b.clone(), c.clone()); + kernel_dplr::(¶ms, step, SAMPLES) + }; + // RNN Form + let discrete = discretize_dplr(&lambda, &p, &p, &b, &c, step, SAMPLES).expect(""); + let (ab, bb, cb) = discrete.into(); + + let k2 = k_conv(&ab, &bb, &cb, SAMPLES); + let k2r = k2.mapv(|i| i.re()); + + assert_atol(&kernel, &k2r, EPSILON); + + let u = Array::range(0.0, SAMPLES as f64, 1.0); + let u2 = u.mapv(|i| Complex::new(i, 0.0)).insert_axis(Axis(1)); + // Apply the CNN + let y1 = casual_convolution(&u, &kernel); + + // Apply the RNN + let x0 = Array::zeros(FEATURES); + let y2 = scan_ssm(&ab, &bb, &cb, &u2, &x0).expect("Failed to scan the SSM"); + let y2r = flatten(y2.mapv(|i| i.re())); + + assert_atol(&y1, &y2r, EPSILON) + +} \ No newline at end of file diff --git a/ml/s4/tests/utils.rs b/ml/s4/tests/utils.rs deleted file mode 100644 index c6954d55..00000000 --- a/ml/s4/tests/utils.rs +++ /dev/null @@ -1,41 +0,0 @@ -#[cfg(test)] -extern crate concision_s4; -use concision_s4 as s4; - -use s4::prelude::{scan_ssm, scanner, SSMStore}; - -use lazy_static::lazy_static; -use ndarray::prelude::*; -use num::Complex; - -const FEATURES: usize = 3; -const SAMPLES: usize = 16; - -lazy_static! { - static ref U: Array2 = Array::range(0.0, FEATURES as f64, 1.0).insert_axis(Axis(1)); - static ref X0: Array1 = Array1::zeros(FEATURES); - static ref X1: Array1> = Array1::zeros(FEATURES); - static ref A: Array2 = Array::range(0.0, (FEATURES * FEATURES) as f64, 1.0) - .into_shape((FEATURES, FEATURES)) - .unwrap(); - static ref B: Array2 = Array::range(0.0, FEATURES as f64, 1.0).insert_axis(Axis(1)); - static ref C: Array2 = Array::range(0.0, FEATURES as f64, 1.0).insert_axis(Axis(0)); - - static ref EXP: Array2 = array![[0.0], [5.0], [70.0]]; -} - -#[test] -fn test_scan() { - let u = U.clone(); - let x0 = X0.clone(); - - let ssm = SSMStore::::from_features(FEATURES); - let (a, b, c, _d) = ssm.clone().into(); - // let scan1 = scanner(&A, &B, &C, &u, &x0); - let scan = scan_ssm(&A, &B, &C, &u, &x0).expect(""); - println!("{:?}", &scan); - - - assert_eq!(&scan, *EXP); - // assert!(false) -}