Add support for on-chain notes to the database

.github/workflows/test.yml

@@ -20,4 +20,4 @@ jobs:
       - name: Install cargo make
         run: cargo install cargo-make
       - name: cargo make - test
-        run: cargo make test
+        run: cargo make test-all

.gitignore

@@ -28,8 +28,8 @@ genesis.dat
 *.sqlite3
 *.sqlite3-shm
 *.sqlite3-wal
-db/
+database/
 
 # Configs
 /genesis.toml
-/miden.toml
+/miden.toml

Makefile.toml

@@ -44,7 +44,7 @@ dependencies = [
 ]
 
 # testing
-[tasks.test]
+[tasks.test-all]
 command = "cargo"
 args = ["test", "--all-features", "--workspace", "--", "--nocapture"]
 

node/miden-node.toml

@@ -17,5 +17,5 @@ store_url = "http://localhost:28943"
 [store]
 # port defined as: sum(ord(c)**p for (p, c) in enumerate('miden-store', 1)) % 2**16
 endpoint = { host = "localhost", port = 28943 }
-database_filepath = "db/miden-store.sqlite3"
+database_filepath = "database/miden-store.sqlite3"
 genesis_filepath = "genesis.dat"

proto/proto/note.proto

@@ -12,6 +12,7 @@ message Note {
     account.AccountId sender  = 4;
     fixed64 tag = 5;
     merkle.MerklePath merkle_path = 7;
+    optional bytes details = 8;
 }
 
 message NoteSyncRecord {
@@ -28,4 +29,5 @@ message NoteCreated {
     digest.Digest note_id = 3;
     account.AccountId sender  = 4;
     fixed64 tag = 5;
-}
+    optional bytes details = 6;
+}

proto/proto/requests.proto

@@ -79,6 +79,11 @@ message SubmitProvenTransactionRequest {
     bytes transaction = 1;
 }
 
+message GetNotesByIdRequest {
+    // List of NoteId's to be queried from the database
+    repeated digest.Digest note_ids = 1;
+}
+
 message ListNullifiersRequest {}
 
 message ListAccountsRequest {}

proto/proto/responses.proto

@@ -99,6 +99,11 @@ message GetTransactionInputsResponse {
 
 message SubmitProvenTransactionResponse {}
 
+message GetNotesByIdResponse {
+    // Lists Note's returned by the database
+    repeated note.Note notes = 1;
+}
+
 message ListNullifiersResponse {
     // Lists all nullifiers of the current chain
     repeated smt.SmtLeafEntry nullifiers = 1;

proto/proto/rpc.proto

@@ -8,6 +8,7 @@ import "responses.proto";
 service Api {
     rpc CheckNullifiers(requests.CheckNullifiersRequest) returns (responses.CheckNullifiersResponse) {}
     rpc GetBlockHeaderByNumber(requests.GetBlockHeaderByNumberRequest) returns (responses.GetBlockHeaderByNumberResponse) {}
+    rpc GetNotesById(requests.GetNotesByIdRequest) returns (responses.GetNotesByIdResponse) {}
     rpc SyncState(requests.SyncStateRequest) returns (responses.SyncStateResponse) {}
     rpc SubmitProvenTransaction(requests.SubmitProvenTransactionRequest) returns (responses.SubmitProvenTransactionResponse) {}
 }

proto/proto/store.proto

@@ -12,6 +12,7 @@ service Api {
     rpc CheckNullifiers(requests.CheckNullifiersRequest) returns (responses.CheckNullifiersResponse) {}
     rpc GetBlockHeaderByNumber(requests.GetBlockHeaderByNumberRequest) returns (responses.GetBlockHeaderByNumberResponse) {}
     rpc GetBlockInputs(requests.GetBlockInputsRequest) returns (responses.GetBlockInputsResponse) {}
+    rpc GetNotesById(requests.GetNotesByIdRequest) returns (responses.GetNotesByIdResponse) {}
     rpc GetTransactionInputs(requests.GetTransactionInputsRequest) returns (responses.GetTransactionInputsResponse) {}
     rpc SyncState(requests.SyncStateRequest) returns (responses.SyncStateResponse) {}
     rpc ListNullifiers(requests.ListNullifiersRequest) returns (responses.ListNullifiersResponse) {}

proto/src/domain/notes.rs

@@ -28,6 +28,7 @@ impl From<(usize, usize, NoteEnvelope)> for note::NoteCreated {
             note_id: Some(note.note_id().into()),
             sender: Some(note.metadata().sender().into()),
             tag: note.metadata().tag().into(),
+            details: None,
         }
     }
 }

proto/src/generated/note.rs

@@ -14,6 +14,8 @@ pub struct Note {
     pub tag: u64,
     #[prost(message, optional, tag = "7")]
     pub merkle_path: ::core::option::Option<super::merkle::MerklePath>,
+    #[prost(bytes = "vec", optional, tag = "8")]
+    pub details: ::core::option::Option<::prost::alloc::vec::Vec<u8>>,
 }
 #[derive(Eq, PartialOrd, Ord, Hash)]
 #[allow(clippy::derive_partial_eq_without_eq)]
@@ -44,4 +46,6 @@ pub struct NoteCreated {
     pub sender: ::core::option::Option<super::account::AccountId>,
     #[prost(fixed64, tag = "5")]
     pub tag: u64,
+    #[prost(bytes = "vec", optional, tag = "6")]
+    pub details: ::core::option::Option<::prost::alloc::vec::Vec<u8>>,
 }

proto/src/generated/requests.rs

@@ -103,6 +103,14 @@ pub struct SubmitProvenTransactionRequest {
 #[derive(Eq, PartialOrd, Ord, Hash)]
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
+pub struct GetNotesByIdRequest {
+    /// List of NoteId's to be queried from the database
+    #[prost(message, repeated, tag = "1")]
+    pub note_ids: ::prost::alloc::vec::Vec<super::digest::Digest>,
+}
+#[derive(Eq, PartialOrd, Ord, Hash)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
 pub struct ListNullifiersRequest {}
 #[derive(Eq, PartialOrd, Ord, Hash)]
 #[allow(clippy::derive_partial_eq_without_eq)]

proto/src/generated/responses.rs

@@ -137,6 +137,14 @@ pub struct SubmitProvenTransactionResponse {}
 #[derive(Eq, PartialOrd, Ord, Hash)]
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
+pub struct GetNotesByIdResponse {
+    /// Lists Note's returned by the database
+    #[prost(message, repeated, tag = "1")]
+    pub notes: ::prost::alloc::vec::Vec<super::note::Note>,
+}
+#[derive(Eq, PartialOrd, Ord, Hash)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
 pub struct ListNullifiersResponse {
     /// Lists all nullifiers of the current chain
     #[prost(message, repeated, tag = "1")]

proto/src/generated/rpc.rs

@@ -134,6 +134,28 @@ pub mod api_client {
                 .insert(GrpcMethod::new("rpc.Api", "GetBlockHeaderByNumber"));
             self.inner.unary(req, path, codec).await
         }
+        pub async fn get_notes_by_id(
+            &mut self,
+            request: impl tonic::IntoRequest<super::super::requests::GetNotesByIdRequest>,
+        ) -> std::result::Result<
+            tonic::Response<super::super::responses::GetNotesByIdResponse>,
+            tonic::Status,
+        > {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::new(
+                        tonic::Code::Unknown,
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/rpc.Api/GetNotesById");
+            let mut req = request.into_request();
+            req.extensions_mut().insert(GrpcMethod::new("rpc.Api", "GetNotesById"));
+            self.inner.unary(req, path, codec).await
+        }
         pub async fn sync_state(
             &mut self,
             request: impl tonic::IntoRequest<super::super::requests::SyncStateRequest>,
@@ -208,6 +230,13 @@ pub mod api_server {
             tonic::Response<super::super::responses::GetBlockHeaderByNumberResponse>,
             tonic::Status,
         >;
+        async fn get_notes_by_id(
+            &self,
+            request: tonic::Request<super::super::requests::GetNotesByIdRequest>,
+        ) -> std::result::Result<
+            tonic::Response<super::super::responses::GetNotesByIdResponse>,
+            tonic::Status,
+        >;
         async fn sync_state(
             &self,
             request: tonic::Request<super::super::requests::SyncStateRequest>,
@@ -403,6 +432,55 @@ pub mod api_server {
                     };
                     Box::pin(fut)
                 }
+                "/rpc.Api/GetNotesById" => {
+                    #[allow(non_camel_case_types)]
+                    struct GetNotesByIdSvc<T: Api>(pub Arc<T>);
+                    impl<
+                        T: Api,
+                    > tonic::server::UnaryService<
+                        super::super::requests::GetNotesByIdRequest,
+                    > for GetNotesByIdSvc<T> {
+                        type Response = super::super::responses::GetNotesByIdResponse;
+                        type Future = BoxFuture<
+                            tonic::Response<Self::Response>,
+                            tonic::Status,
+                        >;
+                        fn call(
+                            &mut self,
+                            request: tonic::Request<
+                                super::super::requests::GetNotesByIdRequest,
+                            >,
+                        ) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move {
+                                <T as Api>::get_notes_by_id(&inner, request).await
+                            };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let inner = inner.0;
+                        let method = GetNotesByIdSvc(inner);
+                        let codec = tonic::codec::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(
+                                accept_compression_encodings,
+                                send_compression_encodings,
+                            )
+                            .apply_max_message_size_config(
+                                max_decoding_message_size,
+                                max_encoding_message_size,
+                            );
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
                 "/rpc.Api/SyncState" => {
                     #[allow(non_camel_case_types)]
                     struct SyncStateSvc<T: Api>(pub Arc<T>);

proto/src/generated/store.rs

@@ -182,6 +182,28 @@ pub mod api_client {
             req.extensions_mut().insert(GrpcMethod::new("store.Api", "GetBlockInputs"));
             self.inner.unary(req, path, codec).await
         }
+        pub async fn get_notes_by_id(
+            &mut self,
+            request: impl tonic::IntoRequest<super::super::requests::GetNotesByIdRequest>,
+        ) -> std::result::Result<
+            tonic::Response<super::super::responses::GetNotesByIdResponse>,
+            tonic::Status,
+        > {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::new(
+                        tonic::Code::Unknown,
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/store.Api/GetNotesById");
+            let mut req = request.into_request();
+            req.extensions_mut().insert(GrpcMethod::new("store.Api", "GetNotesById"));
+            self.inner.unary(req, path, codec).await
+        }
         pub async fn get_transaction_inputs(
             &mut self,
             request: impl tonic::IntoRequest<
@@ -338,6 +360,13 @@ pub mod api_server {
             tonic::Response<super::super::responses::GetBlockInputsResponse>,
             tonic::Status,
         >;
+        async fn get_notes_by_id(
+            &self,
+            request: tonic::Request<super::super::requests::GetNotesByIdRequest>,
+        ) -> std::result::Result<
+            tonic::Response<super::super::responses::GetNotesByIdResponse>,
+            tonic::Status,
+        >;
         async fn get_transaction_inputs(
             &self,
             request: tonic::Request<super::super::requests::GetTransactionInputsRequest>,
@@ -650,6 +679,55 @@ pub mod api_server {
                     };
                     Box::pin(fut)
                 }
+                "/store.Api/GetNotesById" => {
+                    #[allow(non_camel_case_types)]
+                    struct GetNotesByIdSvc<T: Api>(pub Arc<T>);
+                    impl<
+                        T: Api,
+                    > tonic::server::UnaryService<
+                        super::super::requests::GetNotesByIdRequest,
+                    > for GetNotesByIdSvc<T> {
+                        type Response = super::super::responses::GetNotesByIdResponse;
+                        type Future = BoxFuture<
+                            tonic::Response<Self::Response>,
+                            tonic::Status,
+                        >;
+                        fn call(
+                            &mut self,
+                            request: tonic::Request<
+                                super::super::requests::GetNotesByIdRequest,
+                            >,
+                        ) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move {
+                                <T as Api>::get_notes_by_id(&inner, request).await
+                            };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let inner = inner.0;
+                        let method = GetNotesByIdSvc(inner);
+                        let codec = tonic::codec::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(
+                                accept_compression_encodings,
+                                send_compression_encodings,
+                            )
+                            .apply_max_message_size_config(
+                                max_decoding_message_size,
+                                max_encoding_message_size,
+                            );
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
                 "/store.Api/GetTransactionInputs" => {
                     #[allow(non_camel_case_types)]
                     struct GetTransactionInputsSvc<T: Api>(pub Arc<T>);

rpc/README.md

@@ -60,6 +60,18 @@ Retrieves block header by given block number.
 
 * `block_header`: `BlockHeader` – block header.
 
+### GetNotesById
+
+Returns a list of Note's matching a certain list of NoteId's.
+
+**Parameters**
+
+* `note_ids`: `NoteId` - ID of the Note that we want to query.
+
+**Returns**
+
+* `notes`: `Note` - List of notes matching the list of NoteId's.
+
 ### SyncState
 
 Returns info which can be used by the client to sync up to the latest state of the chain