From 3f896a3fc4fccea93b3d90848f9445ac26f29a00 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Wed, 22 Jan 2025 11:41:52 +0100 Subject: [PATCH] wip --- .../grove_apply_partial_batch/mod.rs | 44 ----- .../mod.rs | 59 ------- .../v0/mod.rs | 152 ------------------ .../rs-drive/src/util/grove_operations/mod.rs | 6 - .../mod.rs | 71 -------- .../v0/mod.rs | 77 --------- .../mod.rs | 71 -------- .../v0/mod.rs | 55 ------- packages/rs-drive/src/util/operations/mod.rs | 2 - 9 files changed, 537 deletions(-) delete mode 100644 packages/rs-drive/src/util/grove_operations/grove_apply_partial_batch/mod.rs delete mode 100644 packages/rs-drive/src/util/grove_operations/grove_apply_partial_batch_with_add_costs/mod.rs delete mode 100644 packages/rs-drive/src/util/grove_operations/grove_apply_partial_batch_with_add_costs/v0/mod.rs delete mode 100644 packages/rs-drive/src/util/operations/apply_partial_batch_grovedb_operations/mod.rs delete mode 100644 packages/rs-drive/src/util/operations/apply_partial_batch_grovedb_operations/v0/mod.rs delete mode 100644 packages/rs-drive/src/util/operations/apply_partial_batch_low_level_drive_operations/mod.rs delete mode 100644 packages/rs-drive/src/util/operations/apply_partial_batch_low_level_drive_operations/v0/mod.rs diff --git a/packages/rs-drive/src/util/grove_operations/grove_apply_partial_batch/mod.rs b/packages/rs-drive/src/util/grove_operations/grove_apply_partial_batch/mod.rs deleted file mode 100644 index 8174d224c34..00000000000 --- a/packages/rs-drive/src/util/grove_operations/grove_apply_partial_batch/mod.rs +++ /dev/null @@ -1,44 +0,0 @@ -use crate::drive::Drive; -use crate::error::Error; -use crate::query::GroveError; -use crate::util::batch::GroveDbOpBatch; -use dpp::version::drive_versions::DriveVersion; -use grovedb::batch::{OpsByLevelPath, QualifiedGroveDbOp}; -use grovedb::TransactionArg; -use grovedb_costs::OperationCost; - -impl Drive { - /// Applies the given groveDB operations batch. - /// - /// # Parameters - /// * `ops`: The batch of groveDB operations to apply. - /// * `validate`: Specifies whether to validate that insertions do not override existing entries. - /// * `transaction`: The groveDB transaction associated with this operation. - /// * `add_on_operations`: A closure that takes in the operation cost and optional operation by level path - /// and returns a result of groveDB operations or a grove error. - /// * `drive_version`: The drive version to select the correct function version to run. - /// - /// # Returns - /// * `Ok(())` if the operation was successful. - /// * `Err(DriveError::UnknownVersionMismatch)` if the drive version does not match known versions. - pub fn grove_apply_partial_batch( - &self, - ops: GroveDbOpBatch, - validate: bool, - transaction: TransactionArg, - add_on_operations: impl FnMut( - &OperationCost, - &Option, - ) -> Result, GroveError>, - drive_version: &DriveVersion, - ) -> Result<(), Error> { - self.grove_apply_partial_batch_with_add_costs( - ops, - validate, - transaction, - add_on_operations, - &mut vec![], - drive_version, - ) - } -} diff --git a/packages/rs-drive/src/util/grove_operations/grove_apply_partial_batch_with_add_costs/mod.rs b/packages/rs-drive/src/util/grove_operations/grove_apply_partial_batch_with_add_costs/mod.rs deleted file mode 100644 index 7ba94976f5b..00000000000 --- a/packages/rs-drive/src/util/grove_operations/grove_apply_partial_batch_with_add_costs/mod.rs +++ /dev/null @@ -1,59 +0,0 @@ -mod v0; - -use crate::util::batch::GroveDbOpBatch; - -use crate::drive::Drive; -use crate::error::drive::DriveError; -use crate::error::Error; -use crate::fees::op::LowLevelDriveOperation; -use crate::query::GroveError; - -use dpp::version::drive_versions::DriveVersion; -use grovedb::batch::{OpsByLevelPath, QualifiedGroveDbOp}; -use grovedb::TransactionArg; -use grovedb_costs::OperationCost; - -impl Drive { - /// Applies the given groveDB operations batch, gets and passes the costs to `push_drive_operation_result`. - /// - /// # Parameters - /// * `ops`: The batch of groveDB operations to retrieve costs for. - /// * `validate`: Specifies whether to validate that insertions do not override existing entries. - /// * `transaction`: The groveDB transaction associated with this operation. - /// * `add_on_operations`: A closure that takes in the operation cost and optional operation by level path - /// and returns a result of groveDB operations or a grove error. - /// * `drive_operations`: A vector to collect the costs of operations for later computation. - /// * `drive_version`: The drive version to select the correct function version to run. - /// - /// # Returns - /// * `Ok(())` if the operation was successful. - /// * `Err(DriveError::UnknownVersionMismatch)` if the drive version does not match known versions. - pub fn grove_apply_partial_batch_with_add_costs( - &self, - ops: GroveDbOpBatch, - validate: bool, - transaction: TransactionArg, - add_on_operations: impl FnMut( - &OperationCost, - &Option, - ) -> Result, GroveError>, - drive_operations: &mut Vec, - drive_version: &DriveVersion, - ) -> Result<(), Error> { - match drive_version.grove_methods.apply.grove_apply_partial_batch { - 0 => self.grove_apply_partial_batch_with_add_costs_v0( - ops, - validate, - transaction, - add_on_operations, - drive_operations, - drive_version, - ), - version => Err(Error::Drive(DriveError::UnknownVersionMismatch { - method: "grove_apply_partial_batch_with_add_costs".to_string(), - known_versions: vec![0], - received: version, - })), - } - } -} diff --git a/packages/rs-drive/src/util/grove_operations/grove_apply_partial_batch_with_add_costs/v0/mod.rs b/packages/rs-drive/src/util/grove_operations/grove_apply_partial_batch_with_add_costs/v0/mod.rs deleted file mode 100644 index 9d7a6bc8eba..00000000000 --- a/packages/rs-drive/src/util/grove_operations/grove_apply_partial_batch_with_add_costs/v0/mod.rs +++ /dev/null @@ -1,152 +0,0 @@ -use crate::drive::Drive; -use crate::error::drive::DriveError; -use crate::error::Error; -use crate::fees::op::LowLevelDriveOperation; -use crate::query::GroveError; -use crate::util::batch::grovedb_op_batch::GroveDbOpBatchV0Methods; -use crate::util::batch::GroveDbOpBatch; -use crate::util::grove_operations::push_drive_operation_result; -use crate::util::storage_flags::{MergingOwnersStrategy, StorageFlags}; -use grovedb::batch::{BatchApplyOptions, OpsByLevelPath, QualifiedGroveDbOp}; -use grovedb::TransactionArg; -use grovedb_costs::storage_cost::removal::StorageRemovedBytes::BasicStorageRemoval; -use grovedb_costs::storage_cost::transition::OperationStorageTransitionType; -use grovedb_costs::OperationCost; -use platform_version::version::drive_versions::DriveVersion; - -impl Drive { - /// Applies the given groveDB operations batch and gets and passes the costs to `push_drive_operation_result`. - pub(super) fn grove_apply_partial_batch_with_add_costs_v0( - &self, - ops: GroveDbOpBatch, - validate: bool, - transaction: TransactionArg, - add_on_operations: impl FnMut( - &OperationCost, - &Option, - ) -> Result, GroveError>, - drive_operations: &mut Vec, - drive_version: &DriveVersion, - ) -> Result<(), Error> { - if ops.is_empty() { - return Err(Error::Drive(DriveError::BatchIsEmpty( - "batch is empty when trying to apply partial batch with add costs".to_string(), - ))); - } - // println!("batch {:#?}", ops); - if self.config.batching_consistency_verification { - let consistency_results = - QualifiedGroveDbOp::verify_consistency_of_operations(&ops.operations); - if !consistency_results.is_empty() { - // println!("consistency_results {:#?}", consistency_results); - return Err(Error::Drive(DriveError::GroveDBInsertion( - "insertion order error", - ))); - } - } - - let cost_context = self.grove.apply_partial_batch_with_element_flags_update( - ops.operations, - Some(BatchApplyOptions { - validate_insertion_does_not_override: validate, - validate_insertion_does_not_override_tree: validate, - allow_deleting_non_empty_trees: false, - deleting_non_empty_trees_returns_error: true, - disable_operation_consistency_check: false, - base_root_storage_is_free: true, - batch_pause_height: None, - }), - |cost, old_flags, new_flags| { - // if there were no flags before then the new flags are used - if old_flags.is_none() { - return Ok(false); - } - // This could be none only because the old element didn't exist - // If they were empty we get an error - let maybe_old_storage_flags = StorageFlags::map_some_element_flags_ref(&old_flags) - .map_err(|e| { - GroveError::JustInTimeElementFlagsClientError( - format!("drive did not understand flags of old item being updated {}",e) - ) - })?; - let new_storage_flags = StorageFlags::from_element_flags_ref(new_flags) - .map_err(|e| { - GroveError::JustInTimeElementFlagsClientError( - format!("drive did not understand updated item flag information {}", e) - ) - })? - .ok_or(GroveError::JustInTimeElementFlagsClientError( - "removing flags from an item with flags is not allowed".to_string() - ))?; - match &cost.transition_type() { - OperationStorageTransitionType::OperationUpdateBiggerSize => { - let combined_storage_flags = StorageFlags::optional_combine_added_bytes( - maybe_old_storage_flags, - new_storage_flags, - cost.added_bytes, - MergingOwnersStrategy::RaiseIssue, - ) - .map_err(|e| { - GroveError::JustInTimeElementFlagsClientError( - format!("drive could not combine storage flags (new flags were bigger): {}", e) - ) - })?; - let combined_flags = combined_storage_flags.to_element_flags(); - // it's possible they got bigger in the same epoch - if combined_flags == *new_flags { - // they are the same there was no update - Ok(false) - } else { - *new_flags = combined_flags; - Ok(true) - } - } - OperationStorageTransitionType::OperationUpdateSmallerSize => { - let combined_storage_flags = StorageFlags::optional_combine_removed_bytes( - maybe_old_storage_flags, - new_storage_flags, - &cost.removed_bytes, - MergingOwnersStrategy::RaiseIssue, - ) - .map_err(|e| { - GroveError::JustInTimeElementFlagsClientError( - format!("drive could not combine storage flags (new flags were smaller): {}",e) - ) - })?; - let combined_flags = combined_storage_flags.to_element_flags(); - // it's possible they got bigger in the same epoch - if combined_flags == *new_flags { - // they are the same there was no update - Ok(false) - } else { - *new_flags = combined_flags; - Ok(true) - } - } - _ => Ok(false), - } - }, - |flags, removed_key_bytes, removed_value_bytes| { - let maybe_storage_flags = - StorageFlags::from_element_flags_ref(flags).map_err(|e| { - GroveError::SplitRemovalBytesClientError( - format!("drive did not understand flags of item being updated: {}",e) - ) - })?; - // if there were no flags before then the new flags are used - match maybe_storage_flags { - None => Ok(( - BasicStorageRemoval(removed_key_bytes), - BasicStorageRemoval(removed_value_bytes), - )), - Some(storage_flags) => Ok(storage_flags - .split_storage_removed_bytes(removed_key_bytes, removed_value_bytes)), - } - }, - add_on_operations, - transaction, - &drive_version.grove_version - ); - push_drive_operation_result(cost_context, drive_operations) - } -} diff --git a/packages/rs-drive/src/util/grove_operations/mod.rs b/packages/rs-drive/src/util/grove_operations/mod.rs index 098d8259cde..739264402fd 100644 --- a/packages/rs-drive/src/util/grove_operations/mod.rs +++ b/packages/rs-drive/src/util/grove_operations/mod.rs @@ -102,12 +102,6 @@ pub mod grove_apply_batch; /// Apply batch grove operation with additional costs pub mod grove_apply_batch_with_add_costs; -/// Apply partial batch grove operation -pub mod grove_apply_partial_batch; - -/// Apply partial batch grove operation with additional costs -pub mod grove_apply_partial_batch_with_add_costs; - /// Get cost of grove batch operations pub mod grove_batch_operations_costs; diff --git a/packages/rs-drive/src/util/operations/apply_partial_batch_grovedb_operations/mod.rs b/packages/rs-drive/src/util/operations/apply_partial_batch_grovedb_operations/mod.rs deleted file mode 100644 index 20230350437..00000000000 --- a/packages/rs-drive/src/util/operations/apply_partial_batch_grovedb_operations/mod.rs +++ /dev/null @@ -1,71 +0,0 @@ -mod v0; - -use crate::drive::Drive; -use crate::error::{drive::DriveError, Error}; -use crate::fees::op::LowLevelDriveOperation; -use crate::query::GroveError; -use crate::util::batch::GroveDbOpBatch; - -use dpp::version::drive_versions::DriveVersion; -use grovedb::batch::{KeyInfoPath, OpsByLevelPath, QualifiedGroveDbOp}; -use grovedb::{EstimatedLayerInformation, TransactionArg}; -use grovedb_costs::OperationCost; -use std::collections::HashMap; - -impl Drive { - /// Applies a partial batch of groveDB operations depending on the drive version. - /// - /// This method checks the drive version and calls the appropriate versioned method. - /// If an unsupported version is passed, the function will return an `Error::Drive` with a `DriveError::UnknownVersionMismatch` error. - /// - /// # Arguments - /// - /// * `estimated_costs_only_with_layer_info` - An optional hashmap containing estimated layer information. - /// * `transaction` - The transaction argument to pass to the groveDB. - /// * `batch_operations` - A groveDB operation batch. - /// * `add_on_operations` - A closure that processes additional operations. - /// * `drive_operations` - A mutable reference to a vector of drive operations. - /// * `drive_version` - A `DriveVersion` reference that dictates which version of the method to call. - /// - /// # Returns - /// - /// * `Result<(), Error>` - On success, returns `Ok(())`. On error, returns an `Error`. - /// - #[allow(dead_code)] - #[deprecated(note = "This function is marked as unused.")] - #[allow(deprecated)] - pub(crate) fn apply_partial_batch_grovedb_operations( - &self, - estimated_costs_only_with_layer_info: Option< - HashMap, - >, - transaction: TransactionArg, - batch_operations: GroveDbOpBatch, - add_on_operations: impl FnMut( - &OperationCost, - &Option, - ) -> Result, GroveError>, - drive_operations: &mut Vec, - drive_version: &DriveVersion, - ) -> Result<(), Error> { - match drive_version - .methods - .operations - .apply_partial_batch_grovedb_operations - { - 0 => self.apply_partial_batch_grovedb_operations_v0( - estimated_costs_only_with_layer_info, - transaction, - batch_operations, - add_on_operations, - drive_operations, - drive_version, - ), - version => Err(Error::Drive(DriveError::UnknownVersionMismatch { - method: "apply_partial_batch_grovedb_operations".to_string(), - known_versions: vec![0], - received: version, - })), - } - } -} diff --git a/packages/rs-drive/src/util/operations/apply_partial_batch_grovedb_operations/v0/mod.rs b/packages/rs-drive/src/util/operations/apply_partial_batch_grovedb_operations/v0/mod.rs deleted file mode 100644 index 129f74eb746..00000000000 --- a/packages/rs-drive/src/util/operations/apply_partial_batch_grovedb_operations/v0/mod.rs +++ /dev/null @@ -1,77 +0,0 @@ -use crate::drive::Drive; -use crate::error::Error; -use crate::fees::op::LowLevelDriveOperation; -use crate::query::GroveError; -use crate::util::batch::grovedb_op_batch::GroveDbOpBatchV0Methods; -use crate::util::batch::GroveDbOpBatch; -use dpp::version::drive_versions::DriveVersion; -use grovedb::batch::{KeyInfoPath, OpsByLevelPath, QualifiedGroveDbOp}; -use grovedb::{EstimatedLayerInformation, TransactionArg}; -use grovedb_costs::storage_cost::StorageCost; -use grovedb_costs::OperationCost; -use std::collections::HashMap; - -impl Drive { - #[allow(dead_code)] - #[deprecated(note = "This function is marked as unused.")] - #[allow(deprecated)] - /// Applies a partial batch of groveDB operations if apply is True, otherwise gets the cost of the operations. - pub(crate) fn apply_partial_batch_grovedb_operations_v0( - &self, - estimated_costs_only_with_layer_info: Option< - HashMap, - >, - transaction: TransactionArg, - mut batch_operations: GroveDbOpBatch, - mut add_on_operations: impl FnMut( - &OperationCost, - &Option, - ) -> Result, GroveError>, - drive_operations: &mut Vec, - drive_version: &DriveVersion, - ) -> Result<(), Error> { - if let Some(estimated_layer_info) = estimated_costs_only_with_layer_info { - // Leave this for future debugging - // for (k, v) in estimated_layer_info.iter() { - // let path = k - // .to_path() - // .iter() - // .map(|k| hex::encode(k.as_slice())) - // .join("/"); - // dbg!(path, v); - // } - // the estimated fees are the same for partial batches - let additional_operations = add_on_operations( - &OperationCost { - seek_count: 1, - storage_cost: StorageCost { - added_bytes: 1, - replaced_bytes: 1, - removed_bytes: Default::default(), - }, - storage_loaded_bytes: 1, - hash_node_calls: 1, - }, - &None, - )?; - batch_operations.extend(additional_operations); - self.grove_batch_operations_costs( - batch_operations, - estimated_layer_info, - false, - drive_operations, - drive_version, - )?; - } else { - self.grove_apply_partial_batch_with_add_costs( - batch_operations, - false, - transaction, - add_on_operations, - drive_operations, - drive_version, - )?; - } - Ok(()) - } -} diff --git a/packages/rs-drive/src/util/operations/apply_partial_batch_low_level_drive_operations/mod.rs b/packages/rs-drive/src/util/operations/apply_partial_batch_low_level_drive_operations/mod.rs deleted file mode 100644 index e97d8d7145a..00000000000 --- a/packages/rs-drive/src/util/operations/apply_partial_batch_low_level_drive_operations/mod.rs +++ /dev/null @@ -1,71 +0,0 @@ -mod v0; - -use crate::drive::Drive; -use crate::error::drive::DriveError; -use crate::error::Error; -use crate::fees::op::LowLevelDriveOperation; - -use crate::query::GroveError; -use dpp::version::drive_versions::DriveVersion; -use grovedb::batch::{KeyInfoPath, OpsByLevelPath}; -use grovedb::{EstimatedLayerInformation, TransactionArg}; -use grovedb_costs::OperationCost; -use std::collections::HashMap; - -impl Drive { - /// Applies a partial batch of low level drive operations depending on the drive version. - /// - /// This method checks the drive version and calls the appropriate versioned method. - /// If an unsupported version is passed, the function will return an `Error::Drive` with a `DriveError::UnknownVersionMismatch` error. - /// - /// # Arguments - /// - /// * `estimated_costs_only_with_layer_info` - An optional hashmap containing estimated layer information. - /// * `transaction` - The transaction argument to pass to the groveDB. - /// * `batch_operations` - A vector of low level drive operations. - /// * `add_on_operations` - A closure that processes additional operations. - /// * `drive_operations` - A mutable reference to a vector of drive operations. - /// * `drive_version` - A `DriveVersion` reference that dictates which version of the method to call. - /// - /// # Returns - /// - /// * `Result<(), Error>` - On success, returns `Ok(())`. On error, returns an `Error`. - /// - #[allow(dead_code)] - #[deprecated(note = "This function is marked as unused.")] - #[allow(deprecated)] - pub(crate) fn apply_partial_batch_low_level_drive_operations( - &self, - estimated_costs_only_with_layer_info: Option< - HashMap, - >, - transaction: TransactionArg, - batch_operations: Vec, - add_on_operations: impl FnMut( - &OperationCost, - &Option, - ) -> Result, GroveError>, - drive_operations: &mut Vec, - drive_version: &DriveVersion, - ) -> Result<(), Error> { - match drive_version - .methods - .operations - .apply_partial_batch_low_level_drive_operations - { - 0 => self.apply_partial_batch_low_level_drive_operations_v0( - estimated_costs_only_with_layer_info, - transaction, - batch_operations, - add_on_operations, - drive_operations, - drive_version, - ), - version => Err(Error::Drive(DriveError::UnknownVersionMismatch { - method: "apply_partial_batch_low_level_drive_operations".to_string(), - known_versions: vec![0], - received: version, - })), - } - } -} diff --git a/packages/rs-drive/src/util/operations/apply_partial_batch_low_level_drive_operations/v0/mod.rs b/packages/rs-drive/src/util/operations/apply_partial_batch_low_level_drive_operations/v0/mod.rs deleted file mode 100644 index c0b7b295c0f..00000000000 --- a/packages/rs-drive/src/util/operations/apply_partial_batch_low_level_drive_operations/v0/mod.rs +++ /dev/null @@ -1,55 +0,0 @@ -use crate::drive::Drive; -use crate::error::Error; -use crate::fees::op::LowLevelDriveOperation; -use crate::fees::op::LowLevelDriveOperation::GroveOperation; -use crate::query::GroveError; -use dpp::version::drive_versions::DriveVersion; -use grovedb::batch::{KeyInfoPath, OpsByLevelPath}; -use grovedb::{EstimatedLayerInformation, TransactionArg}; -use grovedb_costs::OperationCost; -use std::collections::HashMap; - -impl Drive { - //this will be used later - /// Applies a batch of Drive operations to groveDB. - #[allow(dead_code)] - #[deprecated(note = "This function is marked as unused.")] - #[allow(deprecated)] - pub(crate) fn apply_partial_batch_low_level_drive_operations_v0( - &self, - estimated_costs_only_with_layer_info: Option< - HashMap, - >, - transaction: TransactionArg, - batch_operations: Vec, - mut add_on_operations: impl FnMut( - &OperationCost, - &Option, - ) -> Result, GroveError>, - drive_operations: &mut Vec, - drive_version: &DriveVersion, - ) -> Result<(), Error> { - let grove_db_operations = - LowLevelDriveOperation::grovedb_operations_batch(&batch_operations); - self.apply_partial_batch_grovedb_operations( - estimated_costs_only_with_layer_info, - transaction, - grove_db_operations, - |cost, left_over_ops| { - let additional_low_level_drive_operations = add_on_operations(cost, left_over_ops)?; - let new_grove_db_operations = LowLevelDriveOperation::grovedb_operations_batch( - &additional_low_level_drive_operations, - ) - .operations; - Ok(new_grove_db_operations) - }, - drive_operations, - drive_version, - )?; - batch_operations.into_iter().for_each(|op| match op { - GroveOperation(_) => (), - _ => drive_operations.push(op), - }); - Ok(()) - } -} diff --git a/packages/rs-drive/src/util/operations/mod.rs b/packages/rs-drive/src/util/operations/mod.rs index f67ada2efa3..cc8dc62a2f1 100644 --- a/packages/rs-drive/src/util/operations/mod.rs +++ b/packages/rs-drive/src/util/operations/mod.rs @@ -1,7 +1,5 @@ mod apply_batch_grovedb_operations; mod apply_batch_low_level_drive_operations; -mod apply_partial_batch_grovedb_operations; -mod apply_partial_batch_low_level_drive_operations; mod commit_transaction; mod drop_cache; mod rollback_transaction;