Add "benchmark" test

cedar-drt/Cargo.toml

@@ -21,6 +21,7 @@ smol_str = { version = "0.2", features = ["serde"] }
 
 [dev-dependencies]
 walkdir = "2.4"
+statrs = "0.16"
 
 [dependencies.uuid]
 version = "1.3.1"

cedar-drt/fuzz/fuzz_targets/formatter.rs

@@ -79,7 +79,7 @@ fn contains_unspecified_entities(p: &StaticPolicy) -> bool {
 fn attach_comment(p: &str, uuids: &mut Vec<String>) -> String {
     let mut tokens = lexer::get_token_stream(p).expect("tokens should exist");
     for t in tokens.iter_mut() {
-        let mut ids: Vec<String> = vec![Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4()]
+        let mut ids: Vec<String> = [Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4()]
             .iter()
             .map(|u| u.to_string())
             .collect();

cedar-drt/fuzz/src/lib.rs

@@ -20,8 +20,11 @@ mod prt;
 pub use dump::*;
 pub use prt::*;
 
-use cedar_drt::{time_function, CedarTestImplementation, ErrorComparisonMode};
-use cedar_policy::{frontend::is_authorized::InterfaceResponse, PolicyId};
+pub use cedar_drt::cedar_test_impl::{
+    time_function, CedarTestImplementation, ErrorComparisonMode, TestResult,
+};
+use cedar_policy::frontend::is_authorized::InterfaceResponse;
+use cedar_policy::PolicyId;
 use cedar_policy_core::ast;
 use cedar_policy_core::authorizer::{AuthorizationError, Authorizer, Response};
 use cedar_policy_core::entities::{Entities, NoEntitiesSchema, TCComputation};
@@ -64,23 +67,33 @@ pub fn run_eval_test(
 
     // `custom_impl.interpret()` returns true when the result of evaluating `expr`
     // matches `expected`
-    let definitional_res = custom_impl.interpret(request.clone(), entities, expr, expected.clone());
+    let definitional_res = custom_impl.interpret(
+        &request,
+        entities,
+        expr,
+        enable_extensions,
+        expected.clone(),
+    );
 
-    // TODO(#175): For now, ignore cases where the definitional code returned an error due to
-    // an unknown extension function.
-    if let Err(err) = definitional_res.clone() {
-        if err.contains("jsonToExtFun: unknown extension function") {
-            return;
+    match definitional_res {
+        TestResult::Failure(err) => {
+            // TODO(#175): Ignore cases where the definitional code returned an error due to
+            // an unknown extension function.
+            if err.contains("jsonToExtFun: unknown extension function") {
+                return;
+            }
+            // No other errors are expected
+            panic!("Unexpected error for {request}\nExpression: {expr}\nError: {err}");
+        }
+        TestResult::Success(response) => {
+            // The definitional interpreter response should be `true`
+            assert!(
+                response,
+                "Incorrect evaluation result for {request}\nExpression: {expr}\nEntities:\n{entities}\nExpected value:\n{:?}\n",
+                expected
+            )
         }
     }
-
-    // Otherwise, `definitional_res` should be `Ok(true)`
-    assert_eq!(
-        definitional_res,
-        Ok(true),
-        "Incorrect evaluation result for {request}\nExpression:\n{expr}\nEntities:\n{entities}\nExpected value:\n{:?}\n",
-        expected
-    )
 }
 
 /// Compare the behavior of the authorizer in `cedar-policy` against a custom Cedar
@@ -108,22 +121,22 @@ pub fn run_auth_test(
         return rust_res;
     }
 
-    let definitional_res = custom_impl.is_authorized(request.clone(), policies, entities);
+    let definitional_res = custom_impl.is_authorized(&request, policies, entities);
 
     match definitional_res {
-        Err(err) => {
+        TestResult::Failure(err) => {
             // TODO(#175): For now, ignore cases where the Lean code returned an error due to
             // an unknown extension function.
             if err.contains("jsonToExtFun: unknown extension function") {
                 rust_res
             } else {
                 panic!(
-                "Unexpected parse error for {request}\nPolicies:\n{}\nEntities:\n{}\nError: {err}",
-                &policies, &entities
-            );
+                    "Unexpected error for {request}\nPolicies:\n{}\nEntities:\n{}\nError: {err}",
+                    &policies, &entities
+                );
             }
         }
-        Ok(definitional_res) => {
+        TestResult::Success(definitional_res) => {
             let rust_res_for_comparison: InterfaceResponse = {
                 let errors = match custom_impl.error_comparison_mode() {
                     ErrorComparisonMode::Ignore => HashSet::new(),
@@ -156,7 +169,7 @@ pub fn run_auth_test(
                 )
             };
             assert_eq!(
-                rust_res_for_comparison, definitional_res,
+                rust_res_for_comparison, definitional_res.response,
                 "Mismatch for {request}\nPolicies:\n{}\nEntities:\n{}",
                 &policies, &entities
             );
@@ -190,22 +203,22 @@ pub fn run_val_test(
 
     if rust_res.validation_passed() {
         match definitional_res {
-            Err(err) => {
+            TestResult::Failure(err) => {
                 // TODO(#175): For now, ignore cases where the Lean code returned an error due to
                 // an unknown extension function.
                 if !err.contains("jsonToExtFun: unknown extension function") {
                     panic!(
-                        "Unexpected parse error\nPolicies:\n{}\nSchema:\n{:?}\nError: {err}",
+                        "Unexpected error\nPolicies:\n{}\nSchema:\n{:?}\nError: {err}",
                         &policies, schema
                     );
                 }
             }
-            Ok(definitional_res) => {
+            TestResult::Success(definitional_res) => {
                 // Even if the Rust validator succeeds, the definitional validator may
                 // return "impossiblePolicy" due to greater precision. In this case, the
                 // input policy is well-typed, although it is guaranteed to always evaluate
                 // to false.
-                if definitional_res.validation_errors == vec!["impossiblePolicy".to_string()] {
+                if definitional_res.errors == vec!["impossiblePolicy".to_string()] {
                     return;
                 }
 

cedar-drt/src/cedar_test_impl.rs

@@ -14,66 +14,105 @@
  * limitations under the License.
  */
 
-//! Definition of a general `CedarTestImplementation` trait that describes an
+//! Definition of a `CedarTestImplementation` trait that describes an
 //! implementation of Cedar to use during testing.
 
-use cedar_policy::frontend::is_authorized::InterfaceResponse;
-pub use cedar_policy::Response;
+pub use cedar_policy::frontend::is_authorized::InterfaceResponse;
 use cedar_policy_core::ast::{Expr, PolicySet, Request, Value};
-pub use cedar_policy_core::*;
-pub use cedar_policy_validator::{ValidationMode, ValidationResult, ValidatorSchema};
-pub use entities::Entities;
+use cedar_policy_core::authorizer::Authorizer;
+use cedar_policy_core::entities::Entities;
+use cedar_policy_core::evaluator::Evaluator;
+use cedar_policy_core::extensions::Extensions;
+use cedar_policy_validator::{ValidationMode, Validator, ValidatorSchema};
 use serde::Deserialize;
+use std::collections::HashMap;
+use std::time::{Duration, Instant};
 
-/// Type alias for convenience. Errors are represented as strings to make
-/// (de)serialization as simple as possible. For an `InterfaceResult`, an
-/// error represents a case where the external Cedar implementation failed
-/// to execute the request (e.g., due to a parse error).
-pub type InterfaceResult<T> = std::result::Result<T, String>;
+/// Return type for `CedarTestImplementation` methods
+#[derive(Debug, Deserialize)]
+pub enum TestResult<T> {
+    /// The request succeeded
+    Success(T),
+    /// The request failed (e.g., due to a parse error)
+    Failure(String),
+}
+
+impl<T> TestResult<T> {
+    /// Get the underlying value of a `TestResult`.
+    /// # Panics
+    /// If the `TestResult` is a `Failure`.
+    /// PANIC SAFETY only used in testing code
+    #[allow(clippy::panic)]
+    pub fn expect(self, msg: &str) -> T {
+        match self {
+            Self::Success(t) => t,
+            Self::Failure(err) => panic!("{msg}: {err}"),
+        }
+    }
+}
+
+/// Simple wrapper around u128 to remind ourselves that timing info is in microseconds.
+#[derive(Debug, Deserialize)]
+pub struct Micros(pub u128);
+
+/// Version of `Response` used for testing. Includes an `InterfaceResponse` and
+/// a map with timing information.
+#[derive(Debug, Deserialize)]
+pub struct TestResponse {
+    /// Actual response
+    pub response: InterfaceResponse,
+    /// Timing info in microseconds. This field is a `HashMap` to allow timing
+    /// multiple components (or none at all).
+    pub timing_info: HashMap<String, Micros>,
+}
 
-/// "Interface" type for `ValidationResult` which represents validation
-/// errors as strings.
+/// Version of `ValidationResult` used for testing.
 #[derive(Debug, Deserialize)]
-pub struct InterfaceValidationResult {
-    #[serde(rename = "validationErrors")]
-    pub validation_errors: Vec<String>,
+pub struct TestValidationResult {
+    /// Validation errors
+    pub errors: Vec<String>,
+    /// Timing info in microseconds. This field is a `HashMap` to allow timing
+    /// multiple components (or none at all).
+    pub timing_info: HashMap<String, Micros>,
 }
 
-impl InterfaceValidationResult {
+impl TestValidationResult {
+    /// Check if validation succeeded
     pub fn validation_passed(&self) -> bool {
-        self.validation_errors.is_empty()
+        self.errors.is_empty()
     }
 }
 
-/// A custom implementation of the Cedar authorizer and validator used for testing.
+/// Custom implementation of the Cedar authorizer, evaluator, and validator for testing.
 pub trait CedarTestImplementation {
     /// Custom authorizer entry point.
     fn is_authorized(
         &self,
-        request: Request,
+        request: &Request,
         policies: &PolicySet,
         entities: &Entities,
-    ) -> InterfaceResult<InterfaceResponse>;
+    ) -> TestResult<TestResponse>;
 
     /// Custom evaluator entry point. The bool return value indicates the whether
     /// evaluating the provided expression produces the expected value.
     /// `expected` is optional to allow for the case where no return value is
     /// expected due to errors.
     fn interpret(
         &self,
-        request: Request,
+        request: &Request,
         entities: &Entities,
         expr: &Expr,
+        enable_extensions: bool,
         expected: Option<Value>,
-    ) -> InterfaceResult<bool>;
+    ) -> TestResult<bool>;
 
     /// Custom validator entry point.
     fn validate(
         &self,
         schema: &ValidatorSchema,
         policies: &PolicySet,
         mode: ValidationMode,
-    ) -> InterfaceResult<InterfaceValidationResult>;
+    ) -> TestResult<TestValidationResult>;
 
     /// `ErrorComparisonMode` that should be used for this `CedarTestImplementation`
     fn error_comparison_mode(&self) -> ErrorComparisonMode;
@@ -98,3 +137,98 @@ pub enum ErrorComparisonMode {
     /// exactly match the Rust implementation's error messages' `Display` text.
     Full,
 }
+
+/// Basic struct to support implementing the `CedarTestImplementation` trait
+#[derive(Debug)]
+pub struct RustEngine {}
+
+impl RustEngine {
+    /// Create a new `RustEngine`
+    pub fn new() -> Self {
+        Self {}
+    }
+}
+
+/// Timing function
+pub fn time_function<X, F>(f: F) -> (X, Duration)
+where
+    F: FnOnce() -> X,
+{
+    let start = Instant::now();
+    let result = f();
+    (result, start.elapsed())
+}
+
+/// An implementation of `CedarTestImplementation` using `cedar-policy`.
+/// Used for running integration tests.
+impl CedarTestImplementation for RustEngine {
+    fn is_authorized(
+        &self,
+        request: &Request,
+        policies: &PolicySet,
+        entities: &Entities,
+    ) -> TestResult<TestResponse> {
+        let authorizer = Authorizer::new();
+        let (response, duration) =
+            time_function(|| authorizer.is_authorized(request.clone(), policies, entities));
+        // Error messages should only include the policy id to use the
+        // `ErrorComparisonMode::PolicyIds` mode.
+        let response = cedar_policy::Response::from(response);
+        let response = InterfaceResponse::new(
+            response.decision(),
+            response.diagnostics().reason().cloned().collect(),
+            response
+                .diagnostics()
+                .errors()
+                .map(cedar_policy::AuthorizationError::id)
+                .map(ToString::to_string)
+                .collect(),
+        );
+        let response = TestResponse {
+            response,
+            timing_info: HashMap::from([("authorize".into(), Micros(duration.as_micros()))]),
+        };
+        TestResult::Success(response)
+    }
+
+    fn interpret(
+        &self,
+        request: &Request,
+        entities: &Entities,
+        expr: &Expr,
+        enable_extensions: bool,
+        expected: Option<Value>,
+    ) -> TestResult<bool> {
+        let exts = if enable_extensions {
+            Extensions::all_available()
+        } else {
+            Extensions::none()
+        };
+        let evaluator = Evaluator::new(request.clone(), entities, &exts);
+        let result = evaluator.interpret(expr, &HashMap::default());
+        let response = result.ok() == expected;
+        TestResult::Success(response)
+    }
+
+    fn validate(
+        &self,
+        schema: &ValidatorSchema,
+        policies: &PolicySet,
+        mode: ValidationMode,
+    ) -> TestResult<TestValidationResult> {
+        let validator = Validator::new(schema.clone());
+        let (result, duration) = time_function(|| validator.validate(policies, mode));
+        let response = TestValidationResult {
+            errors: result
+                .validation_errors()
+                .map(|err| format!("{err:?}"))
+                .collect(),
+            timing_info: HashMap::from([("validate".into(), Micros(duration.as_micros()))]),
+        };
+        TestResult::Success(response)
+    }
+
+    fn error_comparison_mode(&self) -> ErrorComparisonMode {
+        ErrorComparisonMode::PolicyIds
+    }
+}