From 2807cdb7c7da3538c117591c718f5209474aab3f Mon Sep 17 00:00:00 2001 From: Almann Goo Date: Sat, 5 Jun 2021 12:38:22 -0700 Subject: [PATCH 1/3] Adds Pest Grammar to Ion Conversion The purpose of this commit is the start of a tool to generate Ion formatted ASTs from Pest grammars, particularly that of PartiQL. We could then easily process the grammar outside of Rust in scripts that could be used to generate things like TextMate grammars for IDE support or LaTeX fragments for the specification document. Adds the `pestion` crate to provide a simple trait `PestToElement` and implementations over the Pest AST and `&str` syntax definitions to create Ion `Element` serialized forms. Resolves #35. --- Cargo.toml | 1 + pestion/Cargo.toml | 29 +++ pestion/README.md | 9 + pestion/src/lib.rs | 443 ++++++++++++++++++++++++++++++++++++++++++ pestion/src/result.rs | 29 +++ 5 files changed, 511 insertions(+) create mode 100644 pestion/Cargo.toml create mode 100644 pestion/README.md create mode 100644 pestion/src/lib.rs create mode 100644 pestion/src/result.rs diff --git a/Cargo.toml b/Cargo.toml index 81b4b7f0..8864d5cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,4 +6,5 @@ members = [ "partiql-irgen", "partiql-parser", "partiql-rewriter", + "pestion", ] diff --git a/pestion/Cargo.toml b/pestion/Cargo.toml new file mode 100644 index 00000000..eb77c0a1 --- /dev/null +++ b/pestion/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "pestion" +authors = ["PartiQL Team "] +description = "A simple Pest grammar to Ion converter" +homepage = "https://github.com/partiql/partiql-lang-rust/pestion" +repository = "https://github.com/partiql/partiql-lang-rust/pestion" +license = "Apache-2.0" +readme = "README.md" +keywords = ["parser", "peg", "pest", "ion", "cli"] +categories = ["parser-implementations", "command-line-utilities"] +exclude = [ + "**/.git/**", + "**/.github/**", + "**/.travis.yml", + "**/.appveyor.yml", +] +edition = "2018" +version = "0.0.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +thiserror = "~1.0.25" +pest = "~2.1.3" +pest_meta = "~2.1.3" +ion-rs = "~0.6.0" + +[dev-dependencies] +rstest = "~0.10.0" diff --git a/pestion/README.md b/pestion/README.md new file mode 100644 index 00000000..8b7fa2d3 --- /dev/null +++ b/pestion/README.md @@ -0,0 +1,9 @@ +# Pestion + +This is a simple tool and library for converting [Pest] grammars to [Ion] data format. + +The motivation for this is to make a portable way to introspect [Pest] grammars in other tools +as a data format versus having to provide bespoke parsers for the Pest syntax in other platforms. + +[Pest]: https://pest.rs/ +[Ion]: https://amzn.github.io/ion-docs/ \ No newline at end of file diff --git a/pestion/src/lib.rs b/pestion/src/lib.rs new file mode 100644 index 00000000..774ec339 --- /dev/null +++ b/pestion/src/lib.rs @@ -0,0 +1,443 @@ +// Copyright Amazon.com, Inc. or its affiliates. + +//! Provides simple conversion from [Pest] grammar syntax to Amazon [Ion]. +//! +//! ## Example +//! +//! The easiest way to convert [Pest] grammars to Ion is from a `str` slice: +//! +//! ``` +//! use pestion::*; +//! use ion_rs::value::*; +//! +//! fn main() -> PestionResult<()> { +//! // parse a Pest grammar and convert it to Ion element +//! let element = r#"a = @{ "a" | "b" ~ "c" }"#.pest_to_element()?; +//! +//! // the grammar is a struct containing a field for each rule +//! let a_rule = element +//! .as_struct() +//! .and_then(|s| s.get("a")) +//! .and_then(|r| r.as_struct()).unwrap(); +//! +//! // The '@' in the start of the rule means it is atomic +//! assert_eq!("atomic", a_rule.get("type").and_then(|t| t.as_str()).unwrap()); +//! +//! // the first node in the expression tree is a `choice` operator +//! assert_eq!( +//! "choice", +//! a_rule +//! .get("expression") +//! .and_then(|e| e.as_sequence()) +//! .and_then(|s| s.get(0)) +//! .and_then(|h| h.as_str()).unwrap() +//! ); +//! +//! Ok(()) +//! } +//! ``` +//! +//! [Pest]: https://pest.rs/ +//! [Ion]: https://amzn.github.io/ion-docs/ + +pub mod result; + +pub use result::*; + +use ion_rs::value::owned::{text_token, OwnedElement, OwnedValue}; +use ion_rs::value::{Builder, Element}; +use pest::Parser; +use pest_meta::ast::{Expr, Rule as AstRule, RuleType as AstRuleType, RuleType}; +use pest_meta::parser::{consume_rules, PestParser, Rule}; + +/// Converts a representation of a Pest grammar (or part of a grammar) into Ion [`Element`]. +pub trait PestToElement { + type Element: Element; + + /// Converts this into [`Element`] which may imply parsing Pest syntax. + fn pest_to_element(&self) -> PestionResult; +} + +impl PestToElement for &str { + type Element = OwnedElement; + + /// Parses a `str` slice as a Pest grammar and serializes the AST into [`Element`]. + fn pest_to_element(&self) -> PestionResult { + let pairs = PestParser::parse(Rule::grammar_rules, *self)?; + let ast = match consume_rules(pairs) { + Ok(ast) => ast, + Err(errors) => { + return if errors.is_empty() { + invalid("Error converting Pest grammar to AST with no context") + } else { + // TODO deal with more than one error.. + let err = errors.into_iter().next().unwrap(); + Err(err.into()) + }; + } + }; + + ast.pest_to_element() + } +} + +impl PestToElement for Vec { + type Element = OwnedElement; + + /// Converts a body of rules into a `struct` that has a rule for each field. + fn pest_to_element(&self) -> PestionResult { + let mut fields = vec![]; + for rule in self.iter() { + let rule_name = text_token(rule.name.clone()); + let rule_value = rule.pest_to_element()?; + fields.push((rule_name, rule_value)); + } + Ok(Self::Element::new_struct(fields)) + } +} + +impl PestToElement for AstRule { + type Element = OwnedElement; + + /// Converts a Pest Rule into a `struct` that has the field for [`RuleType`] as a symbol + /// and a field for the [`Expr`]. + fn pest_to_element(&self) -> PestionResult { + let fields = vec![ + (text_token("type"), self.ty.pest_to_element()?), + (text_token("expression"), self.expr.pest_to_element()?), + ]; + Ok(Self::Element::new_struct(fields)) + } +} + +impl PestToElement for AstRuleType { + type Element = OwnedElement; + + /// Serializes the enum into a symbolic value. + fn pest_to_element(&self) -> PestionResult { + let sym_tok = text_token(match self { + RuleType::Normal => "normal", + RuleType::Silent => "silent", + RuleType::Atomic => "atomic", + RuleType::CompoundAtomic => "compound_atomic", + RuleType::NonAtomic => "non_atomic", + }); + + Ok(sym_tok.into()) + } +} + +impl PestToElement for Expr { + type Element = OwnedElement; + + /// Generates a `sexp` representation of the rule expression. + fn pest_to_element(&self) -> PestionResult { + use OwnedValue::*; + + let element = Self::Element::new_sexp(match self.clone() { + Expr::Str(text) => vec![ + text_token("string").into(), + text_token("exact").into(), + String(text).into(), + ], + Expr::Insens(text) => vec![ + text_token("string").into(), + text_token("insensitive").into(), + String(text).into(), + ], + Expr::Range(begin, end) => vec![ + text_token("character_range").into(), + String(begin).into(), + String(end).into(), + ], + Expr::Ident(name) => vec![text_token("identifier").into(), String(name).into()], + Expr::PosPred(expr) => vec![ + text_token("predicate").into(), + text_token("positive").into(), + expr.pest_to_element()?, + ], + Expr::NegPred(expr) => vec![ + text_token("predicate").into(), + text_token("negative").into(), + expr.pest_to_element()?, + ], + Expr::Seq(left, right) => vec![ + text_token("sequence").into(), + left.pest_to_element()?, + right.pest_to_element()?, + ], + Expr::Choice(left, right) => vec![ + text_token("choice").into(), + left.pest_to_element()?, + right.pest_to_element()?, + ], + Expr::Opt(expr) => vec![text_token("optional").into(), expr.pest_to_element()?], + Expr::Rep(expr) => vec![ + text_token("repeat_min").into(), + 0.into(), + expr.pest_to_element()?, + ], + Expr::RepOnce(expr) => vec![ + text_token("repeat_min").into(), + 1.into(), + expr.pest_to_element()?, + ], + Expr::RepMin(expr, min) => vec![ + text_token("repeat_min").into(), + (min as i64).into(), + expr.pest_to_element()?, + ], + Expr::RepMax(expr, max) => vec![ + text_token("repeat_max").into(), + (max as i64).into(), + expr.pest_to_element()?, + ], + Expr::RepExact(expr, exact) => vec![ + text_token("repeat_range").into(), + (exact as i64).into(), + (exact as i64).into(), + expr.pest_to_element()?, + ], + Expr::RepMinMax(expr, min, max) => vec![ + text_token("repeat_range").into(), + (min as i64).into(), + (max as i64).into(), + expr.pest_to_element()?, + ], + // TODO implement these + Expr::Skip(_) => unimplemented!(), + Expr::Push(_) => unimplemented!(), + Expr::PeekSlice(_, _) => unimplemented!(), + }); + + Ok(element) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ion_rs::value::reader::*; + use ion_rs::value::writer::*; + use rstest::*; + use std::fmt::Debug; + use std::str::from_utf8; + + #[rstest] + #[case::string( + r#"a = { "hello" }"#, + r#" + { + a: { + type: normal, + expression: (string exact "hello") + } + }"# + )] + #[case::case_insensitive_string_atomic( + r#"a = @{ ^"world" }"#, + r#" + { + a: { + type: atomic, + expression: (string insensitive "world") + } + }"# + )] + #[case::range_silent( + r#"a = _{ 'a'..'z' }"#, + r#" + { + a: { + type: silent, + expression: (character_range "a" "z") + } + }"# + )] + #[case::range_identifier_compound( + r#"a = ${ ANY }"#, + r#" + { + a: { + type: compound_atomic, + expression: (identifier "ANY") + } + }"# + )] + #[case::predicates_non_atomic( + r#"a = !{ &(b) } + b = !{ !"hi" }"#, + r#" + { + a: { + type: non_atomic, + expression: (predicate positive (identifier "b")) + }, + b: { + type: non_atomic, + expression: (predicate negative (string exact "hi")) + } + }"# + )] + #[case::sequence( + r#"a = { "a" ~ ^"b" ~ "c" }"#, + r#" + { + a: { + type: normal, + expression: + (sequence + (sequence + (string exact "a") + (string insensitive "b") + ) + (string exact "c") + ) + } + }"# + )] + #[case::choice( + r#"a = { "a" | ^"b" | "c" }"#, + r#" + { + a: { + type: normal, + expression: + (choice + (choice + (string exact "a") + (string insensitive "b") + ) + (string exact "c") + ) + } + }"# + )] + #[case::mix_choice_seq( + r#"a = { "a" ~ ^"b" | "c" ~ ^"d" ~ "e" | "f" ~ "g" }"#, + r#" + { + a: { + type: normal, + expression: + (choice + (choice + (sequence + (string exact "a") + (string insensitive "b") + ) + (sequence + (sequence + (string exact "c") + (string insensitive "d") + ) + (string exact "e") + ) + ) + (sequence + (string exact "f") + (string exact "g") + ) + ) + } + }"# + )] + #[case::optional( + r#"a = { "a"? }"#, + r#" + { + a: { + type: normal, + expression: (optional (string exact "a")) + } + }"# + )] + #[case::repeat_min( + r#"a = { "a"* } + b = { "b"+ } + c = { "c"{1,} } + d = { "d"{2,} }"#, + r#" + { + a: { + type: normal, + expression: (repeat_min 0 (string exact "a")) + }, + b: { + type: normal, + expression: (repeat_min 1 (string exact "b")) + }, + c: { + type: normal, + expression: (repeat_min 1 (string exact "c")) + }, + d: { + type: normal, + expression: (repeat_min 2 (string exact "d")) + }, + }"# + )] + #[case::repeat_max( + r#"a = { "a"{,100} }"#, + r#" + { + a: { + type: normal, + expression: (repeat_max 100 (string exact "a")) + }, + }"# + )] + #[case::repeat_range( + r#"a = { "a"{5} ~ "b"{7, 10} }"#, + r#" + { + a: { + type: normal, + expression: + (sequence + (repeat_range 5 5 (string exact "a")) + (repeat_range 7 10 (string exact "b")) + ) + }, + }"# + )] + fn good(#[case] input: T, #[case] ion_literal: S) -> PestionResult<()> + where + T: PestToElement + Debug, + S: AsRef, + { + let actual = input.pest_to_element()?; + let expected = element_reader().read_one(ion_literal.as_ref().as_bytes())?; + + const BUF_SIZE: usize = 16 * 1024 * 1024; + let mut buf = vec![0u8; BUF_SIZE]; + let mut writer = Format::Text(TextKind::Pretty).element_writer_for_slice(&mut buf)?; + writer.write(&actual)?; + let actual_converted_text = from_utf8(writer.finish()?).unwrap(); + + assert_eq!( + expected, + actual, + "Expected \n{}\nbut was\n{}", + ion_literal.as_ref(), + actual_converted_text + ); + Ok(()) + } + + /// The goal here is not to test Pest's meta parsing, but just to ensure that we get errors + /// from our APIs when we expect to. + #[rstest] + #[case::empty_rule(r#"a = {}"#)] + #[case::self_reference(r#"a = { a }"#)] + #[case::double_rule(r#"a = { "a" }\n a = { "b" }"#)] + fn pest_errors(#[case] input: T) -> PestionResult<()> { + match input.pest_to_element() { + Err(PestionError::Pest(_)) => {} + something => { + unreachable!("Got result we did not expect: {:?}", something); + } + } + Ok(()) + } +} diff --git a/pestion/src/result.rs b/pestion/src/result.rs new file mode 100644 index 00000000..a1e069a4 --- /dev/null +++ b/pestion/src/result.rs @@ -0,0 +1,29 @@ +// Copyright Amazon.com, Inc. or its affiliates. + +//! [`Error`] and [`Result`] types for working with Pest to Ion. + +use thiserror::Error; + +/// Main [`Result`] type for Pest to Ion. +pub type PestionResult = Result; + +/// Error type for problems in this crate. +#[derive(Error, Debug)] +pub enum PestionError { + /// An error working with [`pest_meta`]. + #[error("Pest Error: {0}")] + Pest(#[from] pest::error::Error), + + /// An error working with [`ion_rs`]. + #[error("Ion Error: {0}")] + Ion(#[from] ion_rs::result::IonError), + + /// General error from this library. + #[error("Pestion Error: {0}")] + Invalid(String), +} + +/// Convenience function to create a general error result. +pub fn invalid>(message: S) -> PestionResult { + Err(PestionError::Invalid(message.into())) +} From 783de7fae7c9f175f33af6e661a8c843cc49037a Mon Sep 17 00:00:00 2001 From: Almann Goo Date: Mon, 7 Jun 2021 16:38:10 -0700 Subject: [PATCH 2/3] Adds fixes to avoid heap allocations As per feedback from @zslayton. Renames `PestToElement` to `TryPestToElement` which captures the fallible conversion (e.g. from `str` slices) and makes `PestToElement` be an infallible version of the API. This is a more correct factoring, but also needed to avoid the intermediate copy for `Vec` conversion. Changes `AstRule` structure generation to use an array as it has a fixed number of fields. Adds `smallvec` to make it easier to model dynamically sized yet stack allocated fields for sub-structures for `Expr` conversion. Adjusts the doc test to be more illustrative. --- pestion/Cargo.toml | 1 + pestion/src/lib.rs | 169 +++++++++++++++++++++++++-------------------- 2 files changed, 96 insertions(+), 74 deletions(-) diff --git a/pestion/Cargo.toml b/pestion/Cargo.toml index eb77c0a1..5c08fcd5 100644 --- a/pestion/Cargo.toml +++ b/pestion/Cargo.toml @@ -24,6 +24,7 @@ thiserror = "~1.0.25" pest = "~2.1.3" pest_meta = "~2.1.3" ion-rs = "~0.6.0" +smallvec = "~1.6.1" [dev-dependencies] rstest = "~0.10.0" diff --git a/pestion/src/lib.rs b/pestion/src/lib.rs index 774ec339..17e66f35 100644 --- a/pestion/src/lib.rs +++ b/pestion/src/lib.rs @@ -9,29 +9,30 @@ //! ``` //! use pestion::*; //! use ion_rs::value::*; +//! use ion_rs::value::reader::*; //! //! fn main() -> PestionResult<()> { //! // parse a Pest grammar and convert it to Ion element -//! let element = r#"a = @{ "a" | "b" ~ "c" }"#.pest_to_element()?; +//! let actual = r#"a = @{ "a" | "b" ~ "c" }"#.try_pest_to_element()?; //! -//! // the grammar is a struct containing a field for each rule -//! let a_rule = element -//! .as_struct() -//! .and_then(|s| s.get("a")) -//! .and_then(|r| r.as_struct()).unwrap(); +//! // here is the equivalent Ion element +//! let ion_text = r#"{ +//! a: { +//! type: atomic, +//! expression: +//! (choice +//! (string exact "a") +//! (sequence +//! (string exact "b") +//! (string exact "c") +//! ) +//! ) +//! } +//! }"#; +//! let expected = element_reader().read_one(ion_text.as_bytes())?; //! -//! // The '@' in the start of the rule means it is atomic -//! assert_eq!("atomic", a_rule.get("type").and_then(|t| t.as_str()).unwrap()); -//! -//! // the first node in the expression tree is a `choice` operator -//! assert_eq!( -//! "choice", -//! a_rule -//! .get("expression") -//! .and_then(|e| e.as_sequence()) -//! .and_then(|s| s.get(0)) -//! .and_then(|h| h.as_str()).unwrap() -//! ); +//! // these should be equivalent +//! assert_eq!(expected, actual); //! //! Ok(()) //! } @@ -49,20 +50,35 @@ use ion_rs::value::{Builder, Element}; use pest::Parser; use pest_meta::ast::{Expr, Rule as AstRule, RuleType as AstRuleType, RuleType}; use pest_meta::parser::{consume_rules, PestParser, Rule}; +use smallvec::{smallvec, SmallVec}; -/// Converts a representation of a Pest grammar (or part of a grammar) into Ion [`Element`]. -pub trait PestToElement { +/// Converts representation of a Pest grammar (or part of a grammar) into Ion [`Element`]. +pub trait TryPestToElement { type Element: Element; /// Converts this into [`Element`] which may imply parsing Pest syntax. - fn pest_to_element(&self) -> PestionResult; + /// + /// This returns `Err` if the the conversion fails. For example, this can happen if the + /// source is not a valid Pest grammar. + fn try_pest_to_element(&self) -> PestionResult; +} + +/// Infallible conversion of a Pest grammar (or part of a grammar) into Ion [`Element`]. +pub trait PestToElement { + type Element: Element; + + /// Converts this into an [`Element`] representation. + /// + /// This operation cannot fail and therefore it is implied that it represents some + /// well formed Pest grammar or component thereof. + fn pest_to_element(&self) -> Self::Element; } -impl PestToElement for &str { +impl TryPestToElement for &str { type Element = OwnedElement; /// Parses a `str` slice as a Pest grammar and serializes the AST into [`Element`]. - fn pest_to_element(&self) -> PestionResult { + fn try_pest_to_element(&self) -> PestionResult { let pairs = PestParser::parse(Rule::grammar_rules, *self)?; let ast = match consume_rules(pairs) { Ok(ast) => ast, @@ -77,7 +93,7 @@ impl PestToElement for &str { } }; - ast.pest_to_element() + Ok(ast.pest_to_element()) } } @@ -85,14 +101,13 @@ impl PestToElement for Vec { type Element = OwnedElement; /// Converts a body of rules into a `struct` that has a rule for each field. - fn pest_to_element(&self) -> PestionResult { - let mut fields = vec![]; - for rule in self.iter() { + fn pest_to_element(&self) -> Self::Element { + let fields = self.iter().map(|rule| { let rule_name = text_token(rule.name.clone()); - let rule_value = rule.pest_to_element()?; - fields.push((rule_name, rule_value)); - } - Ok(Self::Element::new_struct(fields)) + let rule_value = rule.pest_to_element(); + (rule_name, rule_value) + }); + Self::Element::new_struct(fields) } } @@ -101,12 +116,12 @@ impl PestToElement for AstRule { /// Converts a Pest Rule into a `struct` that has the field for [`RuleType`] as a symbol /// and a field for the [`Expr`]. - fn pest_to_element(&self) -> PestionResult { - let fields = vec![ - (text_token("type"), self.ty.pest_to_element()?), - (text_token("expression"), self.expr.pest_to_element()?), - ]; - Ok(Self::Element::new_struct(fields)) + fn pest_to_element(&self) -> Self::Element { + let fields = std::array::IntoIter::new([ + (text_token("type"), self.ty.pest_to_element()), + (text_token("expression"), self.expr.pest_to_element()), + ]); + Self::Element::new_struct(fields) } } @@ -114,7 +129,7 @@ impl PestToElement for AstRuleType { type Element = OwnedElement; /// Serializes the enum into a symbolic value. - fn pest_to_element(&self) -> PestionResult { + fn pest_to_element(&self) -> Self::Element { let sym_tok = text_token(match self { RuleType::Normal => "normal", RuleType::Silent => "silent", @@ -123,7 +138,7 @@ impl PestToElement for AstRuleType { RuleType::NonAtomic => "non_atomic", }); - Ok(sym_tok.into()) + sym_tok.into() } } @@ -131,86 +146,92 @@ impl PestToElement for Expr { type Element = OwnedElement; /// Generates a `sexp` representation of the rule expression. - fn pest_to_element(&self) -> PestionResult { + fn pest_to_element(&self) -> Self::Element { use OwnedValue::*; - let element = Self::Element::new_sexp(match self.clone() { - Expr::Str(text) => vec![ + const STACK_LEN: usize = 4; + let values: SmallVec<[_; STACK_LEN]> = match self.clone() { + Expr::Str(text) => smallvec![ text_token("string").into(), text_token("exact").into(), String(text).into(), ], - Expr::Insens(text) => vec![ + Expr::Insens(text) => smallvec![ text_token("string").into(), text_token("insensitive").into(), String(text).into(), ], - Expr::Range(begin, end) => vec![ + Expr::Range(begin, end) => smallvec![ text_token("character_range").into(), String(begin).into(), String(end).into(), ], - Expr::Ident(name) => vec![text_token("identifier").into(), String(name).into()], - Expr::PosPred(expr) => vec![ + Expr::Ident(name) => smallvec![text_token("identifier").into(), String(name).into()], + Expr::PosPred(expr) => smallvec![ text_token("predicate").into(), text_token("positive").into(), - expr.pest_to_element()?, + expr.pest_to_element(), ], - Expr::NegPred(expr) => vec![ + Expr::NegPred(expr) => smallvec![ text_token("predicate").into(), text_token("negative").into(), - expr.pest_to_element()?, + expr.pest_to_element(), ], - Expr::Seq(left, right) => vec![ + Expr::Seq(left, right) => smallvec![ text_token("sequence").into(), - left.pest_to_element()?, - right.pest_to_element()?, + left.pest_to_element(), + right.pest_to_element(), ], - Expr::Choice(left, right) => vec![ + Expr::Choice(left, right) => smallvec![ text_token("choice").into(), - left.pest_to_element()?, - right.pest_to_element()?, + left.pest_to_element(), + right.pest_to_element(), ], - Expr::Opt(expr) => vec![text_token("optional").into(), expr.pest_to_element()?], - Expr::Rep(expr) => vec![ + Expr::Opt(expr) => { + smallvec![text_token("optional").into(), expr.pest_to_element()] + } + Expr::Rep(expr) => smallvec![ text_token("repeat_min").into(), 0.into(), - expr.pest_to_element()?, + expr.pest_to_element(), ], - Expr::RepOnce(expr) => vec![ + Expr::RepOnce(expr) => smallvec![ text_token("repeat_min").into(), 1.into(), - expr.pest_to_element()?, + expr.pest_to_element(), ], - Expr::RepMin(expr, min) => vec![ + Expr::RepMin(expr, min) => smallvec![ text_token("repeat_min").into(), (min as i64).into(), - expr.pest_to_element()?, + expr.pest_to_element(), ], - Expr::RepMax(expr, max) => vec![ + Expr::RepMax(expr, max) => smallvec![ text_token("repeat_max").into(), (max as i64).into(), - expr.pest_to_element()?, + expr.pest_to_element(), ], - Expr::RepExact(expr, exact) => vec![ + Expr::RepExact(expr, exact) => smallvec![ text_token("repeat_range").into(), (exact as i64).into(), (exact as i64).into(), - expr.pest_to_element()?, + expr.pest_to_element(), ], - Expr::RepMinMax(expr, min, max) => vec![ + Expr::RepMinMax(expr, min, max) => smallvec![ text_token("repeat_range").into(), (min as i64).into(), (max as i64).into(), - expr.pest_to_element()?, + expr.pest_to_element(), ], // TODO implement these Expr::Skip(_) => unimplemented!(), Expr::Push(_) => unimplemented!(), Expr::PeekSlice(_, _) => unimplemented!(), - }); + }; + assert!(values.len() <= STACK_LEN); + + let element = Self::Element::new_sexp(values); - Ok(element) + element } } @@ -403,10 +424,10 @@ mod tests { )] fn good(#[case] input: T, #[case] ion_literal: S) -> PestionResult<()> where - T: PestToElement + Debug, + T: TryPestToElement + Debug, S: AsRef, { - let actual = input.pest_to_element()?; + let actual = input.try_pest_to_element()?; let expected = element_reader().read_one(ion_literal.as_ref().as_bytes())?; const BUF_SIZE: usize = 16 * 1024 * 1024; @@ -431,8 +452,8 @@ mod tests { #[case::empty_rule(r#"a = {}"#)] #[case::self_reference(r#"a = { a }"#)] #[case::double_rule(r#"a = { "a" }\n a = { "b" }"#)] - fn pest_errors(#[case] input: T) -> PestionResult<()> { - match input.pest_to_element() { + fn pest_errors(#[case] input: T) -> PestionResult<()> { + match input.try_pest_to_element() { Err(PestionError::Pest(_)) => {} something => { unreachable!("Got result we did not expect: {:?}", something); From 5ae9e615fe88787ac0793cc4d386582f7d2cc722 Mon Sep 17 00:00:00 2001 From: Almann Goo Date: Mon, 7 Jun 2021 21:55:22 -0700 Subject: [PATCH 3/3] Renames `pestion` to `pest-ion`. Updates surrounding meta-data and README. Also renames the following types: * `PestionResult` => `PestToIonResult` * `PestionError` => `PestToIonError` --- Cargo.toml | 2 +- {pestion => pest-ion}/Cargo.toml | 6 +++--- {pestion => pest-ion}/README.md | 2 +- {pestion => pest-ion}/src/lib.rs | 14 +++++++------- {pestion => pest-ion}/src/result.rs | 10 +++++----- 5 files changed, 17 insertions(+), 17 deletions(-) rename {pestion => pest-ion}/Cargo.toml (80%) rename {pestion => pest-ion}/README.md (96%) rename {pestion => pest-ion}/src/lib.rs (98%) rename {pestion => pest-ion}/src/result.rs (72%) diff --git a/Cargo.toml b/Cargo.toml index 8864d5cc..eba2fcda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,5 +6,5 @@ members = [ "partiql-irgen", "partiql-parser", "partiql-rewriter", - "pestion", + "pest-ion", ] diff --git a/pestion/Cargo.toml b/pest-ion/Cargo.toml similarity index 80% rename from pestion/Cargo.toml rename to pest-ion/Cargo.toml index 5c08fcd5..44718328 100644 --- a/pestion/Cargo.toml +++ b/pest-ion/Cargo.toml @@ -1,9 +1,9 @@ [package] -name = "pestion" +name = "pest-ion" authors = ["PartiQL Team "] description = "A simple Pest grammar to Ion converter" -homepage = "https://github.com/partiql/partiql-lang-rust/pestion" -repository = "https://github.com/partiql/partiql-lang-rust/pestion" +homepage = "https://github.com/partiql/partiql-lang-rust/pest-ion" +repository = "https://github.com/partiql/partiql-lang-rust/pest-ion" license = "Apache-2.0" readme = "README.md" keywords = ["parser", "peg", "pest", "ion", "cli"] diff --git a/pestion/README.md b/pest-ion/README.md similarity index 96% rename from pestion/README.md rename to pest-ion/README.md index 8b7fa2d3..3950722c 100644 --- a/pestion/README.md +++ b/pest-ion/README.md @@ -1,4 +1,4 @@ -# Pestion +# Pest to Ion This is a simple tool and library for converting [Pest] grammars to [Ion] data format. diff --git a/pestion/src/lib.rs b/pest-ion/src/lib.rs similarity index 98% rename from pestion/src/lib.rs rename to pest-ion/src/lib.rs index 17e66f35..e96d4f05 100644 --- a/pestion/src/lib.rs +++ b/pest-ion/src/lib.rs @@ -7,11 +7,11 @@ //! The easiest way to convert [Pest] grammars to Ion is from a `str` slice: //! //! ``` -//! use pestion::*; +//! use pest_ion::*; //! use ion_rs::value::*; //! use ion_rs::value::reader::*; //! -//! fn main() -> PestionResult<()> { +//! fn main() -> PestToIonResult<()> { //! // parse a Pest grammar and convert it to Ion element //! let actual = r#"a = @{ "a" | "b" ~ "c" }"#.try_pest_to_element()?; //! @@ -60,7 +60,7 @@ pub trait TryPestToElement { /// /// This returns `Err` if the the conversion fails. For example, this can happen if the /// source is not a valid Pest grammar. - fn try_pest_to_element(&self) -> PestionResult; + fn try_pest_to_element(&self) -> PestToIonResult; } /// Infallible conversion of a Pest grammar (or part of a grammar) into Ion [`Element`]. @@ -78,7 +78,7 @@ impl TryPestToElement for &str { type Element = OwnedElement; /// Parses a `str` slice as a Pest grammar and serializes the AST into [`Element`]. - fn try_pest_to_element(&self) -> PestionResult { + fn try_pest_to_element(&self) -> PestToIonResult { let pairs = PestParser::parse(Rule::grammar_rules, *self)?; let ast = match consume_rules(pairs) { Ok(ast) => ast, @@ -422,7 +422,7 @@ mod tests { }, }"# )] - fn good(#[case] input: T, #[case] ion_literal: S) -> PestionResult<()> + fn good(#[case] input: T, #[case] ion_literal: S) -> PestToIonResult<()> where T: TryPestToElement + Debug, S: AsRef, @@ -452,9 +452,9 @@ mod tests { #[case::empty_rule(r#"a = {}"#)] #[case::self_reference(r#"a = { a }"#)] #[case::double_rule(r#"a = { "a" }\n a = { "b" }"#)] - fn pest_errors(#[case] input: T) -> PestionResult<()> { + fn pest_errors(#[case] input: T) -> PestToIonResult<()> { match input.try_pest_to_element() { - Err(PestionError::Pest(_)) => {} + Err(PestToIonError::Pest(_)) => {} something => { unreachable!("Got result we did not expect: {:?}", something); } diff --git a/pestion/src/result.rs b/pest-ion/src/result.rs similarity index 72% rename from pestion/src/result.rs rename to pest-ion/src/result.rs index a1e069a4..326acb08 100644 --- a/pestion/src/result.rs +++ b/pest-ion/src/result.rs @@ -5,11 +5,11 @@ use thiserror::Error; /// Main [`Result`] type for Pest to Ion. -pub type PestionResult = Result; +pub type PestToIonResult = Result; /// Error type for problems in this crate. #[derive(Error, Debug)] -pub enum PestionError { +pub enum PestToIonError { /// An error working with [`pest_meta`]. #[error("Pest Error: {0}")] Pest(#[from] pest::error::Error), @@ -19,11 +19,11 @@ pub enum PestionError { Ion(#[from] ion_rs::result::IonError), /// General error from this library. - #[error("Pestion Error: {0}")] + #[error("Pest to Ion Error: {0}")] Invalid(String), } /// Convenience function to create a general error result. -pub fn invalid>(message: S) -> PestionResult { - Err(PestionError::Invalid(message.into())) +pub fn invalid>(message: S) -> PestToIonResult { + Err(PestToIonError::Invalid(message.into())) }