diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/docs.rs | 10 | ||||
| -rw-r--r-- | src/generator.rs | 143 | ||||
| -rw-r--r-- | src/lib.rs | 55 |
3 files changed, 171 insertions, 37 deletions
diff --git a/src/docs.rs b/src/docs.rs index f1ce188..ccc82e7 100644 --- a/src/docs.rs +++ b/src/docs.rs @@ -119,4 +119,14 @@ mod tests { doc_comment.grammar_doc ); } + + #[test] + fn test_empty_grammar_doc() { + assert!(parser::parse(Rule::grammar_rules, "//!").is_ok()); + assert!(parser::parse(Rule::grammar_rules, "///").is_ok()); + assert!(parser::parse(Rule::grammar_rules, "//").is_ok()); + assert!(parser::parse(Rule::grammar_rules, "/// Line Doc").is_ok()); + assert!(parser::parse(Rule::grammar_rules, "//! Grammar Doc").is_ok()); + assert!(parser::parse(Rule::grammar_rules, "// Comment").is_ok()); + } } diff --git a/src/generator.rs b/src/generator.rs index 0dbcaa3..7a527c5 100644 --- a/src/generator.rs +++ b/src/generator.rs @@ -11,17 +11,17 @@ use std::path::PathBuf; use proc_macro2::TokenStream; use quote::{ToTokens, TokenStreamExt}; -use syn::{self, Generics, Ident}; +use syn::{self, Ident}; use pest::unicode::unicode_property_names; use pest_meta::ast::*; use pest_meta::optimizer::*; use crate::docs::DocComment; +use crate::ParsedDerive; pub(crate) fn generate( - name: Ident, - generics: &Generics, + parsed_derive: ParsedDerive, paths: Vec<PathBuf>, rules: Vec<OptimizedRule>, defaults: Vec<&str>, @@ -29,14 +29,14 @@ pub(crate) fn generate( include_grammar: bool, ) -> TokenStream { let uses_eoi = defaults.iter().any(|name| *name == "EOI"); - + let name = parsed_derive.name; let builtins = generate_builtin_rules(); let include_fix = if include_grammar { generate_include(&name, paths) } else { quote!() }; - let rule_enum = generate_enum(&rules, doc_comment, uses_eoi); + let rule_enum = generate_enum(&rules, doc_comment, uses_eoi, parsed_derive.non_exhaustive); let patterns = generate_patterns(&rules, uses_eoi); let skip = generate_skip(&rules); @@ -49,7 +49,7 @@ pub(crate) fn generate( } })); - let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + let (impl_generics, ty_generics, where_clause) = parsed_derive.generics.split_for_impl(); let result = result_type(); @@ -197,8 +197,13 @@ fn generate_include(name: &Ident, paths: Vec<PathBuf>) -> TokenStream { } } -fn generate_enum(rules: &[OptimizedRule], doc_comment: &DocComment, uses_eoi: bool) -> TokenStream { - let rules = rules.iter().map(|rule| { +fn generate_enum( + rules: &[OptimizedRule], + doc_comment: &DocComment, + uses_eoi: bool, + non_exhaustive: bool, +) -> TokenStream { + let rule_variants = rules.iter().map(|rule| { let rule_name = format_ident!("r#{}", rule.name); match doc_comment.line_docs.get(&rule.name) { @@ -213,26 +218,49 @@ fn generate_enum(rules: &[OptimizedRule], doc_comment: &DocComment, uses_eoi: bo }); let grammar_doc = &doc_comment.grammar_doc; + let mut result = quote! { + #[doc = #grammar_doc] + #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] + #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] + }; + if non_exhaustive { + result.append_all(quote! { + #[non_exhaustive] + }); + } + result.append_all(quote! { + pub enum Rule + }); if uses_eoi { - quote! { - #[doc = #grammar_doc] - #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] - #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] - pub enum Rule { + result.append_all(quote! { + { + #[doc = "End-of-input"] EOI, - #( #rules ),* + #( #rule_variants ),* } - } + }); } else { - quote! { - #[doc = #grammar_doc] - #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] - #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] - pub enum Rule { - #( #rules ),* + result.append_all(quote! { + { + #( #rule_variants ),* + } + }) + }; + + let rules = rules.iter().map(|rule| { + let rule_name = format_ident!("r#{}", rule.name); + quote! { #rule_name } + }); + + result.append_all(quote! { + impl Rule { + pub fn all_rules() -> &'static[Rule] { + &[ #(Rule::#rules), * ] } } - } + }); + + result } fn generate_patterns(rules: &[OptimizedRule], uses_eoi: bool) -> TokenStream { @@ -496,6 +524,26 @@ fn generate_expr(expr: OptimizedExpr) -> TokenStream { }) } } + #[cfg(feature = "grammar-extras")] + OptimizedExpr::RepOnce(expr) => { + let expr = generate_expr(*expr); + + quote! { + state.sequence(|state| { + #expr.and_then(|state| { + state.repeat(|state| { + state.sequence(|state| { + super::hidden::skip( + state + ).and_then(|state| { + #expr + }) + }) + }) + }) + }) + } + } OptimizedExpr::Skip(strings) => { quote! { let strings = [#(#strings),*]; @@ -517,6 +565,13 @@ fn generate_expr(expr: OptimizedExpr) -> TokenStream { state.restore_on_err(|state| #expr) } } + #[cfg(feature = "grammar-extras")] + OptimizedExpr::NodeTag(expr, tag) => { + let expr = generate_expr(*expr); + quote! { + #expr.and_then(|state| state.tag_node(#tag)) + } + } } } @@ -628,6 +683,22 @@ fn generate_expr_atomic(expr: OptimizedExpr) -> TokenStream { }) } } + #[cfg(feature = "grammar-extras")] + OptimizedExpr::RepOnce(expr) => { + let expr = generate_expr_atomic(*expr); + + quote! { + state.sequence(|state| { + #expr.and_then(|state| { + state.repeat(|state| { + state.sequence(|state| { + #expr + }) + }) + }) + }) + } + } OptimizedExpr::Skip(strings) => { quote! { let strings = [#(#strings),*]; @@ -649,6 +720,13 @@ fn generate_expr_atomic(expr: OptimizedExpr) -> TokenStream { state.restore_on_err(|state| #expr) } } + #[cfg(feature = "grammar-extras")] + OptimizedExpr::NodeTag(expr, tag) => { + let expr = generate_expr_atomic(*expr); + quote! { + #expr.and_then(|state| state.tag_node(#tag)) + } + } } } @@ -694,6 +772,7 @@ mod tests { use proc_macro2::Span; use std::collections::HashMap; + use syn::Generics; #[test] fn rule_enum_simple() { @@ -712,7 +791,7 @@ mod tests { }; assert_eq!( - generate_enum(&rules, doc_comment, false).to_string(), + generate_enum(&rules, doc_comment, false, false).to_string(), quote! { #[doc = "Rule doc\nhello"] #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] @@ -721,6 +800,11 @@ mod tests { #[doc = "This is rule comment"] r#f } + impl Rule { + pub fn all_rules() -> &'static [Rule] { + &[Rule::r#f] + } + } } .to_string() ); @@ -1033,9 +1117,13 @@ mod tests { let base_path = current_dir.join("base.pest").to_str().unwrap().to_string(); let test_path = current_dir.join("test.pest").to_str().unwrap().to_string(); - + let parsed_derive = ParsedDerive { + name, + generics, + non_exhaustive: false, + }; assert_eq!( - generate(name, &generics, vec![PathBuf::from("base.pest"), PathBuf::from("test.pest")], rules, defaults, doc_comment, true).to_string(), + generate(parsed_derive, vec![PathBuf::from("base.pest"), PathBuf::from("test.pest")], rules, defaults, doc_comment, true).to_string(), quote! { #[allow(non_upper_case_globals)] const _PEST_GRAMMAR_MyParser: [&'static str; 2usize] = [include_str!(#base_path), include_str!(#test_path)]; @@ -1048,6 +1136,11 @@ mod tests { #[doc = "If statement"] r#if } + impl Rule { + pub fn all_rules() -> &'static [Rule] { + &[Rule::r#a, Rule::r#if] + } + } #[allow(clippy::all)] impl ::pest::Parser<Rule> for MyParser { @@ -42,7 +42,7 @@ use pest_meta::{optimizer, unwrap_or_report, validator}; /// "include_str" statement (done in pest_derive, but turned off in the local bootstrap). pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream { let ast: DeriveInput = syn::parse2(input).unwrap(); - let (name, generics, contents) = parse_derive(ast); + let (parsed_derive, contents) = parse_derive(ast); let mut data = String::new(); let mut paths = vec![]; @@ -81,9 +81,8 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream { }; data.push_str(&_data); - match _path { - Some(path) => paths.push(path), - None => (), + if let Some(path) = _path { + paths.push(path); } } @@ -98,8 +97,7 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream { let optimized = optimizer::optimize(ast); generator::generate( - name, - &generics, + parsed_derive, paths, optimized, defaults, @@ -121,7 +119,13 @@ enum GrammarSource { Inline(String), } -fn parse_derive(ast: DeriveInput) -> (Ident, Generics, Vec<GrammarSource>) { +struct ParsedDerive { + pub(crate) name: Ident, + pub(crate) generics: Generics, + pub(crate) non_exhaustive: bool, +} + +fn parse_derive(ast: DeriveInput) -> (ParsedDerive, Vec<GrammarSource>) { let name = ast.ident; let generics = ast.generics; @@ -143,7 +147,19 @@ fn parse_derive(ast: DeriveInput) -> (Ident, Generics, Vec<GrammarSource>) { grammar_sources.push(get_attribute(attr)) } - (name, generics, grammar_sources) + let non_exhaustive = ast + .attrs + .iter() + .any(|attr| attr.meta.path().is_ident("non_exhaustive")); + + ( + ParsedDerive { + name, + generics, + non_exhaustive, + }, + grammar_sources, + ) } fn get_attribute(attr: &Attribute) -> GrammarSource { @@ -178,7 +194,7 @@ mod tests { pub struct MyParser<'a, T>; "; let ast = syn::parse_str(definition).unwrap(); - let (_, _, filenames) = parse_derive(ast); + let (_, filenames) = parse_derive(ast); assert_eq!(filenames, [GrammarSource::Inline("GRAMMAR".to_string())]); } @@ -190,8 +206,9 @@ mod tests { pub struct MyParser<'a, T>; "; let ast = syn::parse_str(definition).unwrap(); - let (_, _, filenames) = parse_derive(ast); + let (parsed_derive, filenames) = parse_derive(ast); assert_eq!(filenames, [GrammarSource::File("myfile.pest".to_string())]); + assert!(!parsed_derive.non_exhaustive); } #[test] @@ -203,7 +220,7 @@ mod tests { pub struct MyParser<'a, T>; "; let ast = syn::parse_str(definition).unwrap(); - let (_, _, filenames) = parse_derive(ast); + let (_, filenames) = parse_derive(ast); assert_eq!( filenames, [ @@ -214,6 +231,19 @@ mod tests { } #[test] + fn derive_nonexhaustive() { + let definition = " + #[non_exhaustive] + #[grammar = \"myfile.pest\"] + pub struct MyParser<'a, T>; + "; + let ast = syn::parse_str(definition).unwrap(); + let (parsed_derive, filenames) = parse_derive(ast); + assert_eq!(filenames, [GrammarSource::File("myfile.pest".to_string())]); + assert!(parsed_derive.non_exhaustive); + } + + #[test] #[should_panic(expected = "grammar attribute must be a string")] fn derive_wrong_arg() { let definition = " @@ -243,6 +273,7 @@ mod tests { fn test_generate_doc() { let input = quote! { #[derive(Parser)] + #[non_exhaustive] #[grammar = "../tests/test.pest"] pub struct TestParser; }; @@ -253,7 +284,7 @@ mod tests { #[doc = "A parser for JSON file.\nAnd this is a example for JSON parser.\n\n indent-4-space\n"] #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] - + #[non_exhaustive] pub enum Rule { #[doc = "Matches foo str, e.g.: `foo`"] r#foo, |