From 68139347466f47811db615ea017aec9ff7fadcfc Mon Sep 17 00:00:00 2001 From: Jeff Vander Stoep Date: Tue, 13 Dec 2022 09:45:07 +0100 Subject: Upgrade pest_derive to 2.5.1 This project was upgraded with external_updater. Usage: tools/external_updater/updater.sh update rust/crates/pest_derive For more info, check https://cs.android.com/android/platform/superproject/+/master:tools/external_updater/README.md Test: TreeHugger Change-Id: If3a6c8423e9aefd7a856b933f105d339c6efc146 --- Android.bp | 10 ++- Cargo.lock.saved | 195 ++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 42 ++++++----- Cargo.toml.orig | 19 ++--- METADATA | 13 ++-- _README.md | 80 +++++++++++++++----- examples/calc.pest | 16 ++++ examples/calc.rs | 109 +++++++++++++++++++++++++++ src/lib.rs | 95 ++++++++++++++--------- tests/grammar.pest | 4 + tests/grammar.rs | 33 +++++++- tests/grammar_inline.rs | 4 + tests/lists.rs | 4 + tests/reporting.rs | 4 + 14 files changed, 537 insertions(+), 91 deletions(-) create mode 100644 Cargo.lock.saved create mode 100644 examples/calc.pest create mode 100644 examples/calc.rs diff --git a/Android.bp b/Android.bp index 616f116..4cc2568 100644 --- a/Android.bp +++ b/Android.bp @@ -1,8 +1,6 @@ // This file is generated by cargo2android.py --run. // Do not modify this file as changes will be overridden on upgrade. - - package { default_applicable_licenses: ["external_rust_crates_pest_derive_license"], } @@ -43,9 +41,13 @@ rust_proc_macro { name: "libpest_derive", crate_name: "pest_derive", cargo_env_compat: true, - cargo_pkg_version: "2.1.0", + cargo_pkg_version: "2.5.1", srcs: ["src/lib.rs"], - edition: "2015", + edition: "2021", + features: [ + "default", + "std", + ], rustlibs: [ "libpest", "libpest_generator", diff --git a/Cargo.lock.saved b/Cargo.lock.saved new file mode 100644 index 0000000..1d1a350 --- /dev/null +++ b/Cargo.lock.saved @@ -0,0 +1,195 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "block-buffer" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cpufeatures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "generic-array" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "libc" +version = "0.2.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" + +[[package]] +name = "once_cell" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" + +[[package]] +name = "pest" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc8bed3549e0f9b0a2a78bf7c0018237a2cdf085eecbbc048e52612438e4e9d0" +dependencies = [ + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.5.1" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28a1af60b1c4148bb269006a750cff8e2ea36aff34d2d96cf7be0b14d1bed23c" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fec8605d59fc2ae0c6c1aefc0c7c7a9769732017c0ce07f7a9cfffa7b4404f20" +dependencies = [ + "once_cell", + "pest", + "sha1", +] + +[[package]] +name = "proc-macro2" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "syn" +version = "1.0.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typenum" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" + +[[package]] +name = "ucd-trie" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" + +[[package]] +name = "unicode-ident" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" diff --git a/Cargo.toml b/Cargo.toml index 47dfec4..7266a1b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,22 +3,28 @@ # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies -# to registry (e.g. crates.io) dependencies +# to registry (e.g., crates.io) dependencies. # -# If you believe there's an error in this file please file an -# issue against the rust-lang/cargo repository. If you're -# editing this file be aware that the upstream Cargo.toml -# will likely look very different (and much more reasonable) +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. [package] +edition = "2021" +rust-version = "1.56" name = "pest_derive" -version = "2.1.0" +version = "2.5.1" authors = ["Dragoș Tiselice "] description = "pest's derive macro" -homepage = "https://pest-parser.github.io/" +homepage = "https://pest.rs/" documentation = "https://docs.rs/pest" readme = "_README.md" -keywords = ["pest", "parser", "peg", "grammar"] +keywords = [ + "pest", + "parser", + "peg", + "grammar", +] categories = ["parsing"] license = "MIT/Apache-2.0" repository = "https://github.com/pest-parser/pest" @@ -26,16 +32,18 @@ repository = "https://github.com/pest-parser/pest" [lib] name = "pest_derive" proc-macro = true + [dependencies.pest] -version = "2.1.0" +version = "2.5.1" +default-features = false [dependencies.pest_generator] -version = "2.1.0" -[badges.codecov] -repository = "pest-parser/pest" - -[badges.maintenance] -status = "actively-developed" +version = "2.5.1" +default-features = false -[badges.travis-ci] -repository = "pest-parser/pest" +[features] +default = ["std"] +std = [ + "pest/std", + "pest_generator/std", +] diff --git a/Cargo.toml.orig b/Cargo.toml.orig index 2740871..2dd43a8 100644 --- a/Cargo.toml.orig +++ b/Cargo.toml.orig @@ -1,26 +1,27 @@ [package] name = "pest_derive" description = "pest's derive macro" -version = "2.1.0" +version = "2.5.1" +edition = "2021" authors = ["Dragoș Tiselice "] -homepage = "https://pest-parser.github.io/" +homepage = "https://pest.rs/" repository = "https://github.com/pest-parser/pest" documentation = "https://docs.rs/pest" keywords = ["pest", "parser", "peg", "grammar"] categories = ["parsing"] license = "MIT/Apache-2.0" readme = "_README.md" +rust-version = "1.56" [lib] name = "pest_derive" proc-macro = true +[features] +default = ["std"] +std = ["pest/std", "pest_generator/std"] + [dependencies] # for tests, included transitively anyway -pest = { path = "../pest", version = "2.1.0" } -pest_generator = { path = "../generator", version = "2.1.0" } - -[badges] -codecov = { repository = "pest-parser/pest" } -maintenance = { status = "actively-developed" } -travis-ci = { repository = "pest-parser/pest" } +pest = { path = "../pest", version = "2.5.1", default-features = false } +pest_generator = { path = "../generator", version = "2.5.1", default-features = false } diff --git a/METADATA b/METADATA index 096be95..abf251a 100644 --- a/METADATA +++ b/METADATA @@ -1,3 +1,7 @@ +# This project was upgraded with external_updater. +# Usage: tools/external_updater/updater.sh update rust/crates/pest_derive +# For more info, check https://cs.android.com/android/platform/superproject/+/master:tools/external_updater/README.md + name: "pest_derive" description: "pest\'s derive macro" third_party { @@ -7,14 +11,13 @@ third_party { } url { type: ARCHIVE - value: "https://static.crates.io/crates/pest_derive/pest_derive-2.1.0.crate" + value: "https://static.crates.io/crates/pest_derive/pest_derive-2.5.1.crate" } - version: "2.1.0" - # Dual-licensed, using the least restrictive per go/thirdpartylicenses#same. + version: "2.5.1" license_type: NOTICE last_upgrade_date { year: 2022 - month: 1 - day: 27 + month: 12 + day: 13 } } diff --git a/_README.md b/_README.md index 2c94a72..da30ab7 100644 --- a/_README.md +++ b/_README.md @@ -1,15 +1,18 @@ +

# pest. The Elegant Parser -[![Join the chat at https://gitter.im/dragostis/pest](https://badges.gitter.im/dragostis/pest.svg)](https://gitter.im/dragostis/pest?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -[![Book](https://img.shields.io/badge/book-WIP-4d76ae.svg)](https://pest-parser.github.io/book) +[![Join the chat at https://gitter.im/pest-parser/pest](https://badges.gitter.im/dragostis/pest.svg)](https://gitter.im/pest-parser/pest?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Book](https://img.shields.io/badge/book-WIP-4d76ae.svg)](https://pest.rs/book) [![Docs](https://docs.rs/pest/badge.svg)](https://docs.rs/pest) -[![Build Status](https://travis-ci.org/pest-parser/pest.svg?branch=master)](https://travis-ci.org/pest-parser/pest) +[![pest Continuous Integration](https://github.com/pest-parser/pest/actions/workflows/ci.yml/badge.svg)](https://github.com/pest-parser/pest/actions/workflows/ci.yml) [![codecov](https://codecov.io/gh/pest-parser/pest/branch/master/graph/badge.svg)](https://codecov.io/gh/pest-parser/pest) +Rustc Version 1.56.1+ + [![Crates.io](https://img.shields.io/crates/d/pest.svg)](https://crates.io/crates/pest) [![Crates.io](https://img.shields.io/crates/v/pest.svg)](https://crates.io/crates/pest) @@ -28,25 +31,28 @@ Other helpful resources: * API reference on [docs.rs] * play with grammars and share them on our [fiddle] -* leave feedback, ask questions, or greet us on [Gitter] +* find previous common questions answered or ask questions on [GitHub Discussions] +* leave feedback, ask questions, or greet us on [Gitter] or [Discord] -[book]: https://pest-parser.github.io/book +[book]: https://pest.rs/book [docs.rs]: https://docs.rs/pest -[fiddle]: https://pest-parser.github.io/#editor -[Gitter]: https://gitter.im/dragostis/pest +[fiddle]: https://pest.rs/#editor +[Gitter]: https://gitter.im/pest-parser/pest +[Discord]: https://discord.gg/XEGACtWpT2 +[GitHub Discussions]: https://github.com/pest-parser/pest/discussions ## Example -The following is an example of a grammar for a list of alpha-numeric identifiers -where the first identifier does not start with a digit: +The following is an example of a grammar for a list of alphanumeric identifiers +where all identifiers don't start with a digit: ```rust alpha = { 'a'..'z' | 'A'..'Z' } digit = { '0'..'9' } -ident = { (alpha | digit)+ } +ident = { !digit ~ (alpha | digit)+ } -ident_list = _{ !digit ~ ident ~ (" " ~ ident)+ } +ident_list = _{ ident ~ (" " ~ ident)* } // ^ // ident_list rule is silent which means it produces no tokens ``` @@ -78,6 +84,9 @@ thread 'main' panicked at ' --> 1:1 = expected ident', src/main.rs:12 ``` +These error messages can be obtained from their default `Display` implementation, +e.g. `panic!("{}", parser_result.unwrap_err())` or `println!("{}", e)`. + ## Pairs API The grammar can be used to derive a `Parser` implementation automatically. @@ -99,19 +108,16 @@ fn main() { // Because ident_list is silent, the iterator will contain idents for pair in pairs { - - let span = pair.clone().into_span(); // A pair is a combination of the rule which matched and a span of input println!("Rule: {:?}", pair.as_rule()); - println!("Span: {:?}", span); - println!("Text: {}", span.as_str()); + println!("Span: {:?}", pair.as_span()); + println!("Text: {}", pair.as_str()); // A pair can be converted to an iterator of the tokens which make it up: for inner_pair in pair.into_inner() { - let inner_span = inner_pair.clone().into_span(); match inner_pair.as_rule() { - Rule::alpha => println!("Letter: {}", inner_span.as_str()), - Rule::digit => println!("Digit: {}", inner_span.as_str()), + Rule::alpha => println!("Letter: {}", inner_pair.as_str()), + Rule::digit => println!("Digit: {}", inner_pair.as_str()), _ => unreachable!() }; } @@ -133,6 +139,25 @@ Letter: b Digit: 2 ``` +### Defining multiple parsers in a single file +The current automatic `Parser` derivation will produce the `Rule` enum +which would have name conflicts if one tried to define multiple such structs +that automatically derive `Parser`. One possible way around it is to put each +parser struct in a separate namespace: + +```rust +mod a { + #[derive(Parser)] + #[grammar = "a.pest"] + pub struct ParserA; +} +mod b { + #[derive(Parser)] + #[grammar = "b.pest"] + pub struct ParserB; +} +``` + ## Other features * Precedence climbing @@ -143,15 +168,20 @@ Digit: 2 ## Projects using pest * [pest_meta](https://github.com/pest-parser/pest/blob/master/meta/src/grammar.pest) (bootstrapped) +* [AshPaper](https://github.com/shnewto/ashpaper) * [brain](https://github.com/brain-lang/brain) -* [Chelone](https://github.com/Aaronepower/chelone) +* [cicada](https://github.com/mitnk/cicada) * [comrak](https://github.com/kivikakk/comrak) +* [elastic-rs](https://github.com/cch123/elastic-rs) * [graphql-parser](https://github.com/Keats/graphql-parser) * [handlebars-rust](https://github.com/sunng87/handlebars-rust) * [hexdino](https://github.com/Luz/hexdino) * [Huia](https://gitlab.com/jimsy/huia/) +* [insta](https://github.com/mitsuhiko/insta) +* [jql](https://github.com/yamafaktory/jql) * [json5-rs](https://github.com/callum-oakley/json5-rs) * [mt940](https://github.com/svenstaro/mt940-rs) +* [Myoxine](https://github.com/d3bate/myoxine) * [py_literal](https://github.com/jturner314/py_literal) * [rouler](https://github.com/jarcane/rouler) * [RuSh](https://github.com/lwandrebeck/RuSh) @@ -160,6 +190,18 @@ Digit: 2 * [tera](https://github.com/Keats/tera) * [ui_gen](https://github.com/emoon/ui_gen) * [ukhasnet-parser](https://github.com/adamgreig/ukhasnet-parser) +* [ZoKrates](https://github.com/ZoKrates/ZoKrates) +* [Vector](https://github.com/timberio/vector) +* [AutoCorrect](https://github.com/huacnlee/autocorrect) +* [yaml-peg](https://github.com/aofdev/yaml-peg) +* [qubit](https://github.com/abhimanyu003/qubit) +* [caith](https://github.com/Geobert/caith) (a dice roller crate) +* [Melody](https://github.com/yoav-lavi/melody) + +## Minimum Supported Rust Version (MSRV) + +This library should always compile with default features on **Rust 1.56.1** +or **Rust 1.61** with `const_prec_climber`. ## Special thanks diff --git a/examples/calc.pest b/examples/calc.pest new file mode 100644 index 0000000..9f2cc3b --- /dev/null +++ b/examples/calc.pest @@ -0,0 +1,16 @@ +WHITESPACE = _{ " " | "\t" | NEWLINE } + + program = { SOI ~ expr ~ EOI } + expr = { prefix* ~ primary ~ postfix* ~ (infix ~ prefix* ~ primary ~ postfix* )* } + infix = _{ add | sub | mul | div | pow } + add = { "+" } // Addition + sub = { "-" } // Subtraction + mul = { "*" } // Multiplication + div = { "/" } // Division + pow = { "^" } // Exponentiation + prefix = _{ neg } + neg = { "-" } // Negation + postfix = _{ fac } + fac = { "!" } // Factorial + primary = _{ int | "(" ~ expr ~ ")" } + int = @{ (ASCII_NONZERO_DIGIT ~ ASCII_DIGIT+ | ASCII_DIGIT) } \ No newline at end of file diff --git a/examples/calc.rs b/examples/calc.rs new file mode 100644 index 0000000..efc6b7b --- /dev/null +++ b/examples/calc.rs @@ -0,0 +1,109 @@ +mod parser { + use pest_derive::Parser; + + #[derive(Parser)] + #[grammar = "../examples/calc.pest"] + pub struct Parser; +} + +use parser::Rule; +use pest::{ + iterators::Pairs, + pratt_parser::{Assoc::*, Op, PrattParser}, + Parser, +}; +use std::io::{stdin, stdout, Write}; + +fn parse_to_str(pairs: Pairs, pratt: &PrattParser) -> String { + pratt + .map_primary(|primary| match primary.as_rule() { + Rule::int => primary.as_str().to_owned(), + Rule::expr => parse_to_str(primary.into_inner(), pratt), + _ => unreachable!(), + }) + .map_prefix(|op, rhs| match op.as_rule() { + Rule::neg => format!("(-{})", rhs), + _ => unreachable!(), + }) + .map_postfix(|lhs, op| match op.as_rule() { + Rule::fac => format!("({}!)", lhs), + _ => unreachable!(), + }) + .map_infix(|lhs, op, rhs| match op.as_rule() { + Rule::add => format!("({}+{})", lhs, rhs), + Rule::sub => format!("({}-{})", lhs, rhs), + Rule::mul => format!("({}*{})", lhs, rhs), + Rule::div => format!("({}/{})", lhs, rhs), + Rule::pow => format!("({}^{})", lhs, rhs), + _ => unreachable!(), + }) + .parse(pairs) +} + +fn parse_to_i32(pairs: Pairs, pratt: &PrattParser) -> i128 { + pratt + .map_primary(|primary| match primary.as_rule() { + Rule::int => primary.as_str().parse().unwrap(), + Rule::expr => parse_to_i32(primary.into_inner(), pratt), + _ => unreachable!(), + }) + .map_prefix(|op, rhs| match op.as_rule() { + Rule::neg => -rhs, + _ => unreachable!(), + }) + .map_postfix(|lhs, op| match op.as_rule() { + Rule::fac => (1..lhs + 1).product(), + _ => unreachable!(), + }) + .map_infix(|lhs, op, rhs| match op.as_rule() { + Rule::add => lhs + rhs, + Rule::sub => lhs - rhs, + Rule::mul => lhs * rhs, + Rule::div => lhs / rhs, + Rule::pow => (1..rhs + 1).map(|_| lhs).product(), + _ => unreachable!(), + }) + .parse(pairs) +} + +fn main() { + let pratt = PrattParser::new() + .op(Op::infix(Rule::add, Left) | Op::infix(Rule::sub, Left)) + .op(Op::infix(Rule::mul, Left) | Op::infix(Rule::div, Left)) + .op(Op::infix(Rule::pow, Right)) + .op(Op::postfix(Rule::fac)) + .op(Op::prefix(Rule::neg)); + + let stdin = stdin(); + let mut stdout = stdout(); + + loop { + let source = { + print!("> "); + let _ = stdout.flush(); + let mut input = String::new(); + let _ = stdin.read_line(&mut input); + input.trim().to_string() + }; + + let pairs = match parser::Parser::parse(Rule::program, &source) { + Ok(mut parse_tree) => { + parse_tree + .next() + .unwrap() + .into_inner() // inner of program + .next() + .unwrap() + .into_inner() // inner of expr + } + Err(err) => { + println!("Failed parsing input: {:}", err); + continue; + } + }; + + print!("{} => ", source); + print!("{} => ", parse_to_str(pairs.clone(), &pratt)); + println!("{}", parse_to_i32(pairs.clone(), &pratt)); + } +} diff --git a/src/lib.rs b/src/lib.rs index f60d5d9..a908897 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,7 +6,12 @@ // license , at your // option. All files in the project carrying such notice may not be copied, // modified, or distributed except according to those terms. - +#![doc( + html_root_url = "https://docs.rs/pest_derive", + html_logo_url = "https://raw.githubusercontent.com/pest-parser/pest/master/pest-logo.svg", + html_favicon_url = "https://raw.githubusercontent.com/pest-parser/pest/master/pest-logo.svg" +)] +#![warn(missing_docs, rust_2018_idioms, unused_qualifications)] //! # pest. The Elegant Parser //! //! pest is a general purpose parser written in Rust with a focus on accessibility, correctness, @@ -24,12 +29,16 @@ //! //! * API reference on [docs.rs] //! * play with grammars and share them on our [fiddle] -//! * leave feedback, ask questions, or greet us on [Gitter] +//! * find previous common questions answered or ask questions on [GitHub Discussions] +//! * leave feedback, ask questions, or greet us on [Gitter] or [Discord] //! -//! [book]: https://pest-parser.github.io/book +//! [book]: https://pest.rs/book //! [docs.rs]: https://docs.rs/pest -//! [fiddle]: https://pest-parser.github.io/#editor -//! [Gitter]: https://gitter.im/dragostis/pest +//! [fiddle]: https://pest.rs/#editor +//! [Gitter]: https://gitter.im/pest-parser/pest +//! [Discord]: https://discord.gg/XEGACtWpT2 +//! [GitHub Discussions]: https://github.com/pest-parser/pest/discussions +//! //! //! ## `.pest` files //! @@ -55,7 +64,7 @@ //! //! Comments start with `//` and end at the end of the line. //! -//! ```ignore +//! ```text //! // a comment //! ``` //! @@ -143,37 +152,50 @@ //! //! 1. Terminals //! -//! | Terminal | Usage | -//! |------------|----------------------------------------------------------------| -//! | `"a"` | matches the exact string `"a"` | -//! | `^"a"` | matches the exact string `"a"` case insensitively (ASCII only) | -//! | `'a'..'z'` | matches one character between `'a'` and `'z'` | -//! | `a` | matches rule `a` | +//! | Terminal | Usage | +//! |------------|----------------------------------------------------------------| +//! | `"a"` | matches the exact string `"a"` | +//! | `^"a"` | matches the exact string `"a"` case insensitively (ASCII only) | +//! | `'a'..'z'` | matches one character between `'a'` and `'z'` | +//! | `a` | matches rule `a` | //! //! Strings and characters follow //! [Rust's escape mechanisms](https://doc.rust-lang.org/reference/tokens.html#byte-escapes), while -//! identifiers can contain alpha-numeric characters and underscores (`_`), as long as they do not +//! identifiers can contain alphanumeric characters and underscores (`_`), as long as they do not //! start with a digit. //! //! 2. Non-terminals //! -//! | Non-terminal | Usage | -//! |-----------------------|------------------------------------------------------------| -//! | `(e)` | matches `e` | -//! | `e1 ~ e2` | matches the sequence `e1` `e2` | -//! | e1 \| e2 | matches either `e1` or `e2` | -//! | `e*` | matches `e` zero or more times | -//! | `e+` | matches `e` one or more times | -//! | `e{n}` | matches `e` exactly `n` times | -//! | `e{, n}` | matches `e` at most `n` times | -//! | `e{n,} ` | matches `e` at least `n` times | -//! | `e{m, n}` | matches `e` between `m` and `n` times inclusively | -//! | `e?` | optionally matches `e` | -//! | `&e` | matches `e` without making progress | -//! | `!e` | matches if `e` doesn't match without making progress | -//! | `PUSH(e)` | matches `e` and pushes it's captured string down the stack | -//! -//! where `e`, `e1`, and `e2` are expressions. +//! | Non-terminal | Usage | +//! |-----------------------|------------------------------------------------------------| +//! | `(e)` | matches `e` | +//! | `e1 ~ e2` | matches the sequence `e1` `e2` | +//! | e1 \| e2 | matches either `e1` or `e2` | +//! | `e*` | matches `e` zero or more times | +//! | `e+` | matches `e` one or more times | +//! | `e{n}` | matches `e` exactly `n` times | +//! | `e{, n}` | matches `e` at most `n` times | +//! | `e{n,}` | matches `e` at least `n` times | +//! | `e{m, n}` | matches `e` between `m` and `n` times inclusively | +//! | `e?` | optionally matches `e` | +//! | `&e` | matches `e` without making progress | +//! | `!e` | matches if `e` doesn't match without making progress | +//! | `PUSH(e)` | matches `e` and pushes it's captured string down the stack | +//! +//! where `e`, `e1`, and `e2` are expressions. +//! +//! Matching is greedy, without backtracking. Note the difference in behavior for +//! these two rules in matching identifiers that don't end in an underscore: +//! +//! ```ignore +//! // input: ab_bb_b +//! +//! identifier = @{ "a" ~ ("b"|"_")* ~ "b" } +//! // matches: a b_bb_b nothing -> error! +//! +//! identifier = @{ "a" ~ ("_"* ~ "b")* } +//! // matches: a b, _bb, _b in three repetitions +//! ``` //! //! Expressions can modify the stack only if they match the input. For example, //! if `e1` in the compound expression `e1 | e2` does not match the input, then @@ -181,6 +203,10 @@ //! `e1` did. Repetitions and optionals (`e*`, `e+`, `e{, n}`, `e{n,}`, //! `e{m,n}`, `e?`) can modify the stack each time `e` matches. The `!e` and `&e` //! expressions are a special case; they never modify the stack. +//! Many languages have "keyword" tokens (e.g. if, for, while) as well as general +//! tokens (e.g. identifier) that matches any word. In order to match a keyword, +//! generally, you may need to restrict that is not immediately followed by another +//! letter or digit (otherwise it would be matched as an identifier). //! //! ## Special rules //! @@ -266,7 +292,7 @@ //! ``` //! //! For historical reasons, `PEEK_ALL` matches from top to bottom, while `PEEK[start..end]` matches -//! from bottom to top. There is currectly no syntax to match a slice of the stack top to bottom. +//! from bottom to top. There is currently no syntax to match a slice of the stack top to bottom. //! //! ## `Rule` //! @@ -289,13 +315,10 @@ //! * `ASCII` - matches a character from \x00..\x7f //! * `NEWLINE` - matches either "\n" or "\r\n" or "\r" -#![doc(html_root_url = "https://docs.rs/pest_derive")] - -extern crate pest_generator; -extern crate proc_macro; - use proc_macro::TokenStream; +/// The main method that's called by the proc macro +/// (a wrapper around `pest_generator::derive_parser`) #[proc_macro_derive(Parser, attributes(grammar, grammar_inline))] pub fn derive_parser(input: TokenStream) -> TokenStream { pest_generator::derive_parser(input.into(), true).into() diff --git a/tests/grammar.pest b/tests/grammar.pest index 43a7b8c..126f112 100644 --- a/tests/grammar.pest +++ b/tests/grammar.pest @@ -22,6 +22,7 @@ sequence_atomic_compound = @{ sequence_compound } sequence_nested = { string ~ string } sequence_compound_nested = ${ sequence_nested } choice = { string | range } +choice_prefix = { | string | range } optional = { string? } repeat = { string* } repeat_atomic = @{ string* } @@ -36,6 +37,9 @@ repeat_max = { string{, 2} } repeat_max_atomic = @{ string{, 2} } soi_at_start = { SOI ~ string } repeat_mutate_stack = { (PUSH('a'..'c') ~ ",")* ~ POP ~ POP ~ POP } +repeat_mutate_stack_pop_all = { (PUSH('a'..'c') ~ ",")* ~ POP_ALL } +will_fail = { repeat_mutate_stack_pop_all ~ "FAIL" } +stack_resume_after_fail = { will_fail | repeat_mutate_stack_pop_all } peek_ = { PUSH(range) ~ PUSH(range) ~ PEEK ~ PEEK } peek_all = { PUSH(range) ~ PUSH(range) ~ PEEK_ALL } peek_slice_23 = { PUSH(range) ~ PUSH(range) ~ PUSH(range) ~ PUSH(range) ~ PUSH(range) ~ PEEK[1..-2] } diff --git a/tests/grammar.rs b/tests/grammar.rs index 799beb7..cf5a4a6 100644 --- a/tests/grammar.rs +++ b/tests/grammar.rs @@ -6,6 +6,9 @@ // license , at your // option. All files in the project carrying such notice may not be copied, // modified, or distributed except according to those terms. +#![cfg_attr(not(feature = "std"), no_std)] +extern crate alloc; +use alloc::{format, vec::Vec}; #[macro_use] extern crate pest; @@ -13,7 +16,7 @@ extern crate pest; extern crate pest_derive; #[derive(Parser)] -#[grammar = "../tests/grammar.pest"] +#[grammar = "tests/grammar.pest"] struct GrammarParser; #[test] @@ -245,6 +248,20 @@ fn choice_range() { }; } +#[test] +fn choice_prefix() { + parses_to! { + parser: GrammarParser, + input: "abc", + rule: Rule::choice_prefix, + tokens: [ + choice_prefix(0, 3, [ + string(0, 3) + ]) + ] + }; +} + #[test] fn optional_string() { parses_to! { @@ -782,6 +799,20 @@ fn repeat_mutate_stack() { }; } +#[test] +fn stack_resume_after_fail() { + parses_to! { + parser: GrammarParser, + input: "a,b,c,cba", + rule: Rule::stack_resume_after_fail, + tokens: [ + stack_resume_after_fail(0, 9, [ + repeat_mutate_stack_pop_all(0, 9) + ]) + ] + }; +} + #[test] fn checkpoint_restore() { parses_to! { diff --git a/tests/grammar_inline.rs b/tests/grammar_inline.rs index 2cc730a..549a814 100644 --- a/tests/grammar_inline.rs +++ b/tests/grammar_inline.rs @@ -4,6 +4,10 @@ // option. All files in the project carrying such notice may not be copied, // modified, or distributed except according to those terms. +#![cfg_attr(not(feature = "std"), no_std)] +extern crate alloc; +use alloc::{format, vec::Vec}; + #[macro_use] extern crate pest; #[macro_use] diff --git a/tests/lists.rs b/tests/lists.rs index 4ba0eff..3678b60 100644 --- a/tests/lists.rs +++ b/tests/lists.rs @@ -7,6 +7,10 @@ // option. All files in the project carrying such notice may not be copied, // modified, or distributed except according to those terms. +#![cfg_attr(not(feature = "std"), no_std)] +extern crate alloc; +use alloc::{format, vec::Vec}; + #[macro_use] extern crate pest; #[macro_use] diff --git a/tests/reporting.rs b/tests/reporting.rs index aa0a974..8a41bef 100644 --- a/tests/reporting.rs +++ b/tests/reporting.rs @@ -7,6 +7,10 @@ // option. All files in the project carrying such notice may not be copied, // modified, or distributed except according to those terms. +#![cfg_attr(not(feature = "std"), no_std)] +extern crate alloc; +use alloc::vec; + #[macro_use] extern crate pest; #[macro_use] -- cgit v1.2.3