From 4a8e734e53eff9b3f001ee924358db0db596c6a5 Mon Sep 17 00:00:00 2001 From: Jeff Vander Stoep Date: Tue, 13 Dec 2022 13:07:23 +0100 Subject: Upgrade proc-macro2 to 1.0.47 This project was upgraded with external_updater. Usage: tools/external_updater/updater.sh update rust/crates/proc-macro2 For more info, check https://cs.android.com/android/platform/superproject/+/master:tools/external_updater/README.md Test: TreeHugger Change-Id: Ie945a24b3e0d36cd8b4110031ace1b70e847e6ff --- .cargo_vcs_info.json | 2 +- .github/FUNDING.yml | 1 + .github/workflows/ci.yml | 63 ++++++++++--- Android.bp | 14 ++- Cargo.toml | 33 +++++-- Cargo.toml.orig | 20 ++-- METADATA | 12 ++- README.md | 2 +- build.rs | 34 +++---- src/detection.rs | 2 +- src/fallback.rs | 233 +++++++++++++++++++++++++++++------------------ src/lib.rs | 75 ++++++++++----- src/marker.rs | 4 +- src/parse.rs | 77 +++++++++------- src/rcvec.rs | 142 +++++++++++++++++++++++++++++ src/wrapper.rs | 56 ++++++++---- tests/comments.rs | 2 + tests/test.rs | 80 +++++++++++++++- 18 files changed, 622 insertions(+), 230 deletions(-) create mode 100644 .github/FUNDING.yml create mode 100644 src/rcvec.rs diff --git a/.cargo_vcs_info.json b/.cargo_vcs_info.json index 59bd97e..99281c2 100644 --- a/.cargo_vcs_info.json +++ b/.cargo_vcs_info.json @@ -1,6 +1,6 @@ { "git": { - "sha1": "7ecea3b88fe72672ca4270631b5d4585c0f7c715" + "sha1": "47c91c8525088a055bbfec5df6ea4cd131d01504" }, "path_in_vcs": "" } \ No newline at end of file diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..7507077 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: dtolnay diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2063a5c..32773a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,12 @@ on: pull_request: schedule: [cron: "40 1 * * *"] +permissions: + contents: read + +env: + RUSTFLAGS: -Dwarnings + jobs: test: name: Rust ${{matrix.rust}} @@ -14,57 +20,92 @@ jobs: matrix: rust: [1.31.0, stable, beta] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@master with: toolchain: ${{matrix.rust}} - run: cargo test - run: cargo test --no-default-features - run: cargo test --features span-locations - - run: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test - - run: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test --no-default-features + - name: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test + run: cargo test + env: + RUSTFLAGS: --cfg procmacro2_semver_exempt ${{env.RUSTFLAGS}} + - name: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test --no-default-features + run: cargo test --no-default-features + env: + RUSTFLAGS: --cfg procmacro2_semver_exempt ${{env.RUSTFLAGS}} nightly: name: Rust nightly runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@nightly - run: cargo test - run: cargo test --no-default-features - run: cargo test --no-default-features -- --ignored # run the ignored test to make sure the `proc-macro` feature is disabled - run: cargo test --features span-locations - run: cargo test --manifest-path tests/ui/Cargo.toml - - run: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test - - run: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test --no-default-features - - run: RUSTFLAGS='-Z allow-features=' cargo test + - name: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test + run: cargo test + env: + RUSTFLAGS: --cfg procmacro2_semver_exempt ${{env.RUSTFLAGS}} + - name: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test --no-default-features + run: cargo test --no-default-features + env: + RUSTFLAGS: --cfg procmacro2_semver_exempt ${{env.RUSTFLAGS}} + - name: RUSTFLAGS='-Z allow-features=' cargo test + run: cargo test + env: + RUSTFLAGS: -Z allow-features= ${{env.RUSTFLAGS}} - run: cargo update -Z minimal-versions && cargo build webassembly: name: WebAssembly runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@nightly with: target: wasm32-unknown-unknown - run: cargo test --target wasm32-unknown-unknown --no-run + fuzz: + name: Fuzz + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@nightly + - uses: dtolnay/install@cargo-fuzz + - run: cargo fuzz build -O + clippy: name: Clippy runs-on: ubuntu-latest if: github.event_name != 'pull_request' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@clippy - run: cargo clippy --tests -- -Dclippy::all -Dclippy::pedantic - run: cargo clippy --tests --all-features -- -Dclippy::all -Dclippy::pedantic + miri: + name: Miri + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@miri + - run: cargo miri test + env: + MIRIFLAGS: -Zmiri-strict-provenance + outdated: name: Outdated runs-on: ubuntu-latest if: github.event_name != 'pull_request' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: dtolnay/install@cargo-outdated - - run: cargo outdated --exit-code 1 + - run: cargo outdated --workspace --exit-code 1 + - run: cargo outdated --manifest-path fuzz/Cargo.toml --exit-code 1 diff --git a/Android.bp b/Android.bp index 410a4f4..58461ea 100644 --- a/Android.bp +++ b/Android.bp @@ -41,7 +41,7 @@ rust_library_host { name: "libproc_macro2", crate_name: "proc_macro2", cargo_env_compat: true, - cargo_pkg_version: "1.0.36", + cargo_pkg_version: "1.0.47", srcs: ["src/lib.rs"], edition: "2018", features: [ @@ -50,21 +50,26 @@ rust_library_host { "span-locations", ], cfgs: [ + "proc_macro_span", "span_locations", "use_proc_macro", "wrap_proc_macro", ], rustlibs: [ - "libunicode_xid", + "libunicode_ident", ], compile_multilib: "first", + apex_available: [ + "//apex_available:platform", + "//apex_available:anyapex", + ], } rust_defaults { name: "proc-macro2_test_defaults", crate_name: "proc_macro2", cargo_env_compat: true, - cargo_pkg_version: "1.0.36", + cargo_pkg_version: "1.0.47", test_suites: ["general-tests"], auto_gen_config: true, edition: "2018", @@ -74,6 +79,7 @@ rust_defaults { "span-locations", ], cfgs: [ + "proc_macro_span", "span_locations", "use_proc_macro", "wrap_proc_macro", @@ -81,7 +87,7 @@ rust_defaults { rustlibs: [ "libproc_macro2", "libquote", - "libunicode_xid", + "libunicode_ident", ], } diff --git a/Cargo.toml b/Cargo.toml index 1272f37..1bda7e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,25 +13,42 @@ edition = "2018" rust-version = "1.31" name = "proc-macro2" -version = "1.0.36" -authors = ["David Tolnay ", "Alex Crichton "] +version = "1.0.47" +authors = [ + "David Tolnay ", + "Alex Crichton ", +] autobenches = false -description = "A substitute implementation of the compiler's `proc_macro` API to decouple\ntoken-based libraries from the procedural macro use case.\n" +description = "A substitute implementation of the compiler's `proc_macro` API to decouple token-based libraries from the procedural macro use case." documentation = "https://docs.rs/proc-macro2" readme = "README.md" -keywords = ["macros"] +keywords = [ + "macros", + "syn", +] categories = ["development-tools::procedural-macro-helpers"] license = "MIT OR Apache-2.0" repository = "https://github.com/dtolnay/proc-macro2" + [package.metadata.docs.rs] -rustc-args = ["--cfg", "procmacro2_semver_exempt"] -rustdoc-args = ["--cfg", "procmacro2_semver_exempt", "--cfg", "doc_cfg"] +rustc-args = [ + "--cfg", + "procmacro2_semver_exempt", +] +rustdoc-args = [ + "--cfg", + "procmacro2_semver_exempt", + "--cfg", + "doc_cfg", +] targets = ["x86_64-unknown-linux-gnu"] [package.metadata.playground] features = ["span-locations"] -[dependencies.unicode-xid] -version = "0.2" + +[dependencies.unicode-ident] +version = "1.0" + [dev-dependencies.quote] version = "1.0" default_features = false diff --git a/Cargo.toml.orig b/Cargo.toml.orig index 401bfaf..273683a 100644 --- a/Cargo.toml.orig +++ b/Cargo.toml.orig @@ -1,20 +1,16 @@ [package] name = "proc-macro2" -version = "1.0.36" # remember to update html_root_url +version = "1.0.47" # remember to update html_root_url authors = ["David Tolnay ", "Alex Crichton "] -license = "MIT OR Apache-2.0" -readme = "README.md" -keywords = ["macros"] -repository = "https://github.com/dtolnay/proc-macro2" -documentation = "https://docs.rs/proc-macro2" +autobenches = false categories = ["development-tools::procedural-macro-helpers"] +description = "A substitute implementation of the compiler's `proc_macro` API to decouple token-based libraries from the procedural macro use case." +documentation = "https://docs.rs/proc-macro2" edition = "2018" -autobenches = false +keywords = ["macros", "syn"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/dtolnay/proc-macro2" rust-version = "1.31" -description = """ -A substitute implementation of the compiler's `proc_macro` API to decouple -token-based libraries from the procedural macro use case. -""" [package.metadata.docs.rs] rustc-args = ["--cfg", "procmacro2_semver_exempt"] @@ -25,7 +21,7 @@ targets = ["x86_64-unknown-linux-gnu"] features = ["span-locations"] [dependencies] -unicode-xid = "0.2" +unicode-ident = "1.0" [dev-dependencies] quote = { version = "1.0", default_features = false } diff --git a/METADATA b/METADATA index e2ded2b..ffab613 100644 --- a/METADATA +++ b/METADATA @@ -1,3 +1,7 @@ +# This project was upgraded with external_updater. +# Usage: tools/external_updater/updater.sh update rust/crates/proc-macro2 +# For more info, check https://cs.android.com/android/platform/superproject/+/master:tools/external_updater/README.md + name: "proc-macro2" description: "A substitute implementation of the compiler\'s `proc_macro` API to decouple token-based libraries from the procedural macro use case." third_party { @@ -7,13 +11,13 @@ third_party { } url { type: ARCHIVE - value: "https://static.crates.io/crates/proc-macro2/proc-macro2-1.0.36.crate" + value: "https://static.crates.io/crates/proc-macro2/proc-macro2-1.0.47.crate" } - version: "1.0.36" + version: "1.0.47" license_type: NOTICE last_upgrade_date { year: 2022 - month: 3 - day: 1 + month: 12 + day: 13 } } diff --git a/README.md b/README.md index 35e1876..70b6c86 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [github](https://github.com/dtolnay/proc-macro2) [crates.io](https://crates.io/crates/proc-macro2) -[docs.rs](https://docs.rs/proc-macro2) +[docs.rs](https://docs.rs/proc-macro2) [build status](https://github.com/dtolnay/proc-macro2/actions?query=branch%3Amaster) A wrapper around the procedural macro API of the compiler's `proc_macro` crate. diff --git a/build.rs b/build.rs index 946be6e..b69d813 100644 --- a/build.rs +++ b/build.rs @@ -41,7 +41,6 @@ // 1.57+. use std::env; -use std::iter; use std::process::{self, Command}; use std::str; @@ -85,6 +84,10 @@ fn main() { println!("cargo:rustc-cfg=no_hygiene"); } + if version.minor < 47 { + println!("cargo:rustc-cfg=no_ident_new_raw"); + } + if version.minor < 54 { println!("cargo:rustc-cfg=no_literal_from_str"); } @@ -108,7 +111,10 @@ fn main() { println!("cargo:rustc-cfg=wrap_proc_macro"); } - if version.nightly && feature_allowed("proc_macro_span") { + if version.nightly + && feature_allowed("proc_macro_span") + && feature_allowed("proc_macro_span_shrink") + { println!("cargo:rustc-cfg=proc_macro_span"); } @@ -154,23 +160,13 @@ fn feature_allowed(feature: &str) -> bool { let flags_var; let flags_var_string; - let mut flags_var_split; - let mut flags_none; - let flags: &mut dyn Iterator = - if let Some(encoded_rustflags) = env::var_os("CARGO_ENCODED_RUSTFLAGS") { - flags_var = encoded_rustflags; - flags_var_string = flags_var.to_string_lossy(); - flags_var_split = flags_var_string.split('\x1f'); - &mut flags_var_split - } else if let Some(rustflags) = env::var_os("RUSTFLAGS") { - flags_var = rustflags; - flags_var_string = flags_var.to_string_lossy(); - flags_var_split = flags_var_string.split(' '); - &mut flags_var_split - } else { - flags_none = iter::empty(); - &mut flags_none - }; + let flags = if let Some(encoded_rustflags) = env::var_os("CARGO_ENCODED_RUSTFLAGS") { + flags_var = encoded_rustflags; + flags_var_string = flags_var.to_string_lossy(); + flags_var_string.split('\x1f') + } else { + return true; + }; for mut flag in flags { if flag.starts_with("-Z") { diff --git a/src/detection.rs b/src/detection.rs index d139b73..beba7b2 100644 --- a/src/detection.rs +++ b/src/detection.rs @@ -1,4 +1,4 @@ -use std::sync::atomic::{AtomicUsize, Ordering}; +use core::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Once; static WORKS: AtomicUsize = AtomicUsize::new(0); diff --git a/src/fallback.rs b/src/fallback.rs index ac5437d..fe4f248 100644 --- a/src/fallback.rs +++ b/src/fallback.rs @@ -1,19 +1,19 @@ use crate::parse::{self, Cursor}; +use crate::rcvec::{RcVec, RcVecBuilder, RcVecIntoIter, RcVecMut}; use crate::{Delimiter, Spacing, TokenTree}; #[cfg(span_locations)] -use std::cell::RefCell; +use core::cell::RefCell; #[cfg(span_locations)] -use std::cmp; -use std::fmt::{self, Debug, Display}; -use std::iter::FromIterator; -use std::mem; -use std::ops::RangeBounds; +use core::cmp; +use core::fmt::{self, Debug, Display, Write}; +use core::iter::FromIterator; +use core::mem::ManuallyDrop; +use core::ops::RangeBounds; +use core::ptr; +use core::str::FromStr; #[cfg(procmacro2_semver_exempt)] use std::path::Path; use std::path::PathBuf; -use std::str::FromStr; -use std::vec; -use unicode_xid::UnicodeXID; /// Force use of proc-macro2's fallback implementation of the API for now, even /// if the compiler's implementation is available. @@ -31,7 +31,7 @@ pub fn unforce() { #[derive(Clone)] pub(crate) struct TokenStream { - inner: Vec, + inner: RcVec, } #[derive(Debug)] @@ -53,71 +53,69 @@ impl LexError { impl TokenStream { pub fn new() -> Self { - TokenStream { inner: Vec::new() } + TokenStream { + inner: RcVecBuilder::new().build(), + } } pub fn is_empty(&self) -> bool { self.inner.len() == 0 } - fn take_inner(&mut self) -> Vec { - mem::replace(&mut self.inner, Vec::new()) + fn take_inner(self) -> RcVecBuilder { + let nodrop = ManuallyDrop::new(self); + unsafe { ptr::read(&nodrop.inner) }.make_owned() } +} - fn push_token(&mut self, token: TokenTree) { - // https://github.com/dtolnay/proc-macro2/issues/235 - match token { - #[cfg(not(no_bind_by_move_pattern_guard))] - TokenTree::Literal(crate::Literal { - #[cfg(wrap_proc_macro)] - inner: crate::imp::Literal::Fallback(literal), - #[cfg(not(wrap_proc_macro))] - inner: literal, - .. - }) if literal.repr.starts_with('-') => { - push_negative_literal(self, literal); - } - #[cfg(no_bind_by_move_pattern_guard)] - TokenTree::Literal(crate::Literal { - #[cfg(wrap_proc_macro)] - inner: crate::imp::Literal::Fallback(literal), - #[cfg(not(wrap_proc_macro))] - inner: literal, - .. - }) => { - if literal.repr.starts_with('-') { - push_negative_literal(self, literal); - } else { - self.inner - .push(TokenTree::Literal(crate::Literal::_new_stable(literal))); - } - } - _ => self.inner.push(token), +fn push_token_from_proc_macro(mut vec: RcVecMut, token: TokenTree) { + // https://github.com/dtolnay/proc-macro2/issues/235 + match token { + #[cfg(not(no_bind_by_move_pattern_guard))] + TokenTree::Literal(crate::Literal { + #[cfg(wrap_proc_macro)] + inner: crate::imp::Literal::Fallback(literal), + #[cfg(not(wrap_proc_macro))] + inner: literal, + .. + }) if literal.repr.starts_with('-') => { + push_negative_literal(vec, literal); } - - #[cold] - fn push_negative_literal(stream: &mut TokenStream, mut literal: Literal) { - literal.repr.remove(0); - let mut punct = crate::Punct::new('-', Spacing::Alone); - punct.set_span(crate::Span::_new_stable(literal.span)); - stream.inner.push(TokenTree::Punct(punct)); - stream - .inner - .push(TokenTree::Literal(crate::Literal::_new_stable(literal))); + #[cfg(no_bind_by_move_pattern_guard)] + TokenTree::Literal(crate::Literal { + #[cfg(wrap_proc_macro)] + inner: crate::imp::Literal::Fallback(literal), + #[cfg(not(wrap_proc_macro))] + inner: literal, + .. + }) => { + if literal.repr.starts_with('-') { + push_negative_literal(vec, literal); + } else { + vec.push(TokenTree::Literal(crate::Literal::_new_stable(literal))); + } } + _ => vec.push(token), } -} -impl From> for TokenStream { - fn from(inner: Vec) -> Self { - TokenStream { inner } + #[cold] + fn push_negative_literal(mut vec: RcVecMut, mut literal: Literal) { + literal.repr.remove(0); + let mut punct = crate::Punct::new('-', Spacing::Alone); + punct.set_span(crate::Span::_new_stable(literal.span)); + vec.push(TokenTree::Punct(punct)); + vec.push(TokenTree::Literal(crate::Literal::_new_stable(literal))); } } // Nonrecursive to prevent stack overflow. impl Drop for TokenStream { fn drop(&mut self) { - while let Some(token) = self.inner.pop() { + let mut inner = match self.inner.get_mut() { + Some(inner) => inner, + None => return, + }; + while let Some(token) = inner.pop() { let group = match token { TokenTree::Group(group) => group.inner, _ => continue, @@ -127,8 +125,35 @@ impl Drop for TokenStream { crate::imp::Group::Fallback(group) => group, crate::imp::Group::Compiler(_) => continue, }; - let mut group = group; - self.inner.extend(group.stream.take_inner()); + inner.extend(group.stream.take_inner()); + } + } +} + +pub(crate) struct TokenStreamBuilder { + inner: RcVecBuilder, +} + +impl TokenStreamBuilder { + pub fn new() -> Self { + TokenStreamBuilder { + inner: RcVecBuilder::new(), + } + } + + pub fn with_capacity(cap: usize) -> Self { + TokenStreamBuilder { + inner: RcVecBuilder::with_capacity(cap), + } + } + + pub fn push_token_from_parser(&mut self, tt: TokenTree) { + self.inner.push(tt); + } + + pub fn build(self) -> TokenStream { + TokenStream { + inner: self.inner.build(), } } } @@ -157,7 +182,13 @@ impl FromStr for TokenStream { fn from_str(src: &str) -> Result { // Create a dummy file & add it to the source map - let cursor = get_cursor(src); + let mut cursor = get_cursor(src); + + // Strip a byte order mark if present + const BYTE_ORDER_MARK: &str = "\u{feff}"; + if cursor.starts_with(BYTE_ORDER_MARK) { + cursor = cursor.advance(BYTE_ORDER_MARK.len()); + } parse::token_stream(cursor) } @@ -221,9 +252,11 @@ impl From for proc_macro::TokenStream { impl From for TokenStream { fn from(tree: TokenTree) -> TokenStream { - let mut stream = TokenStream::new(); - stream.push_token(tree); - stream + let mut stream = RcVecBuilder::new(); + push_token_from_proc_macro(stream.as_mut(), tree); + TokenStream { + inner: stream.build(), + } } } @@ -237,35 +270,38 @@ impl FromIterator for TokenStream { impl FromIterator for TokenStream { fn from_iter>(streams: I) -> Self { - let mut v = Vec::new(); + let mut v = RcVecBuilder::new(); - for mut stream in streams { + for stream in streams { v.extend(stream.take_inner()); } - TokenStream { inner: v } + TokenStream { inner: v.build() } } } impl Extend for TokenStream { fn extend>(&mut self, tokens: I) { - tokens.into_iter().for_each(|token| self.push_token(token)); + let mut vec = self.inner.make_mut(); + tokens + .into_iter() + .for_each(|token| push_token_from_proc_macro(vec.as_mut(), token)); } } impl Extend for TokenStream { fn extend>(&mut self, streams: I) { - self.inner.extend(streams.into_iter().flatten()); + self.inner.make_mut().extend(streams.into_iter().flatten()); } } -pub(crate) type TokenTreeIter = vec::IntoIter; +pub(crate) type TokenTreeIter = RcVecIntoIter; impl IntoIterator for TokenStream { type Item = TokenTree; type IntoIter = TokenTreeIter; - fn into_iter(mut self) -> TokenTreeIter { + fn into_iter(self) -> TokenTreeIter { self.take_inner().into_iter() } } @@ -384,7 +420,7 @@ impl SourceMap { fn add_file(&mut self, name: &str, src: &str) -> Span { let (len, lines) = lines_offsets(src); let lo = self.next_start_pos(); - // XXX(nika): Shouild we bother doing a checked cast or checked add here? + // XXX(nika): Should we bother doing a checked cast or checked add here? let span = Span { lo, hi: lo + (len as u32), @@ -482,6 +518,26 @@ impl Span { }) } + #[cfg(procmacro2_semver_exempt)] + pub fn before(&self) -> Span { + Span { + #[cfg(span_locations)] + lo: self.lo, + #[cfg(span_locations)] + hi: self.lo, + } + } + + #[cfg(procmacro2_semver_exempt)] + pub fn after(&self) -> Span { + Span { + #[cfg(span_locations)] + lo: self.hi, + #[cfg(span_locations)] + hi: self.hi, + } + } + #[cfg(not(span_locations))] pub fn join(&self, _other: Span) -> Option { Some(Span {}) @@ -639,7 +695,7 @@ pub(crate) struct Ident { impl Ident { fn _new(string: &str, raw: bool, span: Span) -> Self { - validate_ident(string); + validate_ident(string, raw); Ident { sym: string.to_owned(), @@ -666,27 +722,19 @@ impl Ident { } pub(crate) fn is_ident_start(c: char) -> bool { - ('a' <= c && c <= 'z') - || ('A' <= c && c <= 'Z') - || c == '_' - || (c > '\x7f' && UnicodeXID::is_xid_start(c)) + c == '_' || unicode_ident::is_xid_start(c) } pub(crate) fn is_ident_continue(c: char) -> bool { - ('a' <= c && c <= 'z') - || ('A' <= c && c <= 'Z') - || c == '_' - || ('0' <= c && c <= '9') - || (c > '\x7f' && UnicodeXID::is_xid_continue(c)) + unicode_ident::is_xid_continue(c) } -fn validate_ident(string: &str) { - let validate = string; - if validate.is_empty() { +fn validate_ident(string: &str, raw: bool) { + if string.is_empty() { panic!("Ident is not allowed to be empty; use Option"); } - if validate.bytes().all(|digit| digit >= b'0' && digit <= b'9') { + if string.bytes().all(|digit| digit >= b'0' && digit <= b'9') { panic!("Ident cannot be a number; use Literal instead"); } @@ -704,9 +752,18 @@ fn validate_ident(string: &str) { true } - if !ident_ok(validate) { + if !ident_ok(string) { panic!("{:?} is not a valid Ident", string); } + + if raw { + match string { + "_" | "super" | "self" | "Self" | "crate" => { + panic!("`r#{}` cannot be a raw identifier", string); + } + _ => {} + } + } } impl PartialEq for Ident { @@ -883,7 +940,9 @@ impl Literal { b'"' => escaped.push_str("\\\""), b'\\' => escaped.push_str("\\\\"), b'\x20'..=b'\x7E' => escaped.push(*b as char), - _ => escaped.push_str(&format!("\\x{:02X}", b)), + _ => { + let _ = write!(escaped, "\\x{:02X}", b); + } } } escaped.push('"'); diff --git a/src/lib.rs b/src/lib.rs index 6edaf42..3fda02d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,7 +2,7 @@ //! //! [github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github //! [crates-io]: https://img.shields.io/badge/crates.io-fc8d62?style=for-the-badge&labelColor=555555&logo=rust -//! [docs-rs]: https://img.shields.io/badge/docs.rs-66c2a5?style=for-the-badge&labelColor=555555&logoColor=white&logo=data:image/svg+xml;base64,PHN2ZyByb2xlPSJpbWciIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgdmlld0JveD0iMCAwIDUxMiA1MTIiPjxwYXRoIGZpbGw9IiNmNWY1ZjUiIGQ9Ik00ODguNiAyNTAuMkwzOTIgMjE0VjEwNS41YzAtMTUtOS4zLTI4LjQtMjMuNC0zMy43bC0xMDAtMzcuNWMtOC4xLTMuMS0xNy4xLTMuMS0yNS4zIDBsLTEwMCAzNy41Yy0xNC4xIDUuMy0yMy40IDE4LjctMjMuNCAzMy43VjIxNGwtOTYuNiAzNi4yQzkuMyAyNTUuNSAwIDI2OC45IDAgMjgzLjlWMzk0YzAgMTMuNiA3LjcgMjYuMSAxOS45IDMyLjJsMTAwIDUwYzEwLjEgNS4xIDIyLjEgNS4xIDMyLjIgMGwxMDMuOS01MiAxMDMuOSA1MmMxMC4xIDUuMSAyMi4xIDUuMSAzMi4yIDBsMTAwLTUwYzEyLjItNi4xIDE5LjktMTguNiAxOS45LTMyLjJWMjgzLjljMC0xNS05LjMtMjguNC0yMy40LTMzLjd6TTM1OCAyMTQuOGwtODUgMzEuOXYtNjguMmw4NS0zN3Y3My4zek0xNTQgMTA0LjFsMTAyLTM4LjIgMTAyIDM4LjJ2LjZsLTEwMiA0MS40LTEwMi00MS40di0uNnptODQgMjkxLjFsLTg1IDQyLjV2LTc5LjFsODUtMzguOHY3NS40em0wLTExMmwtMTAyIDQxLjQtMTAyLTQxLjR2LS42bDEwMi0zOC4yIDEwMiAzOC4ydi42em0yNDAgMTEybC04NSA0Mi41di03OS4xbDg1LTM4Ljh2NzUuNHptMC0xMTJsLTEwMiA0MS40LTEwMi00MS40di0uNmwxMDItMzguMiAxMDIgMzguMnYuNnoiPjwvcGF0aD48L3N2Zz4K +//! [docs-rs]: https://img.shields.io/badge/docs.rs-66c2a5?style=for-the-badge&labelColor=555555&logo=docs.rs //! //!
//! @@ -86,8 +86,11 @@ //! a different thread. // Proc-macro2 types in rustdoc of other crates get linked to here. -#![doc(html_root_url = "https://docs.rs/proc-macro2/1.0.36")] -#![cfg_attr(any(proc_macro_span, super_unstable), feature(proc_macro_span))] +#![doc(html_root_url = "https://docs.rs/proc-macro2/1.0.47")] +#![cfg_attr( + any(proc_macro_span, super_unstable), + feature(proc_macro_span, proc_macro_span_shrink) +)] #![cfg_attr(super_unstable, feature(proc_macro_def_site))] #![cfg_attr(doc_cfg, feature(doc_cfg))] #![allow( @@ -120,6 +123,7 @@ extern crate proc_macro; mod marker; mod parse; +mod rcvec; #[cfg(wrap_proc_macro)] mod detection; @@ -136,15 +140,15 @@ use crate::fallback as imp; mod imp; use crate::marker::Marker; -use std::cmp::Ordering; +use core::cmp::Ordering; +use core::fmt::{self, Debug, Display}; +use core::hash::{Hash, Hasher}; +use core::iter::FromIterator; +use core::ops::RangeBounds; +use core::str::FromStr; use std::error::Error; -use std::fmt::{self, Debug, Display}; -use std::hash::{Hash, Hasher}; -use std::iter::FromIterator; -use std::ops::RangeBounds; #[cfg(procmacro2_semver_exempt)] use std::path::PathBuf; -use std::str::FromStr; /// An abstract stream of tokens, or more concretely a sequence of token trees. /// @@ -508,6 +512,24 @@ impl Span { LineColumn { line, column } } + /// Creates an empty span pointing to directly before this span. + /// + /// This method is semver exempt and not exposed by default. + #[cfg(all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)))] + #[cfg_attr(doc_cfg, doc(cfg(procmacro2_semver_exempt)))] + pub fn before(&self) -> Span { + Span::_new(self.inner.before()) + } + + /// Creates an empty span pointing to directly after this span. + /// + /// This method is semver exempt and not exposed by default. + #[cfg(all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)))] + #[cfg_attr(doc_cfg, doc(cfg(procmacro2_semver_exempt)))] + pub fn after(&self) -> Span { + Span::_new(self.inner.after()) + } + /// Create a new span encompassing `self` and `other`. /// /// Returns `None` if `self` and `other` are from different files. @@ -952,11 +974,11 @@ impl Ident { Ident::_new(imp::Ident::new(string, span.inner)) } - /// Same as `Ident::new`, but creates a raw identifier (`r#ident`). - /// - /// This method is semver exempt and not exposed by default. - #[cfg(procmacro2_semver_exempt)] - #[cfg_attr(doc_cfg, doc(cfg(procmacro2_semver_exempt)))] + /// Same as `Ident::new`, but creates a raw identifier (`r#ident`). The + /// `string` argument must be a valid identifier permitted by the language + /// (including keywords, e.g. `fn`). Keywords which are usable in path + /// segments (e.g. `self`, `super`) are not supported, and will cause a + /// panic. pub fn new_raw(string: &str, span: Span) -> Self { Ident::_new_raw(string, span) } @@ -1128,9 +1150,9 @@ impl Literal { /// This constructor is similar to those like `Literal::i8_unsuffixed` where /// the float's value is emitted directly into the token but no suffix is /// used, so it may be inferred to be a `f64` later in the compiler. - /// Literals created from negative numbers may not survive rountrips through - /// `TokenStream` or strings and may be broken into two tokens (`-` and - /// positive literal). + /// Literals created from negative numbers may not survive round-trips + /// through `TokenStream` or strings and may be broken into two tokens (`-` + /// and positive literal). /// /// # Panics /// @@ -1147,7 +1169,7 @@ impl Literal { /// specified is the preceding part of the token and `f64` is the suffix of /// the token. This token will always be inferred to be an `f64` in the /// compiler. Literals created from negative numbers may not survive - /// rountrips through `TokenStream` or strings and may be broken into two + /// round-trips through `TokenStream` or strings and may be broken into two /// tokens (`-` and positive literal). /// /// # Panics @@ -1164,9 +1186,9 @@ impl Literal { /// This constructor is similar to those like `Literal::i8_unsuffixed` where /// the float's value is emitted directly into the token but no suffix is /// used, so it may be inferred to be a `f64` later in the compiler. - /// Literals created from negative numbers may not survive rountrips through - /// `TokenStream` or strings and may be broken into two tokens (`-` and - /// positive literal). + /// Literals created from negative numbers may not survive round-trips + /// through `TokenStream` or strings and may be broken into two tokens (`-` + /// and positive literal). /// /// # Panics /// @@ -1183,7 +1205,7 @@ impl Literal { /// specified is the preceding part of the token and `f32` is the suffix of /// the token. This token will always be inferred to be an `f32` in the /// compiler. Literals created from negative numbers may not survive - /// rountrips through `TokenStream` or strings and may be broken into two + /// round-trips through `TokenStream` or strings and may be broken into two /// tokens (`-` and positive literal). /// /// # Panics @@ -1270,7 +1292,7 @@ impl Display for Literal { pub mod token_stream { use crate::marker::Marker; use crate::{imp, TokenTree}; - use std::fmt::{self, Debug}; + use core::fmt::{self, Debug}; pub use crate::TokenStream; @@ -1290,11 +1312,16 @@ pub mod token_stream { fn next(&mut self) -> Option { self.inner.next() } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } } impl Debug for IntoIter { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - Debug::fmt(&self.inner, f) + f.write_str("TokenStream ")?; + f.debug_list().entries(self.clone()).finish() } } diff --git a/src/marker.rs b/src/marker.rs index 58729ba..59fd096 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -1,4 +1,4 @@ -use std::marker::PhantomData; +use core::marker::PhantomData; use std::panic::{RefUnwindSafe, UnwindSafe}; use std::rc::Rc; @@ -9,7 +9,7 @@ pub(crate) type Marker = PhantomData; pub(crate) use self::value::*; mod value { - pub(crate) use std::marker::PhantomData as Marker; + pub(crate) use core::marker::PhantomData as Marker; } pub(crate) struct ProcMacroAutoTraits(Rc<()>); diff --git a/src/parse.rs b/src/parse.rs index f77213a..307e065 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -1,9 +1,10 @@ use crate::fallback::{ is_ident_continue, is_ident_start, Group, LexError, Literal, Span, TokenStream, + TokenStreamBuilder, }; use crate::{Delimiter, Punct, Spacing, TokenTree}; -use std::char; -use std::str::{Bytes, CharIndices, Chars}; +use core::char; +use core::str::{Bytes, CharIndices, Chars}; #[derive(Copy, Clone, Eq, PartialEq)] pub(crate) struct Cursor<'a> { @@ -13,7 +14,7 @@ pub(crate) struct Cursor<'a> { } impl<'a> Cursor<'a> { - fn advance(&self, bytes: usize) -> Cursor<'a> { + pub fn advance(&self, bytes: usize) -> Cursor<'a> { let (_front, rest) = self.rest.split_at(bytes); Cursor { rest, @@ -22,7 +23,7 @@ impl<'a> Cursor<'a> { } } - fn starts_with(&self, s: &str) -> bool { + pub fn starts_with(&self, s: &str) -> bool { self.rest.starts_with(s) } @@ -115,9 +116,9 @@ fn block_comment(input: Cursor) -> PResult<&str> { return Err(Reject); } - let mut depth = 0; + let mut depth = 0usize; let bytes = input.as_bytes(); - let mut i = 0; + let mut i = 0usize; let upper = bytes.len() - 1; while i < upper { @@ -150,14 +151,13 @@ fn word_break(input: Cursor) -> Result { } pub(crate) fn token_stream(mut input: Cursor) -> Result { - let mut trees = Vec::new(); + let mut trees = TokenStreamBuilder::new(); let mut stack = Vec::new(); loop { input = skip_whitespace(input); - if let Ok((rest, tt)) = doc_comment(input) { - trees.extend(tt); + if let Ok((rest, ())) = doc_comment(input, &mut trees) { input = rest; continue; } @@ -168,7 +168,7 @@ pub(crate) fn token_stream(mut input: Cursor) -> Result { let first = match input.bytes().next() { Some(first) => first, None => match stack.last() { - None => return Ok(TokenStream::from(trees)), + None => return Ok(trees.build()), #[cfg(span_locations)] Some((lo, _frame)) => { return Err(LexError { @@ -191,7 +191,7 @@ pub(crate) fn token_stream(mut input: Cursor) -> Result { #[cfg(span_locations)] let frame = (lo, frame); stack.push(frame); - trees = Vec::new(); + trees = TokenStreamBuilder::new(); } else if let Some(close_delimiter) = match first { b')' => Some(Delimiter::Parenthesis), b']' => Some(Delimiter::Bracket), @@ -209,7 +209,7 @@ pub(crate) fn token_stream(mut input: Cursor) -> Result { return Err(lex_error(input)); } input = input.advance(1); - let mut g = Group::new(open_delimiter, TokenStream::from(trees)); + let mut g = Group::new(open_delimiter, trees.build()); g.set_span(Span { #[cfg(span_locations)] lo, @@ -217,7 +217,7 @@ pub(crate) fn token_stream(mut input: Cursor) -> Result { hi: input.off, }); trees = outer; - trees.push(TokenTree::Group(crate::Group::_new_stable(g))); + trees.push_token_from_parser(TokenTree::Group(crate::Group::_new_stable(g))); } else { let (rest, mut tt) = match leaf_token(input) { Ok((rest, tt)) => (rest, tt), @@ -229,7 +229,7 @@ pub(crate) fn token_stream(mut input: Cursor) -> Result { #[cfg(span_locations)] hi: rest.off, })); - trees.push(tt); + trees.push_token_from_parser(tt); input = rest; } } @@ -283,8 +283,9 @@ fn ident_any(input: Cursor) -> PResult { return Ok((rest, ident)); } - if sym == "_" { - return Err(Reject); + match sym { + "_" | "super" | "self" | "Self" | "crate" => return Err(Reject), + _ => {} } let ident = crate::Ident::_new_raw(sym, crate::Span::call_site()); @@ -786,7 +787,7 @@ fn punct_char(input: Cursor) -> PResult { } } -fn doc_comment(input: Cursor) -> PResult> { +fn doc_comment<'a>(input: Cursor<'a>, trees: &mut TokenStreamBuilder) -> PResult<'a, ()> { #[cfg(span_locations)] let lo = input.off; let (rest, (comment, inner)) = doc_comment_contents(input)?; @@ -806,25 +807,31 @@ fn doc_comment(input: Cursor) -> PResult> { scan_for_bare_cr = rest; } - let mut trees = Vec::new(); - trees.push(TokenTree::Punct(Punct::new('#', Spacing::Alone))); + let mut pound = Punct::new('#', Spacing::Alone); + pound.set_span(span); + trees.push_token_from_parser(TokenTree::Punct(pound)); + if inner { - trees.push(Punct::new('!', Spacing::Alone).into()); - } - let mut stream = vec![ - TokenTree::Ident(crate::Ident::new("doc", span)), - TokenTree::Punct(Punct::new('=', Spacing::Alone)), - TokenTree::Literal(crate::Literal::string(comment)), - ]; - for tt in &mut stream { - tt.set_span(span); - } - let group = Group::new(Delimiter::Bracket, TokenStream::from(stream)); - trees.push(crate::Group::_new_stable(group).into()); - for tt in &mut trees { - tt.set_span(span); - } - Ok((rest, trees)) + let mut bang = Punct::new('!', Spacing::Alone); + bang.set_span(span); + trees.push_token_from_parser(TokenTree::Punct(bang)); + } + + let doc_ident = crate::Ident::new("doc", span); + let mut equal = Punct::new('=', Spacing::Alone); + equal.set_span(span); + let mut literal = crate::Literal::string(comment); + literal.set_span(span); + let mut bracketed = TokenStreamBuilder::with_capacity(3); + bracketed.push_token_from_parser(TokenTree::Ident(doc_ident)); + bracketed.push_token_from_parser(TokenTree::Punct(equal)); + bracketed.push_token_from_parser(TokenTree::Literal(literal)); + let group = Group::new(Delimiter::Bracket, bracketed.build()); + let mut group = crate::Group::_new_stable(group); + group.set_span(span); + trees.push_token_from_parser(TokenTree::Group(group)); + + Ok((rest, ())) } fn doc_comment_contents(input: Cursor) -> PResult<(&str, bool)> { diff --git a/src/rcvec.rs b/src/rcvec.rs new file mode 100644 index 0000000..86ca7d8 --- /dev/null +++ b/src/rcvec.rs @@ -0,0 +1,142 @@ +use core::mem; +use core::slice; +use std::rc::Rc; +use std::vec; + +pub(crate) struct RcVec { + inner: Rc>, +} + +pub(crate) struct RcVecBuilder { + inner: Vec, +} + +pub(crate) struct RcVecMut<'a, T> { + inner: &'a mut Vec, +} + +#[derive(Clone)] +pub(crate) struct RcVecIntoIter { + inner: vec::IntoIter, +} + +impl RcVec { + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + pub fn len(&self) -> usize { + self.inner.len() + } + + pub fn iter(&self) -> slice::Iter { + self.inner.iter() + } + + pub fn make_mut(&mut self) -> RcVecMut + where + T: Clone, + { + RcVecMut { + inner: Rc::make_mut(&mut self.inner), + } + } + + pub fn get_mut(&mut self) -> Option> { + let inner = Rc::get_mut(&mut self.inner)?; + Some(RcVecMut { inner }) + } + + pub fn make_owned(mut self) -> RcVecBuilder + where + T: Clone, + { + let vec = if let Some(owned) = Rc::get_mut(&mut self.inner) { + mem::replace(owned, Vec::new()) + } else { + Vec::clone(&self.inner) + }; + RcVecBuilder { inner: vec } + } +} + +impl RcVecBuilder { + pub fn new() -> Self { + RcVecBuilder { inner: Vec::new() } + } + + pub fn with_capacity(cap: usize) -> Self { + RcVecBuilder { + inner: Vec::with_capacity(cap), + } + } + + pub fn push(&mut self, element: T) { + self.inner.push(element); + } + + pub fn extend(&mut self, iter: impl IntoIterator) { + self.inner.extend(iter); + } + + pub fn as_mut(&mut self) -> RcVecMut { + RcVecMut { + inner: &mut self.inner, + } + } + + pub fn build(self) -> RcVec { + RcVec { + inner: Rc::new(self.inner), + } + } +} + +impl<'a, T> RcVecMut<'a, T> { + pub fn push(&mut self, element: T) { + self.inner.push(element); + } + + pub fn extend(&mut self, iter: impl IntoIterator) { + self.inner.extend(iter); + } + + pub fn pop(&mut self) -> Option { + self.inner.pop() + } + + pub fn as_mut(&mut self) -> RcVecMut { + RcVecMut { inner: self.inner } + } +} + +impl Clone for RcVec { + fn clone(&self) -> Self { + RcVec { + inner: Rc::clone(&self.inner), + } + } +} + +impl IntoIterator for RcVecBuilder { + type Item = T; + type IntoIter = RcVecIntoIter; + + fn into_iter(self) -> Self::IntoIter { + RcVecIntoIter { + inner: self.inner.into_iter(), + } + } +} + +impl Iterator for RcVecIntoIter { + type Item = T; + + fn next(&mut self) -> Option { + self.inner.next() + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} diff --git a/src/wrapper.rs b/src/wrapper.rs index 2ba76cc..47d1494 100644 --- a/src/wrapper.rs +++ b/src/wrapper.rs @@ -1,12 +1,12 @@ use crate::detection::inside_proc_macro; use crate::{fallback, Delimiter, Punct, Spacing, TokenTree}; -use std::fmt::{self, Debug, Display}; -use std::iter::FromIterator; -use std::ops::RangeBounds; +use core::fmt::{self, Debug, Display}; +use core::iter::FromIterator; +use core::ops::RangeBounds; +use core::str::FromStr; use std::panic; #[cfg(super_unstable)] use std::path::PathBuf; -use std::str::FromStr; #[derive(Clone)] pub(crate) enum TokenStream { @@ -350,12 +350,6 @@ impl Iterator for TokenTreeIter { } } -impl Debug for TokenTreeIter { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("TokenTreeIter").finish() - } -} - #[derive(Clone, PartialEq, Eq)] #[cfg(super_unstable)] pub(crate) enum SourceFile { @@ -511,6 +505,22 @@ impl Span { } } + #[cfg(super_unstable)] + pub fn before(&self) -> Span { + match self { + Span::Compiler(s) => Span::Compiler(s.before()), + Span::Fallback(s) => Span::Fallback(s.before()), + } + } + + #[cfg(super_unstable)] + pub fn after(&self) -> Span { + match self { + Span::Compiler(s) => Span::Compiler(s.after()), + Span::Fallback(s) => Span::Fallback(s.after()), + } + } + pub fn join(&self, other: Span) -> Option { let ret = match (self, other) { #[cfg(proc_macro_span)] @@ -694,16 +704,26 @@ impl Ident { pub fn new_raw(string: &str, span: Span) -> Self { match span { + #[cfg(not(no_ident_new_raw))] + Span::Compiler(s) => Ident::Compiler(proc_macro::Ident::new_raw(string, s)), + #[cfg(no_ident_new_raw)] Span::Compiler(s) => { - let p: proc_macro::TokenStream = string.parse().unwrap(); - let ident = match p.into_iter().next() { - Some(proc_macro::TokenTree::Ident(mut i)) => { - i.set_span(s); - i + let _ = proc_macro::Ident::new(string, s); + // At this point the un-r#-prefixed string is known to be a + // valid identifier. Try to produce a valid raw identifier by + // running the `TokenStream` parser, and unwrapping the first + // token as an `Ident`. + let raw_prefixed = format!("r#{}", string); + if let Ok(ts) = raw_prefixed.parse::() { + let mut iter = ts.into_iter(); + if let (Some(proc_macro::TokenTree::Ident(mut id)), None) = + (iter.next(), iter.next()) + { + id.set_span(s); + return Ident::Compiler(id); } - _ => panic!(), - }; - Ident::Compiler(ident) + } + panic!("not allowed as a raw identifier: `{}`", raw_prefixed) } Span::Fallback(s) => Ident::Fallback(fallback::Ident::new_raw(string, s)), } diff --git a/tests/comments.rs b/tests/comments.rs index 7174108..4f7236d 100644 --- a/tests/comments.rs +++ b/tests/comments.rs @@ -1,3 +1,5 @@ +#![allow(clippy::assertions_on_result_states)] + use proc_macro2::{Delimiter, Literal, Spacing, TokenStream, TokenTree}; // #[doc = "..."] -> "..." diff --git a/tests/test.rs b/tests/test.rs index ab82390..8f5624d 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -1,4 +1,4 @@ -#![allow(clippy::non_ascii_literal)] +#![allow(clippy::assertions_on_result_states, clippy::non_ascii_literal)] use proc_macro2::{Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree}; use std::panic; @@ -15,14 +15,24 @@ fn idents() { } #[test] -#[cfg(procmacro2_semver_exempt)] fn raw_idents() { assert_eq!( Ident::new_raw("String", Span::call_site()).to_string(), "r#String" ); assert_eq!(Ident::new_raw("fn", Span::call_site()).to_string(), "r#fn"); - assert_eq!(Ident::new_raw("_", Span::call_site()).to_string(), "r#_"); +} + +#[test] +#[should_panic(expected = "`r#_` cannot be a raw identifier")] +fn ident_raw_underscore() { + Ident::new_raw("_", Span::call_site()); +} + +#[test] +#[should_panic(expected = "`r#super` cannot be a raw identifier")] +fn ident_raw_reserved() { + Ident::new_raw("super", Span::call_site()); } #[test] @@ -106,6 +116,15 @@ fn literal_raw_string() { "r\"\r\n\"".parse::().unwrap(); } +#[test] +fn literal_byte_string() { + assert_eq!(Literal::byte_string(b"").to_string(), "b\"\""); + assert_eq!( + Literal::byte_string(b"\0\t\n\r\"\\2\x10").to_string(), + "b\"\\0\\t\\n\\r\\\"\\\\2\\x10\"", + ); +} + #[test] fn literal_character() { assert_eq!(Literal::character('x').to_string(), "'x'"); @@ -113,9 +132,44 @@ fn literal_character() { assert_eq!(Literal::character('"').to_string(), "'\"'"); } +#[test] +fn literal_integer() { + assert_eq!(Literal::u8_suffixed(10).to_string(), "10u8"); + assert_eq!(Literal::u16_suffixed(10).to_string(), "10u16"); + assert_eq!(Literal::u32_suffixed(10).to_string(), "10u32"); + assert_eq!(Literal::u64_suffixed(10).to_string(), "10u64"); + assert_eq!(Literal::u128_suffixed(10).to_string(), "10u128"); + assert_eq!(Literal::usize_suffixed(10).to_string(), "10usize"); + + assert_eq!(Literal::i8_suffixed(10).to_string(), "10i8"); + assert_eq!(Literal::i16_suffixed(10).to_string(), "10i16"); + assert_eq!(Literal::i32_suffixed(10).to_string(), "10i32"); + assert_eq!(Literal::i64_suffixed(10).to_string(), "10i64"); + assert_eq!(Literal::i128_suffixed(10).to_string(), "10i128"); + assert_eq!(Literal::isize_suffixed(10).to_string(), "10isize"); + + assert_eq!(Literal::u8_unsuffixed(10).to_string(), "10"); + assert_eq!(Literal::u16_unsuffixed(10).to_string(), "10"); + assert_eq!(Literal::u32_unsuffixed(10).to_string(), "10"); + assert_eq!(Literal::u64_unsuffixed(10).to_string(), "10"); + assert_eq!(Literal::u128_unsuffixed(10).to_string(), "10"); + assert_eq!(Literal::usize_unsuffixed(10).to_string(), "10"); + + assert_eq!(Literal::i8_unsuffixed(10).to_string(), "10"); + assert_eq!(Literal::i16_unsuffixed(10).to_string(), "10"); + assert_eq!(Literal::i32_unsuffixed(10).to_string(), "10"); + assert_eq!(Literal::i64_unsuffixed(10).to_string(), "10"); + assert_eq!(Literal::i128_unsuffixed(10).to_string(), "10"); + assert_eq!(Literal::isize_unsuffixed(10).to_string(), "10"); +} + #[test] fn literal_float() { + assert_eq!(Literal::f32_suffixed(10.0).to_string(), "10f32"); + assert_eq!(Literal::f64_suffixed(10.0).to_string(), "10f64"); + assert_eq!(Literal::f32_unsuffixed(10.0).to_string(), "10.0"); + assert_eq!(Literal::f64_unsuffixed(10.0).to_string(), "10.0"); } #[test] @@ -493,6 +547,13 @@ fn default_tokenstream_is_empty() { assert!(default_token_stream.is_empty()); } +#[test] +fn tokenstream_size_hint() { + let tokens = "a b (c d) e".parse::().unwrap(); + + assert_eq!(tokens.into_iter().size_hint(), (4, Some(4))); +} + #[test] fn tuple_indexing() { // This behavior may change depending on https://github.com/rust-lang/rust/pull/71322 @@ -569,3 +630,16 @@ fn check_spans_internal(ts: TokenStream, lines: &mut &[(usize, usize, usize, usi } } } + +#[test] +fn byte_order_mark() { + let string = "\u{feff}foo"; + let tokens = string.parse::().unwrap(); + match tokens.into_iter().next().unwrap() { + TokenTree::Ident(ident) => assert_eq!(ident, "foo"), + _ => unreachable!(), + } + + let string = "foo\u{feff}"; + string.parse::().unwrap_err(); +} -- cgit v1.2.3