diff options
author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2024-05-10 15:41:57 +0000 |
---|---|---|
committer | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2024-05-10 15:41:57 +0000 |
commit | 21a4167aa3286775e678cc7134f12078b13cbfdc (patch) | |
tree | 844ee2157e5b3f316f75f240afd2005305603718 | |
parent | 766b5aa0719cb72bfd61deb2b818af0d13b19578 (diff) | |
parent | 21a62799aa52b66ce43dd836fbb2cbc84d1a8629 (diff) | |
download | proc-macro2-21a4167aa3286775e678cc7134f12078b13cbfdc.tar.gz |
Snap for 11819167 from 21a62799aa52b66ce43dd836fbb2cbc84d1a8629 to busytown-mac-infra-release
Change-Id: I519a929cc0f3ab158a775927c3dfd111d6b11c1f
-rw-r--r-- | .cargo_vcs_info.json | 2 | ||||
-rw-r--r-- | .clippy.toml | 1 | ||||
-rw-r--r-- | .github/workflows/ci.yml | 39 | ||||
-rw-r--r-- | Android.bp | 158 | ||||
-rw-r--r-- | BUILD | 59 | ||||
-rw-r--r-- | Cargo.toml | 10 | ||||
-rw-r--r-- | Cargo.toml.orig | 9 | ||||
-rw-r--r-- | METADATA | 10 | ||||
-rw-r--r-- | README.md | 4 | ||||
-rw-r--r-- | build.rs | 74 | ||||
-rw-r--r-- | cargo2android.json | 6 | ||||
-rw-r--r-- | cargo2rulesmk.json | 3 | ||||
-rw-r--r-- | cargo_embargo.json | 13 | ||||
-rw-r--r-- | rules.mk | 22 | ||||
-rw-r--r-- | rust-toolchain.toml | 2 | ||||
-rw-r--r-- | src/extra.rs | 84 | ||||
-rw-r--r-- | src/fallback.rs | 319 | ||||
-rw-r--r-- | src/lib.rs | 84 | ||||
-rw-r--r-- | src/marker.rs | 4 | ||||
-rw-r--r-- | src/parse.rs | 315 | ||||
-rw-r--r-- | src/rcvec.rs | 9 | ||||
-rw-r--r-- | src/wrapper.rs | 111 | ||||
-rw-r--r-- | tests/marker.rs | 3 | ||||
-rw-r--r-- | tests/test.rs | 134 | ||||
-rw-r--r-- | tests/test_fmt.rs | 2 | ||||
-rw-r--r-- | tests/test_size.rs | 42 |
26 files changed, 1045 insertions, 474 deletions
diff --git a/.cargo_vcs_info.json b/.cargo_vcs_info.json index 3b65b55..9eaf829 100644 --- a/.cargo_vcs_info.json +++ b/.cargo_vcs_info.json @@ -1,6 +1,6 @@ { "git": { - "sha1": "bc369f088f1b4067bc08848e8c6060b51a00e9df" + "sha1": "64b4608278be46fcc8d63ae1138da8cb600e258a" }, "path_in_vcs": "" }
\ No newline at end of file diff --git a/.clippy.toml b/.clippy.toml deleted file mode 100644 index 3d30690..0000000 --- a/.clippy.toml +++ /dev/null @@ -1 +0,0 @@ -msrv = "1.31.0" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d17e0c..296381c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,7 @@ name: CI on: push: pull_request: + workflow_dispatch: schedule: [cron: "40 1 * * *"] permissions: @@ -23,13 +24,14 @@ jobs: strategy: fail-fast: false matrix: - rust: [1.31.0, stable, beta] + rust: [1.56.0, stable, beta] timeout-minutes: 45 steps: - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@master with: toolchain: ${{matrix.rust}} + components: rust-src - run: cargo test - run: cargo test --no-default-features - run: cargo test --features span-locations @@ -51,9 +53,13 @@ jobs: steps: - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@nightly + with: + components: rust-src + - name: Enable type layout randomization + run: echo RUSTFLAGS=${RUSTFLAGS}\ -Zrandomize-layout >> $GITHUB_ENV - run: cargo test - run: cargo test --no-default-features - - run: cargo test --no-default-features -- --ignored # run the ignored test to make sure the `proc-macro` feature is disabled + - run: cargo test --no-default-features --test features -- --ignored make_sure_no_proc_macro # run the ignored test to make sure the `proc-macro` feature is disabled - run: cargo test --features span-locations - run: cargo test --manifest-path tests/ui/Cargo.toml - name: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test @@ -68,7 +74,18 @@ jobs: run: cargo test env: RUSTFLAGS: -Z allow-features= ${{env.RUSTFLAGS}} - - run: cargo update -Z minimal-versions && cargo build + + minimal: + name: Minimal versions + needs: pre_ci + if: needs.pre_ci.outputs.continue + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@nightly + - run: cargo generate-lockfile -Z minimal-versions + - run: cargo check --locked webassembly: name: WebAssembly @@ -81,6 +98,7 @@ jobs: - uses: dtolnay/rust-toolchain@nightly with: target: wasm32-unknown-unknown + components: rust-src - run: cargo test --target wasm32-unknown-unknown --no-run fuzz: @@ -92,8 +110,18 @@ jobs: steps: - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@nightly + with: + components: rust-src - uses: dtolnay/install@cargo-fuzz - run: cargo fuzz check + - run: cargo check --no-default-features --features afl + working-directory: fuzz + - uses: dtolnay/install@honggfuzz + - run: sudo apt-get install binutils-dev libunwind-dev + continue-on-error: true # https://github.com/dtolnay/proc-macro2/issues/387 + - run: cargo hfuzz build --no-default-features --features honggfuzz + working-directory: fuzz + continue-on-error: true # https://github.com/dtolnay/proc-macro2/issues/387 clippy: name: Clippy @@ -102,7 +130,9 @@ jobs: timeout-minutes: 45 steps: - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@clippy + - uses: dtolnay/rust-toolchain@nightly + with: + components: clippy, rust-src - run: cargo clippy --tests -- -Dclippy::all -Dclippy::pedantic - run: cargo clippy --tests --all-features -- -Dclippy::all -Dclippy::pedantic @@ -115,6 +145,7 @@ jobs: steps: - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@miri + - run: cargo miri setup - run: cargo miri test env: MIRIFLAGS: -Zmiri-strict-provenance @@ -1,4 +1,4 @@ -// This file is generated by cargo2android.py --config cargo2android.json. +// This file is generated by cargo_embargo. // Do not modify this file as changes will be overridden on upgrade. package { @@ -41,9 +41,9 @@ rust_library_host { name: "libproc_macro2", crate_name: "proc_macro2", cargo_env_compat: true, - cargo_pkg_version: "1.0.51", + cargo_pkg_version: "1.0.69", srcs: ["src/lib.rs"], - edition: "2018", + edition: "2021", features: [ "default", "proc-macro", @@ -52,23 +52,24 @@ rust_library_host { cfgs: [ "proc_macro_span", "span_locations", - "use_proc_macro", "wrap_proc_macro", ], - rustlibs: [ - "libunicode_ident", - ], + rustlibs: ["libunicode_ident"], compile_multilib: "first", } -rust_defaults { - name: "proc-macro2_test_defaults", - crate_name: "proc_macro2", +rust_test_host { + name: "proc-macro2_test_tests_comments", + crate_name: "comments", cargo_env_compat: true, - cargo_pkg_version: "1.0.51", + cargo_pkg_version: "1.0.69", + srcs: ["tests/comments.rs"], test_suites: ["general-tests"], auto_gen_config: true, - edition: "2018", + test_options: { + unit_test: true, + }, + edition: "2021", features: [ "default", "proc-macro", @@ -77,7 +78,6 @@ rust_defaults { cfgs: [ "proc_macro_span", "span_locations", - "use_proc_macro", "wrap_proc_macro", ], rustlibs: [ @@ -85,49 +85,155 @@ rust_defaults { "libquote", "libunicode_ident", ], -} - -rust_test_host { - name: "proc-macro2_test_tests_comments", - defaults: ["proc-macro2_test_defaults"], - srcs: ["tests/comments.rs"], - test_options: { - unit_test: true, - }, + proc_macros: ["librustversion"], } rust_test_host { name: "proc-macro2_test_tests_features", - defaults: ["proc-macro2_test_defaults"], + crate_name: "features", + cargo_env_compat: true, + cargo_pkg_version: "1.0.69", srcs: ["tests/features.rs"], + test_suites: ["general-tests"], + auto_gen_config: true, test_options: { unit_test: true, }, + edition: "2021", + features: [ + "default", + "proc-macro", + "span-locations", + ], + cfgs: [ + "proc_macro_span", + "span_locations", + "wrap_proc_macro", + ], + rustlibs: [ + "libproc_macro2", + "libquote", + "libunicode_ident", + ], + proc_macros: ["librustversion"], } rust_test_host { name: "proc-macro2_test_tests_marker", - defaults: ["proc-macro2_test_defaults"], + crate_name: "marker", + cargo_env_compat: true, + cargo_pkg_version: "1.0.69", srcs: ["tests/marker.rs"], + test_suites: ["general-tests"], + auto_gen_config: true, test_options: { unit_test: true, }, + edition: "2021", + features: [ + "default", + "proc-macro", + "span-locations", + ], + cfgs: [ + "proc_macro_span", + "span_locations", + "wrap_proc_macro", + ], + rustlibs: [ + "libproc_macro2", + "libquote", + "libunicode_ident", + ], + proc_macros: ["librustversion"], } rust_test_host { name: "proc-macro2_test_tests_test", - defaults: ["proc-macro2_test_defaults"], + crate_name: "test", + cargo_env_compat: true, + cargo_pkg_version: "1.0.69", srcs: ["tests/test.rs"], + test_suites: ["general-tests"], + auto_gen_config: true, test_options: { unit_test: true, }, + edition: "2021", + features: [ + "default", + "proc-macro", + "span-locations", + ], + cfgs: [ + "proc_macro_span", + "span_locations", + "wrap_proc_macro", + ], + rustlibs: [ + "libproc_macro2", + "libquote", + "libunicode_ident", + ], + proc_macros: ["librustversion"], } rust_test_host { name: "proc-macro2_test_tests_test_fmt", - defaults: ["proc-macro2_test_defaults"], + crate_name: "test_fmt", + cargo_env_compat: true, + cargo_pkg_version: "1.0.69", srcs: ["tests/test_fmt.rs"], + test_suites: ["general-tests"], + auto_gen_config: true, test_options: { unit_test: true, }, + edition: "2021", + features: [ + "default", + "proc-macro", + "span-locations", + ], + cfgs: [ + "proc_macro_span", + "span_locations", + "wrap_proc_macro", + ], + rustlibs: [ + "libproc_macro2", + "libquote", + "libunicode_ident", + ], + proc_macros: ["librustversion"], +} + +rust_test_host { + name: "proc-macro2_test_tests_test_size", + crate_name: "test_size", + cargo_env_compat: true, + cargo_pkg_version: "1.0.69", + srcs: ["tests/test_size.rs"], + test_suites: ["general-tests"], + auto_gen_config: true, + test_options: { + unit_test: true, + }, + edition: "2021", + features: [ + "default", + "proc-macro", + "span-locations", + ], + cfgs: [ + "proc_macro_span", + "span_locations", + "wrap_proc_macro", + ], + rustlibs: [ + "libproc_macro2", + "libquote", + "libunicode_ident", + ], + proc_macros: ["librustversion"], } @@ -0,0 +1,59 @@ +load("@rules_license//rules:license.bzl", "license") +load("@rules_license//rules:license_kind.bzl", "license_kind") +load("@rules_rust//cargo:defs.bzl", "cargo_build_script") +load("@rules_rust//rust:defs.bzl", "rust_library") + +package( + default_applicable_licenses = [":license"], + default_visibility = ["//visibility:public"], +) + +license( + name = "license", + license_kinds = [ + ":SPDX-license-identifier-Apache-2.0", + ":SPDX-license-identifier-MIT", + ], + license_text = "LICENSE-APACHE", + visibility = [":__subpackages__"], +) + +license_kind( + name = "SPDX-license-identifier-Apache-2.0", + conditions = ["notice"], + url = "https://spdx.org/licenses/Apache-2.0.html", +) + +license_kind( + name = "SPDX-license-identifier-MIT", + conditions = ["notice"], + url = "", +) + +CRATE_FEATURES = [ + "default", + "proc-macro", + "span-locations", +] + +rust_library( + name = "proc-macro2", + srcs = glob(["**/*.rs"]), + crate_features = CRATE_FEATURES, + edition = "2021", + deps = [ + ":proc-macro2_build_script", + # This should map to repo checked out from Android third party project + # "platform/external/rust/crates/unicode-ident". + "@unicode-ident", + ], +) + +cargo_build_script( + name = "proc-macro2_build_script", + srcs = glob(["**/*.rs"]), + crate_features = CRATE_FEATURES, + crate_root = "build.rs", + edition = "2021", + visibility = ["//visibility:private"], +) @@ -10,10 +10,10 @@ # See Cargo.toml.orig for the original contents. [package] -edition = "2018" -rust-version = "1.31" +edition = "2021" +rust-version = "1.56" name = "proc-macro2" -version = "1.0.51" +version = "1.0.69" authors = [ "David Tolnay <dtolnay@gmail.com>", "Alex Crichton <alex@alexcrichton.com>", @@ -40,6 +40,7 @@ rustdoc-args = [ "procmacro2_semver_exempt", "--cfg", "doc_cfg", + "--generate-link-to-definition", ] targets = ["x86_64-unknown-linux-gnu"] @@ -56,6 +57,9 @@ version = "1.0" version = "1.0" default_features = false +[dev-dependencies.rustversion] +version = "1" + [features] default = ["proc-macro"] nightly = [] diff --git a/Cargo.toml.orig b/Cargo.toml.orig index d653e6f..ca5f7d6 100644 --- a/Cargo.toml.orig +++ b/Cargo.toml.orig @@ -1,20 +1,20 @@ [package] name = "proc-macro2" -version = "1.0.51" # remember to update html_root_url +version = "1.0.69" # remember to update html_root_url authors = ["David Tolnay <dtolnay@gmail.com>", "Alex Crichton <alex@alexcrichton.com>"] autobenches = false categories = ["development-tools::procedural-macro-helpers"] description = "A substitute implementation of the compiler's `proc_macro` API to decouple token-based libraries from the procedural macro use case." documentation = "https://docs.rs/proc-macro2" -edition = "2018" +edition = "2021" keywords = ["macros", "syn"] license = "MIT OR Apache-2.0" repository = "https://github.com/dtolnay/proc-macro2" -rust-version = "1.31" +rust-version = "1.56" [package.metadata.docs.rs] rustc-args = ["--cfg", "procmacro2_semver_exempt"] -rustdoc-args = ["--cfg", "procmacro2_semver_exempt", "--cfg", "doc_cfg"] +rustdoc-args = ["--cfg", "procmacro2_semver_exempt", "--cfg", "doc_cfg", "--generate-link-to-definition"] targets = ["x86_64-unknown-linux-gnu"] [package.metadata.playground] @@ -25,6 +25,7 @@ unicode-ident = "1.0" [dev-dependencies] quote = { version = "1.0", default_features = false } +rustversion = "1" [features] proc-macro = [] @@ -1,6 +1,6 @@ # This project was upgraded with external_updater. # Usage: tools/external_updater/updater.sh update rust/crates/proc-macro2 -# For more info, check https://cs.android.com/android/platform/superproject/+/master:tools/external_updater/README.md +# For more info, check https://cs.android.com/android/platform/superproject/+/main:tools/external_updater/README.md name: "proc-macro2" description: "A substitute implementation of the compiler\'s `proc_macro` API to decouple token-based libraries from the procedural macro use case." @@ -11,13 +11,13 @@ third_party { } url { type: ARCHIVE - value: "https://static.crates.io/crates/proc-macro2/proc-macro2-1.0.51.crate" + value: "https://static.crates.io/crates/proc-macro2/proc-macro2-1.0.66.crate" } - version: "1.0.51" + version: "1.0.66" license_type: NOTICE last_upgrade_date { year: 2023 - month: 2 - day: 6 + month: 8 + day: 15 } } @@ -52,7 +52,7 @@ pub fn my_derive(input: proc_macro::TokenStream) -> proc_macro::TokenStream { If parsing with [Syn], you'll use [`parse_macro_input!`] instead to propagate parse errors correctly back to the compiler when parsing fails. -[`parse_macro_input!`]: https://docs.rs/syn/1.0/syn/macro.parse_macro_input.html +[`parse_macro_input!`]: https://docs.rs/syn/2.0/syn/macro.parse_macro_input.html ## Unstable features @@ -62,7 +62,7 @@ proc-macro2 by default. To opt into the additional APIs available in the most recent nightly compiler, the `procmacro2_semver_exempt` config flag must be passed to rustc. We will -polyfill those nightly-only APIs back to Rust 1.31.0. As these are unstable APIs +polyfill those nightly-only APIs back to Rust 1.56.0. As these are unstable APIs that track the nightly compiler, minor versions of proc-macro2 may make breaking changes to them at any time. @@ -1,11 +1,5 @@ // rustc-cfg emitted by the build script: // -// "use_proc_macro" -// Link to extern crate proc_macro. Available on any compiler and any target -// except wasm32. Requires "proc-macro" Cargo cfg to be enabled (default is -// enabled). On wasm32 we never link to proc_macro even if "proc-macro" cfg -// is enabled. -// // "wrap_proc_macro" // Wrap types from libproc_macro rather than polyfilling the whole API. // Enabled on rustc 1.29+ as long as procmacro2_semver_exempt is not set, @@ -41,21 +35,17 @@ // 1.57+. use std::env; -use std::process::{self, Command}; +use std::process::Command; use std::str; +use std::u32; fn main() { println!("cargo:rerun-if-changed=build.rs"); - let version = match rustc_version() { - Some(version) => version, - None => return, - }; - - if version.minor < 31 { - eprintln!("Minimum supported rustc version is 1.31"); - process::exit(1); - } + let version = rustc_version().unwrap_or(RustcVersion { + minor: u32::MAX, + nightly: false, + }); let docs_rs = env::var_os("DOCS_RS").is_some(); let semver_exempt = cfg!(procmacro2_semver_exempt) || docs_rs; @@ -68,53 +58,23 @@ fn main() { println!("cargo:rustc-cfg=span_locations"); } - if version.minor < 32 { - println!("cargo:rustc-cfg=no_libprocmacro_unwind_safe"); - } - - if version.minor < 39 { - println!("cargo:rustc-cfg=no_bind_by_move_pattern_guard"); - } - - if version.minor < 44 { - println!("cargo:rustc-cfg=no_lexerror_display"); - } - - if version.minor < 45 { - println!("cargo:rustc-cfg=no_hygiene"); - } - - if version.minor < 47 { - println!("cargo:rustc-cfg=no_ident_new_raw"); - } - - if version.minor < 54 { - println!("cargo:rustc-cfg=no_literal_from_str"); - } - - if version.minor < 55 { - println!("cargo:rustc-cfg=no_group_open_close"); - } - if version.minor < 57 { println!("cargo:rustc-cfg=no_is_available"); } - let target = env::var("TARGET").unwrap(); - if !enable_use_proc_macro(&target) { - return; + if version.minor < 66 { + println!("cargo:rustc-cfg=no_source_text"); } - println!("cargo:rustc-cfg=use_proc_macro"); + if !cfg!(feature = "proc-macro") { + return; + } if version.nightly || !semver_exempt { println!("cargo:rustc-cfg=wrap_proc_macro"); } - if version.nightly - && feature_allowed("proc_macro_span") - && feature_allowed("proc_macro_span_shrink") - { + if version.nightly && feature_allowed("proc_macro_span") { println!("cargo:rustc-cfg=proc_macro_span"); } @@ -123,16 +83,6 @@ fn main() { } } -fn enable_use_proc_macro(target: &str) -> bool { - // wasm targets don't have the `proc_macro` crate, disable this feature. - if target.contains("wasm32") { - return false; - } - - // Otherwise, only enable it if our feature is actually enabled. - cfg!(feature = "proc-macro") -} - struct RustcVersion { minor: u32, nightly: bool, diff --git a/cargo2android.json b/cargo2android.json deleted file mode 100644 index 45a2954..0000000 --- a/cargo2android.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "features": "default,span-locations", - "host-first-multilib": true, - "run": true, - "tests": true -}
\ No newline at end of file diff --git a/cargo2rulesmk.json b/cargo2rulesmk.json new file mode 100644 index 0000000..99d0b4b --- /dev/null +++ b/cargo2rulesmk.json @@ -0,0 +1,3 @@ +{ + "features": "default,span-locations" +}
\ No newline at end of file diff --git a/cargo_embargo.json b/cargo_embargo.json new file mode 100644 index 0000000..a64c9e4 --- /dev/null +++ b/cargo_embargo.json @@ -0,0 +1,13 @@ +{ + "features": [ + "default", + "span-locations" + ], + "package": { + "proc-macro2": { + "device_supported": false, + "host_first_multilib": true + } + }, + "tests": true +} diff --git a/rules.mk b/rules.mk new file mode 100644 index 0000000..39075a8 --- /dev/null +++ b/rules.mk @@ -0,0 +1,22 @@ +# This file is generated by cargo2rulesmk.py --run --config cargo2rulesmk.json. +# Do not modify this file as changes will be overridden on upgrade. + +LOCAL_DIR := $(GET_LOCAL_DIR) +MODULE := $(LOCAL_DIR) +MODULE_CRATE_NAME := proc_macro2 +MODULE_SRCS := \ + $(LOCAL_DIR)/src/lib.rs \ + +MODULE_RUST_EDITION := 2021 +MODULE_RUSTFLAGS += \ + --cfg 'feature="default"' \ + --cfg 'feature="proc-macro"' \ + --cfg 'feature="span-locations"' \ + --cfg 'proc_macro_span' \ + --cfg 'span_locations' \ + --cfg 'wrap_proc_macro' \ + +MODULE_LIBRARY_DEPS := \ + external/rust/crates/unicode-ident \ + +include make/library.mk diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..20fe888 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +components = ["rust-src"] diff --git a/src/extra.rs b/src/extra.rs new file mode 100644 index 0000000..4a69d46 --- /dev/null +++ b/src/extra.rs @@ -0,0 +1,84 @@ +//! Items which do not have a correspondence to any API in the proc_macro crate, +//! but are necessary to include in proc-macro2. + +use crate::fallback; +use crate::imp; +use crate::marker::Marker; +use crate::Span; +use core::fmt::{self, Debug}; + +/// An object that holds a [`Group`]'s `span_open()` and `span_close()` together +/// (in a more compact representation than holding those 2 spans individually. +/// +/// [`Group`]: crate::Group +#[derive(Copy, Clone)] +pub struct DelimSpan { + inner: DelimSpanEnum, + _marker: Marker, +} + +#[derive(Copy, Clone)] +enum DelimSpanEnum { + #[cfg(wrap_proc_macro)] + Compiler { + join: proc_macro::Span, + open: proc_macro::Span, + close: proc_macro::Span, + }, + Fallback(fallback::Span), +} + +impl DelimSpan { + pub(crate) fn new(group: &imp::Group) -> Self { + #[cfg(wrap_proc_macro)] + let inner = match group { + imp::Group::Compiler(group) => DelimSpanEnum::Compiler { + join: group.span(), + open: group.span_open(), + close: group.span_close(), + }, + imp::Group::Fallback(group) => DelimSpanEnum::Fallback(group.span()), + }; + + #[cfg(not(wrap_proc_macro))] + let inner = DelimSpanEnum::Fallback(group.span()); + + DelimSpan { + inner, + _marker: Marker, + } + } + + /// Returns a span covering the entire delimited group. + pub fn join(&self) -> Span { + match &self.inner { + #[cfg(wrap_proc_macro)] + DelimSpanEnum::Compiler { join, .. } => Span::_new(imp::Span::Compiler(*join)), + DelimSpanEnum::Fallback(span) => Span::_new_fallback(*span), + } + } + + /// Returns a span for the opening punctuation of the group only. + pub fn open(&self) -> Span { + match &self.inner { + #[cfg(wrap_proc_macro)] + DelimSpanEnum::Compiler { open, .. } => Span::_new(imp::Span::Compiler(*open)), + DelimSpanEnum::Fallback(span) => Span::_new_fallback(span.first_byte()), + } + } + + /// Returns a span for the closing punctuation of the group only. + pub fn close(&self) -> Span { + match &self.inner { + #[cfg(wrap_proc_macro)] + DelimSpanEnum::Compiler { close, .. } => Span::_new(imp::Span::Compiler(*close)), + DelimSpanEnum::Fallback(span) => Span::_new_fallback(span.last_byte()), + } + } +} + +impl Debug for DelimSpan { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.join(), f) + } +} diff --git a/src/fallback.rs b/src/fallback.rs index 587395d..7f559cf 100644 --- a/src/fallback.rs +++ b/src/fallback.rs @@ -3,18 +3,17 @@ use crate::location::LineColumn; use crate::parse::{self, Cursor}; use crate::rcvec::{RcVec, RcVecBuilder, RcVecIntoIter, RcVecMut}; use crate::{Delimiter, Spacing, TokenTree}; -#[cfg(span_locations)] +#[cfg(all(span_locations, not(fuzzing)))] +use alloc::collections::BTreeMap; +#[cfg(all(span_locations, not(fuzzing)))] use core::cell::RefCell; #[cfg(span_locations)] use core::cmp; use core::fmt::{self, Debug, Display, Write}; -use core::iter::FromIterator; use core::mem::ManuallyDrop; use core::ops::RangeBounds; use core::ptr; use core::str::FromStr; -#[cfg(procmacro2_semver_exempt)] -use std::path::Path; use std::path::PathBuf; /// Force use of proc-macro2's fallback implementation of the API for now, even @@ -73,7 +72,6 @@ impl TokenStream { fn push_token_from_proc_macro(mut vec: RcVecMut<TokenTree>, token: TokenTree) { // https://github.com/dtolnay/proc-macro2/issues/235 match token { - #[cfg(not(no_bind_by_move_pattern_guard))] TokenTree::Literal(crate::Literal { #[cfg(wrap_proc_macro)] inner: crate::imp::Literal::Fallback(literal), @@ -83,20 +81,6 @@ fn push_token_from_proc_macro(mut vec: RcVecMut<TokenTree>, token: TokenTree) { }) if literal.repr.starts_with('-') => { push_negative_literal(vec, literal); } - #[cfg(no_bind_by_move_pattern_guard)] - TokenTree::Literal(crate::Literal { - #[cfg(wrap_proc_macro)] - inner: crate::imp::Literal::Fallback(literal), - #[cfg(not(wrap_proc_macro))] - inner: literal, - .. - }) => { - if literal.repr.starts_with('-') { - push_negative_literal(vec, literal); - } else { - vec.push(TokenTree::Literal(crate::Literal::_new_stable(literal))); - } - } _ => vec.push(token), } @@ -104,9 +88,9 @@ fn push_token_from_proc_macro(mut vec: RcVecMut<TokenTree>, token: TokenTree) { fn push_negative_literal(mut vec: RcVecMut<TokenTree>, mut literal: Literal) { literal.repr.remove(0); let mut punct = crate::Punct::new('-', Spacing::Alone); - punct.set_span(crate::Span::_new_stable(literal.span)); + punct.set_span(crate::Span::_new_fallback(literal.span)); vec.push(TokenTree::Punct(punct)); - vec.push(TokenTree::Literal(crate::Literal::_new_stable(literal))); + vec.push(TokenTree::Literal(crate::Literal::_new_fallback(literal))); } } @@ -162,11 +146,14 @@ impl TokenStreamBuilder { #[cfg(span_locations)] fn get_cursor(src: &str) -> Cursor { + #[cfg(fuzzing)] + return Cursor { rest: src, off: 1 }; + // Create a dummy file & add it to the source map + #[cfg(not(fuzzing))] SOURCE_MAP.with(|cm| { let mut cm = cm.borrow_mut(); - let name = format!("<parsed string {}>", cm.files.len()); - let span = cm.add_file(&name, src); + let span = cm.add_file(src); Cursor { rest: src, off: span.lo, @@ -232,7 +219,7 @@ impl Debug for TokenStream { } } -#[cfg(use_proc_macro)] +#[cfg(feature = "proc-macro")] impl From<proc_macro::TokenStream> for TokenStream { fn from(inner: proc_macro::TokenStream) -> Self { inner @@ -242,7 +229,7 @@ impl From<proc_macro::TokenStream> for TokenStream { } } -#[cfg(use_proc_macro)] +#[cfg(feature = "proc-macro")] impl From<TokenStream> for proc_macro::TokenStream { fn from(inner: TokenStream) -> Self { inner @@ -320,7 +307,6 @@ impl SourceFile { } pub fn is_real(&self) -> bool { - // XXX(nika): Support real files in the future? false } } @@ -334,29 +320,29 @@ impl Debug for SourceFile { } } -#[cfg(span_locations)] +#[cfg(all(span_locations, not(fuzzing)))] thread_local! { static SOURCE_MAP: RefCell<SourceMap> = RefCell::new(SourceMap { - // NOTE: We start with a single dummy file which all call_site() and - // def_site() spans reference. + // Start with a single dummy file which all call_site() and def_site() + // spans reference. files: vec![FileInfo { - #[cfg(procmacro2_semver_exempt)] - name: "<unspecified>".to_owned(), + source_text: String::new(), span: Span { lo: 0, hi: 0 }, lines: vec![0], + char_index_to_byte_offset: BTreeMap::new(), }], }); } -#[cfg(span_locations)] +#[cfg(all(span_locations, not(fuzzing)))] struct FileInfo { - #[cfg(procmacro2_semver_exempt)] - name: String, + source_text: String, span: Span, lines: Vec<usize>, + char_index_to_byte_offset: BTreeMap<usize, usize>, } -#[cfg(span_locations)] +#[cfg(all(span_locations, not(fuzzing)))] impl FileInfo { fn offset_line_column(&self, offset: usize) -> LineColumn { assert!(self.span_within(Span { @@ -379,11 +365,47 @@ impl FileInfo { fn span_within(&self, span: Span) -> bool { span.lo >= self.span.lo && span.hi <= self.span.hi } + + fn source_text(&mut self, span: Span) -> String { + let lo_char = (span.lo - self.span.lo) as usize; + + // Look up offset of the largest already-computed char index that is + // less than or equal to the current requested one. We resume counting + // chars from that point. + let (&last_char_index, &last_byte_offset) = self + .char_index_to_byte_offset + .range(..=lo_char) + .next_back() + .unwrap_or((&0, &0)); + + let lo_byte = if last_char_index == lo_char { + last_byte_offset + } else { + let total_byte_offset = match self.source_text[last_byte_offset..] + .char_indices() + .nth(lo_char - last_char_index) + { + Some((additional_offset, _ch)) => last_byte_offset + additional_offset, + None => self.source_text.len(), + }; + self.char_index_to_byte_offset + .insert(lo_char, total_byte_offset); + total_byte_offset + }; + + let trunc_lo = &self.source_text[lo_byte..]; + let char_len = (span.hi - span.lo) as usize; + let source_text = match trunc_lo.char_indices().nth(char_len) { + Some((offset, _ch)) => &trunc_lo[..offset], + None => trunc_lo, + }; + source_text.to_owned() + } } /// Computes the offsets of each line in the given source string /// and the total number of characters -#[cfg(span_locations)] +#[cfg(all(span_locations, not(fuzzing)))] fn lines_offsets(s: &str) -> (usize, Vec<usize>) { let mut lines = vec![0]; let mut total = 0; @@ -398,12 +420,12 @@ fn lines_offsets(s: &str) -> (usize, Vec<usize>) { (total, lines) } -#[cfg(span_locations)] +#[cfg(all(span_locations, not(fuzzing)))] struct SourceMap { files: Vec<FileInfo>, } -#[cfg(span_locations)] +#[cfg(all(span_locations, not(fuzzing)))] impl SourceMap { fn next_start_pos(&self) -> u32 { // Add 1 so there's always space between files. @@ -413,35 +435,55 @@ impl SourceMap { self.files.last().unwrap().span.hi + 1 } - fn add_file(&mut self, name: &str, src: &str) -> Span { + fn add_file(&mut self, src: &str) -> Span { let (len, lines) = lines_offsets(src); let lo = self.next_start_pos(); - // XXX(nika): Should we bother doing a checked cast or checked add here? let span = Span { lo, hi: lo + (len as u32), }; self.files.push(FileInfo { - #[cfg(procmacro2_semver_exempt)] - name: name.to_owned(), + source_text: src.to_owned(), span, lines, + // Populated lazily by source_text(). + char_index_to_byte_offset: BTreeMap::new(), }); - #[cfg(not(procmacro2_semver_exempt))] - let _ = name; - span } + #[cfg(procmacro2_semver_exempt)] + fn filepath(&self, span: Span) -> PathBuf { + for (i, file) in self.files.iter().enumerate() { + if file.span_within(span) { + return PathBuf::from(if i == 0 { + "<unspecified>".to_owned() + } else { + format!("<parsed string {}>", i) + }); + } + } + unreachable!("Invalid span with no related FileInfo!"); + } + fn fileinfo(&self, span: Span) -> &FileInfo { for file in &self.files { if file.span_within(span) { return file; } } - panic!("Invalid span with no related FileInfo!"); + unreachable!("Invalid span with no related FileInfo!"); + } + + fn fileinfo_mut(&mut self, span: Span) -> &mut FileInfo { + for file in &mut self.files { + if file.span_within(span) { + return file; + } + } + unreachable!("Invalid span with no related FileInfo!"); } } @@ -464,7 +506,6 @@ impl Span { Span { lo: 0, hi: 0 } } - #[cfg(not(no_hygiene))] pub fn mixed_site() -> Self { Span::call_site() } @@ -487,17 +528,25 @@ impl Span { #[cfg(procmacro2_semver_exempt)] pub fn source_file(&self) -> SourceFile { + #[cfg(fuzzing)] + return SourceFile { + path: PathBuf::from("<unspecified>"), + }; + + #[cfg(not(fuzzing))] SOURCE_MAP.with(|cm| { let cm = cm.borrow(); - let fi = cm.fileinfo(*self); - SourceFile { - path: Path::new(&fi.name).to_owned(), - } + let path = cm.filepath(*self); + SourceFile { path } }) } #[cfg(span_locations)] pub fn start(&self) -> LineColumn { + #[cfg(fuzzing)] + return LineColumn { line: 0, column: 0 }; + + #[cfg(not(fuzzing))] SOURCE_MAP.with(|cm| { let cm = cm.borrow(); let fi = cm.fileinfo(*self); @@ -507,6 +556,10 @@ impl Span { #[cfg(span_locations)] pub fn end(&self) -> LineColumn { + #[cfg(fuzzing)] + return LineColumn { line: 0, column: 0 }; + + #[cfg(not(fuzzing))] SOURCE_MAP.with(|cm| { let cm = cm.borrow(); let fi = cm.fileinfo(*self); @@ -514,26 +567,6 @@ impl Span { }) } - #[cfg(procmacro2_semver_exempt)] - pub fn before(&self) -> Span { - Span { - #[cfg(span_locations)] - lo: self.lo, - #[cfg(span_locations)] - hi: self.lo, - } - } - - #[cfg(procmacro2_semver_exempt)] - pub fn after(&self) -> Span { - Span { - #[cfg(span_locations)] - lo: self.hi, - #[cfg(span_locations)] - hi: self.hi, - } - } - #[cfg(not(span_locations))] pub fn join(&self, _other: Span) -> Option<Span> { Some(Span {}) @@ -541,6 +574,13 @@ impl Span { #[cfg(span_locations)] pub fn join(&self, other: Span) -> Option<Span> { + #[cfg(fuzzing)] + return { + let _ = other; + None + }; + + #[cfg(not(fuzzing))] SOURCE_MAP.with(|cm| { let cm = cm.borrow(); // If `other` is not within the same FileInfo as us, return None. @@ -555,12 +595,32 @@ impl Span { } #[cfg(not(span_locations))] - fn first_byte(self) -> Self { + pub fn source_text(&self) -> Option<String> { + None + } + + #[cfg(span_locations)] + pub fn source_text(&self) -> Option<String> { + #[cfg(fuzzing)] + return None; + + #[cfg(not(fuzzing))] + { + if self.is_call_site() { + None + } else { + Some(SOURCE_MAP.with(|cm| cm.borrow_mut().fileinfo_mut(*self).source_text(*self))) + } + } + } + + #[cfg(not(span_locations))] + pub(crate) fn first_byte(self) -> Self { self } #[cfg(span_locations)] - fn first_byte(self) -> Self { + pub(crate) fn first_byte(self) -> Self { Span { lo: self.lo, hi: cmp::min(self.lo.saturating_add(1), self.hi), @@ -568,17 +628,22 @@ impl Span { } #[cfg(not(span_locations))] - fn last_byte(self) -> Self { + pub(crate) fn last_byte(self) -> Self { self } #[cfg(span_locations)] - fn last_byte(self) -> Self { + pub(crate) fn last_byte(self) -> Self { Span { lo: cmp::max(self.hi.saturating_sub(1), self.lo), hi: self.hi, } } + + #[cfg(span_locations)] + fn is_call_site(&self) -> bool { + self.lo == 0 && self.hi == 0 + } } impl Debug for Span { @@ -594,7 +659,7 @@ impl Debug for Span { pub(crate) fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) { #[cfg(span_locations)] { - if span.lo == 0 && span.hi == 0 { + if span.is_call_site() { return; } } @@ -730,7 +795,7 @@ fn validate_ident(string: &str, raw: bool) { panic!("Ident is not allowed to be empty; use Option<Ident>"); } - if string.bytes().all(|digit| digit >= b'0' && digit <= b'9') { + if string.bytes().all(|digit| b'0' <= digit && digit <= b'9') { panic!("Ident cannot be a number; use Literal instead"); } @@ -791,6 +856,7 @@ impl Display for Ident { } } +#[allow(clippy::missing_fields_in_debug)] impl Debug for Ident { // Ident(proc_macro), Ident(r#union) #[cfg(not(span_locations))] @@ -899,12 +965,25 @@ impl Literal { pub fn string(t: &str) -> Literal { let mut repr = String::with_capacity(t.len() + 2); repr.push('"'); - for c in t.chars() { - if c == '\'' { + let mut chars = t.chars(); + while let Some(ch) = chars.next() { + if ch == '\0' { + repr.push_str( + if chars + .as_str() + .starts_with(|next| '0' <= next && next <= '7') + { + // circumvent clippy::octal_escapes lint + "\\x00" + } else { + "\\0" + }, + ); + } else if ch == '\'' { // escape_debug turns this into "\'" which is unnecessary. - repr.push(c); + repr.push(ch); } else { - repr.extend(c.escape_debug()); + repr.extend(ch.escape_debug()); } } repr.push('"'); @@ -926,16 +1005,21 @@ impl Literal { pub fn byte_string(bytes: &[u8]) -> Literal { let mut escaped = "b\"".to_string(); - for b in bytes { + let mut bytes = bytes.iter(); + while let Some(&b) = bytes.next() { #[allow(clippy::match_overlapping_arm)] - match *b { - b'\0' => escaped.push_str(r"\0"), + match b { + b'\0' => escaped.push_str(match bytes.as_slice().first() { + // circumvent clippy::octal_escapes lint + Some(b'0'..=b'7') => r"\x00", + _ => r"\0", + }), b'\t' => escaped.push_str(r"\t"), b'\n' => escaped.push_str(r"\n"), b'\r' => escaped.push_str(r"\r"), b'"' => escaped.push_str("\\\""), b'\\' => escaped.push_str("\\\\"), - b'\x20'..=b'\x7E' => escaped.push(*b as char), + b'\x20'..=b'\x7E' => escaped.push(b as char), _ => { let _ = write!(escaped, "\\x{:02X}", b); } @@ -953,28 +1037,75 @@ impl Literal { self.span = span; } - pub fn subspan<R: RangeBounds<usize>>(&self, _range: R) -> Option<Span> { - None + pub fn subspan<R: RangeBounds<usize>>(&self, range: R) -> Option<Span> { + #[cfg(not(span_locations))] + { + let _ = range; + None + } + + #[cfg(span_locations)] + { + use core::ops::Bound; + + let lo = match range.start_bound() { + Bound::Included(start) => { + let start = u32::try_from(*start).ok()?; + self.span.lo.checked_add(start)? + } + Bound::Excluded(start) => { + let start = u32::try_from(*start).ok()?; + self.span.lo.checked_add(start)?.checked_add(1)? + } + Bound::Unbounded => self.span.lo, + }; + let hi = match range.end_bound() { + Bound::Included(end) => { + let end = u32::try_from(*end).ok()?; + self.span.lo.checked_add(end)?.checked_add(1)? + } + Bound::Excluded(end) => { + let end = u32::try_from(*end).ok()?; + self.span.lo.checked_add(end)? + } + Bound::Unbounded => self.span.hi, + }; + if lo <= hi && hi <= self.span.hi { + Some(Span { lo, hi }) + } else { + None + } + } } } impl FromStr for Literal { type Err = LexError; - fn from_str(mut repr: &str) -> Result<Self, Self::Err> { - let negative = repr.starts_with('-'); + fn from_str(repr: &str) -> Result<Self, Self::Err> { + let mut cursor = get_cursor(repr); + #[cfg(span_locations)] + let lo = cursor.off; + + let negative = cursor.starts_with_char('-'); if negative { - repr = &repr[1..]; - if !repr.starts_with(|ch: char| ch.is_ascii_digit()) { + cursor = cursor.advance(1); + if !cursor.starts_with_fn(|ch| ch.is_ascii_digit()) { return Err(LexError::call_site()); } } - let cursor = get_cursor(repr); - if let Ok((_rest, mut literal)) = parse::literal(cursor) { - if literal.repr.len() == repr.len() { + + if let Ok((rest, mut literal)) = parse::literal(cursor) { + if rest.is_empty() { if negative { literal.repr.insert(0, '-'); } + literal.span = Span { + #[cfg(span_locations)] + lo, + #[cfg(span_locations)] + hi: rest.off, + }; return Ok(literal); } } @@ -55,7 +55,7 @@ //! If parsing with [Syn], you'll use [`parse_macro_input!`] instead to //! propagate parse errors correctly back to the compiler when parsing fails. //! -//! [`parse_macro_input!`]: https://docs.rs/syn/1.0/syn/macro.parse_macro_input.html +//! [`parse_macro_input!`]: https://docs.rs/syn/2.0/syn/macro.parse_macro_input.html //! //! # Unstable features //! @@ -65,7 +65,7 @@ //! //! To opt into the additional APIs available in the most recent nightly //! compiler, the `procmacro2_semver_exempt` config flag must be passed to -//! rustc. We will polyfill those nightly-only APIs back to Rust 1.31.0. As +//! rustc. We will polyfill those nightly-only APIs back to Rust 1.56.0. As //! these are unstable APIs that track the nightly compiler, minor versions of //! proc-macro2 may make breaking changes to them at any time. //! @@ -86,11 +86,8 @@ //! a different thread. // Proc-macro2 types in rustdoc of other crates get linked to here. -#![doc(html_root_url = "https://docs.rs/proc-macro2/1.0.51")] -#![cfg_attr( - any(proc_macro_span, super_unstable), - feature(proc_macro_span, proc_macro_span_shrink) -)] +#![doc(html_root_url = "https://docs.rs/proc-macro2/1.0.69")] +#![cfg_attr(any(proc_macro_span, super_unstable), feature(proc_macro_span))] #![cfg_attr(super_unstable, feature(proc_macro_def_site))] #![cfg_attr(doc_cfg, feature(doc_cfg))] #![allow( @@ -98,9 +95,14 @@ clippy::cast_possible_truncation, clippy::doc_markdown, clippy::items_after_statements, + clippy::iter_without_into_iter, + clippy::let_underscore_untyped, clippy::manual_assert, + clippy::manual_range_contains, + clippy::missing_safety_doc, clippy::must_use_candidate, clippy::needless_doctest_main, + clippy::new_without_default, clippy::return_self_not_must_use, clippy::shadow_unrelated, clippy::trivially_copy_pass_by_ref, @@ -118,7 +120,9 @@ compile_error! {"\ build script as well. "} -#[cfg(use_proc_macro)] +extern crate alloc; + +#[cfg(feature = "proc-macro")] extern crate proc_macro; mod marker; @@ -133,6 +137,8 @@ mod detection; #[doc(hidden)] pub mod fallback; +pub mod extra; + #[cfg(not(wrap_proc_macro))] use crate::fallback as imp; #[path = "wrapper.rs"] @@ -142,11 +148,11 @@ mod imp; #[cfg(span_locations)] mod location; +use crate::extra::DelimSpan; use crate::marker::Marker; use core::cmp::Ordering; use core::fmt::{self, Debug, Display}; use core::hash::{Hash, Hasher}; -use core::iter::FromIterator; use core::ops::RangeBounds; use core::str::FromStr; use std::error::Error; @@ -183,7 +189,7 @@ impl TokenStream { } } - fn _new_stable(inner: fallback::TokenStream) -> Self { + fn _new_fallback(inner: fallback::TokenStream) -> Self { TokenStream { inner: inner.into(), _marker: Marker, @@ -229,14 +235,16 @@ impl FromStr for TokenStream { } } -#[cfg(use_proc_macro)] +#[cfg(feature = "proc-macro")] +#[cfg_attr(doc_cfg, doc(cfg(feature = "proc-macro")))] impl From<proc_macro::TokenStream> for TokenStream { fn from(inner: proc_macro::TokenStream) -> Self { TokenStream::_new(inner.into()) } } -#[cfg(use_proc_macro)] +#[cfg(feature = "proc-macro")] +#[cfg_attr(doc_cfg, doc(cfg(feature = "proc-macro")))] impl From<TokenStream> for proc_macro::TokenStream { fn from(inner: TokenStream) -> Self { inner.inner.into() @@ -377,7 +385,7 @@ impl Span { } } - fn _new_stable(inner: fallback::Span) -> Self { + fn _new_fallback(inner: fallback::Span) -> Self { Span { inner: inner.into(), _marker: Marker, @@ -396,9 +404,6 @@ impl Span { /// The span located at the invocation of the procedural macro, but with /// local variables, labels, and `$crate` resolved at the definition site /// of the macro. This is the same hygiene behavior as `macro_rules`. - /// - /// This function requires Rust 1.45 or later. - #[cfg(not(no_hygiene))] pub fn mixed_site() -> Self { Span::_new(imp::Span::mixed_site()) } @@ -485,24 +490,6 @@ impl Span { self.inner.end() } - /// Creates an empty span pointing to directly before this span. - /// - /// This method is semver exempt and not exposed by default. - #[cfg(all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)))] - #[cfg_attr(doc_cfg, doc(cfg(procmacro2_semver_exempt)))] - pub fn before(&self) -> Span { - Span::_new(self.inner.before()) - } - - /// Creates an empty span pointing to directly after this span. - /// - /// This method is semver exempt and not exposed by default. - #[cfg(all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)))] - #[cfg_attr(doc_cfg, doc(cfg(procmacro2_semver_exempt)))] - pub fn after(&self) -> Span { - Span::_new(self.inner.after()) - } - /// Create a new span encompassing `self` and `other`. /// /// Returns `None` if `self` and `other` are from different files. @@ -524,6 +511,17 @@ impl Span { pub fn eq(&self, other: &Span) -> bool { self.inner.eq(&other.inner) } + + /// Returns the source text behind a span. This preserves the original + /// source code, including spaces and comments. It only returns a result if + /// the span corresponds to real source code. + /// + /// Note: The observable result of a macro should only rely on the tokens + /// and not on this source text. The result of this function is a best + /// effort to be used for diagnostics only. + pub fn source_text(&self) -> Option<String> { + self.inner.source_text() + } } /// Prints a span in a form convenient for debugging. @@ -664,7 +662,7 @@ impl Group { Group { inner } } - fn _new_stable(inner: fallback::Group) -> Self { + fn _new_fallback(inner: fallback::Group) -> Self { Group { inner: inner.into(), } @@ -681,7 +679,8 @@ impl Group { } } - /// Returns the delimiter of this `Group` + /// Returns the punctuation used as the delimiter for this group: a set of + /// parentheses, square brackets, or curly braces. pub fn delimiter(&self) -> Delimiter { self.inner.delimiter() } @@ -725,6 +724,13 @@ impl Group { Span::_new(self.inner.span_close()) } + /// Returns an object that holds this group's `span_open()` and + /// `span_close()` together (in a more compact representation than holding + /// those 2 spans individually). + pub fn delim_span(&self) -> DelimSpan { + DelimSpan::new(&self.inner) + } + /// Configures the span for this `Group`'s delimiters, but not its internal /// tokens. /// @@ -848,7 +854,7 @@ impl Debug for Punct { /// Rust keywords. Use `input.call(Ident::parse_any)` when parsing to match the /// behaviour of `Ident::new`. /// -/// [`Parse`]: https://docs.rs/syn/1.0/syn/parse/trait.Parse.html +/// [`Parse`]: https://docs.rs/syn/2.0/syn/parse/trait.Parse.html /// /// # Examples /// @@ -939,7 +945,7 @@ impl Ident { /// Panics if the input string is neither a keyword nor a legal variable /// name. If you are not sure whether the string contains an identifier and /// need to handle an error case, use - /// <a href="https://docs.rs/syn/1.0/syn/fn.parse_str.html"><code + /// <a href="https://docs.rs/syn/2.0/syn/fn.parse_str.html"><code /// style="padding-right:0;">syn::parse_str</code></a><code /// style="padding-left:0;">::<Ident></code> /// rather than `Ident::new`. @@ -1081,7 +1087,7 @@ impl Literal { } } - fn _new_stable(inner: fallback::Literal) -> Self { + fn _new_fallback(inner: fallback::Literal) -> Self { Literal { inner: inner.into(), _marker: Marker, diff --git a/src/marker.rs b/src/marker.rs index 59fd096..e648dd2 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -1,6 +1,6 @@ +use alloc::rc::Rc; use core::marker::PhantomData; -use std::panic::{RefUnwindSafe, UnwindSafe}; -use std::rc::Rc; +use core::panic::{RefUnwindSafe, UnwindSafe}; // Zero sized marker with the correct set of autotrait impls we want all proc // macro types to have. diff --git a/src/parse.rs b/src/parse.rs index 2a87948..1430d73 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -27,7 +27,18 @@ impl<'a> Cursor<'a> { self.rest.starts_with(s) } - fn is_empty(&self) -> bool { + pub fn starts_with_char(&self, ch: char) -> bool { + self.rest.starts_with(ch) + } + + pub fn starts_with_fn<Pattern>(&self, f: Pattern) -> bool + where + Pattern: FnMut(char) -> bool, + { + self.rest.starts_with(f) + } + + pub fn is_empty(&self) -> bool { self.rest.is_empty() } @@ -97,7 +108,7 @@ fn skip_whitespace(input: Cursor) -> Cursor { s = s.advance(1); continue; } - b if b <= 0x7f => {} + b if b.is_ascii() => {} _ => { let ch = s.chars().next().unwrap(); if is_whitespace(ch) { @@ -150,6 +161,10 @@ fn word_break(input: Cursor) -> Result<Cursor, Reject> { } } +// Rustc's representation of a macro expansion error in expression position or +// type position. +const ERROR: &str = "(/*ERROR*/)"; + pub(crate) fn token_stream(mut input: Cursor) -> Result<TokenStream, LexError> { let mut trees = TokenStreamBuilder::new(); let mut stack = Vec::new(); @@ -181,7 +196,7 @@ pub(crate) fn token_stream(mut input: Cursor) -> Result<TokenStream, LexError> { }; if let Some(open_delimiter) = match first { - b'(' => Some(Delimiter::Parenthesis), + b'(' if !input.starts_with(ERROR) => Some(Delimiter::Parenthesis), b'[' => Some(Delimiter::Bracket), b'{' => Some(Delimiter::Brace), _ => None, @@ -217,13 +232,13 @@ pub(crate) fn token_stream(mut input: Cursor) -> Result<TokenStream, LexError> { hi: input.off, }); trees = outer; - trees.push_token_from_parser(TokenTree::Group(crate::Group::_new_stable(g))); + trees.push_token_from_parser(TokenTree::Group(crate::Group::_new_fallback(g))); } else { let (rest, mut tt) = match leaf_token(input) { Ok((rest, tt)) => (rest, tt), Err(Reject) => return Err(lex_error(input)), }; - tt.set_span(crate::Span::_new_stable(Span { + tt.set_span(crate::Span::_new_fallback(Span { #[cfg(span_locations)] lo, #[cfg(span_locations)] @@ -251,20 +266,26 @@ fn lex_error(cursor: Cursor) -> LexError { fn leaf_token(input: Cursor) -> PResult<TokenTree> { if let Ok((input, l)) = literal(input) { // must be parsed before ident - Ok((input, TokenTree::Literal(crate::Literal::_new_stable(l)))) + Ok((input, TokenTree::Literal(crate::Literal::_new_fallback(l)))) } else if let Ok((input, p)) = punct(input) { Ok((input, TokenTree::Punct(p))) } else if let Ok((input, i)) = ident(input) { Ok((input, TokenTree::Ident(i))) + } else if input.starts_with(ERROR) { + let rest = input.advance(ERROR.len()); + let repr = crate::Literal::_new_fallback(Literal::_new(ERROR.to_owned())); + Ok((rest, TokenTree::Literal(repr))) } else { Err(Reject) } } fn ident(input: Cursor) -> PResult<crate::Ident> { - if ["r\"", "r#\"", "r##", "b\"", "b\'", "br\"", "br#"] - .iter() - .any(|prefix| input.starts_with(prefix)) + if [ + "r\"", "r#\"", "r##", "b\"", "b\'", "br\"", "br#", "c\"", "cr\"", "cr#", + ] + .iter() + .any(|prefix| input.starts_with(prefix)) { Err(Reject) } else { @@ -322,6 +343,8 @@ fn literal_nocapture(input: Cursor) -> Result<Cursor, Reject> { Ok(ok) } else if let Ok(ok) = byte_string(input) { Ok(ok) + } else if let Ok(ok) = c_string(input) { + Ok(ok) } else if let Ok(ok) = byte(input) { Ok(ok) } else if let Ok(ok) = character(input) { @@ -352,8 +375,8 @@ fn string(input: Cursor) -> Result<Cursor, Reject> { } } -fn cooked_string(input: Cursor) -> Result<Cursor, Reject> { - let mut chars = input.char_indices().peekable(); +fn cooked_string(mut input: Cursor) -> Result<Cursor, Reject> { + let mut chars = input.char_indices(); while let Some((i, ch)) = chars.next() { match ch { @@ -367,31 +390,16 @@ fn cooked_string(input: Cursor) -> Result<Cursor, Reject> { }, '\\' => match chars.next() { Some((_, 'x')) => { - if !backslash_x_char(&mut chars) { - break; - } + backslash_x_char(&mut chars)?; } - Some((_, 'n')) | Some((_, 'r')) | Some((_, 't')) | Some((_, '\\')) - | Some((_, '\'')) | Some((_, '"')) | Some((_, '0')) => {} + Some((_, 'n' | 'r' | 't' | '\\' | '\'' | '"' | '0')) => {} Some((_, 'u')) => { - if !backslash_u(&mut chars) { - break; - } + backslash_u(&mut chars)?; } - Some((_, ch @ '\n')) | Some((_, ch @ '\r')) => { - let mut last = ch; - loop { - if last == '\r' && chars.next().map_or(true, |(_, ch)| ch != '\n') { - return Err(Reject); - } - match chars.peek() { - Some((_, ch)) if ch.is_whitespace() => { - last = *ch; - chars.next(); - } - _ => break, - } - } + Some((newline, ch @ ('\n' | '\r'))) => { + input = input.advance(newline + 1); + trailing_backslash(&mut input, ch as u8)?; + chars = input.char_indices(); } _ => break, }, @@ -401,11 +409,30 @@ fn cooked_string(input: Cursor) -> Result<Cursor, Reject> { Err(Reject) } +fn raw_string(input: Cursor) -> Result<Cursor, Reject> { + let (input, delimiter) = delimiter_of_raw_string(input)?; + let mut bytes = input.bytes().enumerate(); + while let Some((i, byte)) = bytes.next() { + match byte { + b'"' if input.rest[i + 1..].starts_with(delimiter) => { + let rest = input.advance(i + 1 + delimiter.len()); + return Ok(literal_suffix(rest)); + } + b'\r' => match bytes.next() { + Some((_, b'\n')) => {} + _ => break, + }, + _ => {} + } + } + Err(Reject) +} + fn byte_string(input: Cursor) -> Result<Cursor, Reject> { if let Ok(input) = input.parse("b\"") { cooked_byte_string(input) } else if let Ok(input) = input.parse("br") { - raw_string(input) + raw_byte_string(input) } else { Err(Reject) } @@ -425,68 +452,125 @@ fn cooked_byte_string(mut input: Cursor) -> Result<Cursor, Reject> { }, b'\\' => match bytes.next() { Some((_, b'x')) => { - if !backslash_x_byte(&mut bytes) { - break; - } + backslash_x_byte(&mut bytes)?; } - Some((_, b'n')) | Some((_, b'r')) | Some((_, b't')) | Some((_, b'\\')) - | Some((_, b'0')) | Some((_, b'\'')) | Some((_, b'"')) => {} - Some((newline, b @ b'\n')) | Some((newline, b @ b'\r')) => { - let mut last = b as char; - let rest = input.advance(newline + 1); - let mut chars = rest.char_indices(); - loop { - if last == '\r' && chars.next().map_or(true, |(_, ch)| ch != '\n') { - return Err(Reject); - } - match chars.next() { - Some((_, ch)) if ch.is_whitespace() => last = ch, - Some((offset, _)) => { - input = rest.advance(offset); - bytes = input.bytes().enumerate(); - break; - } - None => return Err(Reject), - } - } + Some((_, b'n' | b'r' | b't' | b'\\' | b'0' | b'\'' | b'"')) => {} + Some((newline, b @ (b'\n' | b'\r'))) => { + input = input.advance(newline + 1); + trailing_backslash(&mut input, b)?; + bytes = input.bytes().enumerate(); } _ => break, }, - b if b < 0x80 => {} + b if b.is_ascii() => {} _ => break, } } Err(Reject) } -fn raw_string(input: Cursor) -> Result<Cursor, Reject> { - let mut chars = input.char_indices(); - let mut n = 0; - for (i, ch) in &mut chars { - match ch { - '"' => { - n = i; - break; +fn delimiter_of_raw_string(input: Cursor) -> PResult<&str> { + for (i, byte) in input.bytes().enumerate() { + match byte { + b'"' => { + if i > 255 { + // https://github.com/rust-lang/rust/pull/95251 + return Err(Reject); + } + return Ok((input.advance(i + 1), &input.rest[..i])); } - '#' => {} - _ => return Err(Reject), + b'#' => {} + _ => break, } } - if n > 255 { - // https://github.com/rust-lang/rust/pull/95251 - return Err(Reject); + Err(Reject) +} + +fn raw_byte_string(input: Cursor) -> Result<Cursor, Reject> { + let (input, delimiter) = delimiter_of_raw_string(input)?; + let mut bytes = input.bytes().enumerate(); + while let Some((i, byte)) = bytes.next() { + match byte { + b'"' if input.rest[i + 1..].starts_with(delimiter) => { + let rest = input.advance(i + 1 + delimiter.len()); + return Ok(literal_suffix(rest)); + } + b'\r' => match bytes.next() { + Some((_, b'\n')) => {} + _ => break, + }, + other => { + if !other.is_ascii() { + break; + } + } + } } + Err(Reject) +} + +fn c_string(input: Cursor) -> Result<Cursor, Reject> { + if let Ok(input) = input.parse("c\"") { + cooked_c_string(input) + } else if let Ok(input) = input.parse("cr") { + raw_c_string(input) + } else { + Err(Reject) + } +} + +fn raw_c_string(input: Cursor) -> Result<Cursor, Reject> { + let (input, delimiter) = delimiter_of_raw_string(input)?; + let mut bytes = input.bytes().enumerate(); + while let Some((i, byte)) = bytes.next() { + match byte { + b'"' if input.rest[i + 1..].starts_with(delimiter) => { + let rest = input.advance(i + 1 + delimiter.len()); + return Ok(literal_suffix(rest)); + } + b'\r' => match bytes.next() { + Some((_, b'\n')) => {} + _ => break, + }, + b'\0' => break, + _ => {} + } + } + Err(Reject) +} + +fn cooked_c_string(mut input: Cursor) -> Result<Cursor, Reject> { + let mut chars = input.char_indices(); + while let Some((i, ch)) = chars.next() { match ch { - '"' if input.rest[i + 1..].starts_with(&input.rest[..n]) => { - let rest = input.advance(i + 1 + n); - return Ok(literal_suffix(rest)); + '"' => { + let input = input.advance(i + 1); + return Ok(literal_suffix(input)); } '\r' => match chars.next() { Some((_, '\n')) => {} _ => break, }, - _ => {} + '\\' => match chars.next() { + Some((_, 'x')) => { + backslash_x_nonzero(&mut chars)?; + } + Some((_, 'n' | 'r' | 't' | '\\' | '\'' | '"')) => {} + Some((_, 'u')) => { + if backslash_u(&mut chars)? == '\0' { + break; + } + } + Some((newline, ch @ ('\n' | '\r'))) => { + input = input.advance(newline + 1); + trailing_backslash(&mut input, ch as u8)?; + chars = input.char_indices(); + } + _ => break, + }, + '\0' => break, + _ch => {} } } Err(Reject) @@ -497,9 +581,8 @@ fn byte(input: Cursor) -> Result<Cursor, Reject> { let mut bytes = input.bytes().enumerate(); let ok = match bytes.next().map(|(_, b)| b) { Some(b'\\') => match bytes.next().map(|(_, b)| b) { - Some(b'x') => backslash_x_byte(&mut bytes), - Some(b'n') | Some(b'r') | Some(b't') | Some(b'\\') | Some(b'0') | Some(b'\'') - | Some(b'"') => true, + Some(b'x') => backslash_x_byte(&mut bytes).is_ok(), + Some(b'n' | b'r' | b't' | b'\\' | b'0' | b'\'' | b'"') => true, _ => false, }, b => b.is_some(), @@ -520,11 +603,9 @@ fn character(input: Cursor) -> Result<Cursor, Reject> { let mut chars = input.char_indices(); let ok = match chars.next().map(|(_, ch)| ch) { Some('\\') => match chars.next().map(|(_, ch)| ch) { - Some('x') => backslash_x_char(&mut chars), - Some('u') => backslash_u(&mut chars), - Some('n') | Some('r') | Some('t') | Some('\\') | Some('0') | Some('\'') | Some('"') => { - true - } + Some('x') => backslash_x_char(&mut chars).is_ok(), + Some('u') => backslash_u(&mut chars).is_ok(), + Some('n' | 'r' | 't' | '\\' | '0' | '\'' | '"') => true, _ => false, }, ch => ch.is_some(), @@ -538,36 +619,49 @@ fn character(input: Cursor) -> Result<Cursor, Reject> { } macro_rules! next_ch { - ($chars:ident @ $pat:pat $(| $rest:pat)*) => { + ($chars:ident @ $pat:pat) => { match $chars.next() { Some((_, ch)) => match ch { - $pat $(| $rest)* => ch, - _ => return false, + $pat => ch, + _ => return Err(Reject), }, - None => return false, + None => return Err(Reject), } }; } -fn backslash_x_char<I>(chars: &mut I) -> bool +fn backslash_x_char<I>(chars: &mut I) -> Result<(), Reject> where I: Iterator<Item = (usize, char)>, { next_ch!(chars @ '0'..='7'); next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F'); - true + Ok(()) } -fn backslash_x_byte<I>(chars: &mut I) -> bool +fn backslash_x_byte<I>(chars: &mut I) -> Result<(), Reject> where I: Iterator<Item = (usize, u8)>, { next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F'); next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F'); - true + Ok(()) } -fn backslash_u<I>(chars: &mut I) -> bool +fn backslash_x_nonzero<I>(chars: &mut I) -> Result<(), Reject> +where + I: Iterator<Item = (usize, char)>, +{ + let first = next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F'); + let second = next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F'); + if first == '0' && second == '0' { + Err(Reject) + } else { + Ok(()) + } +} + +fn backslash_u<I>(chars: &mut I) -> Result<char, Reject> where I: Iterator<Item = (usize, char)>, { @@ -580,17 +674,36 @@ where 'a'..='f' => 10 + ch as u8 - b'a', 'A'..='F' => 10 + ch as u8 - b'A', '_' if len > 0 => continue, - '}' if len > 0 => return char::from_u32(value).is_some(), - _ => return false, + '}' if len > 0 => return char::from_u32(value).ok_or(Reject), + _ => break, }; if len == 6 { - return false; + break; } value *= 0x10; value += u32::from(digit); len += 1; } - false + Err(Reject) +} + +fn trailing_backslash(input: &mut Cursor, mut last: u8) -> Result<(), Reject> { + let mut whitespace = input.bytes().enumerate(); + loop { + if last == b'\r' && whitespace.next().map_or(true, |(_, b)| b != b'\n') { + return Err(Reject); + } + match whitespace.next() { + Some((_, b @ (b' ' | b'\t' | b'\n' | b'\r'))) => { + last = b; + } + Some((offset, _)) => { + *input = input.advance(offset); + return Ok(()); + } + None => return Err(Reject), + } + } } fn float(input: Cursor) -> Result<Cursor, Reject> { @@ -606,7 +719,7 @@ fn float(input: Cursor) -> Result<Cursor, Reject> { fn float_digits(input: Cursor) -> Result<Cursor, Reject> { let mut chars = input.chars().peekable(); match chars.next() { - Some(ch) if ch >= '0' && ch <= '9' => {} + Some(ch) if '0' <= ch && ch <= '9' => {} _ => return Err(Reject), } @@ -756,7 +869,7 @@ fn digits(mut input: Cursor) -> Result<Cursor, Reject> { fn punct(input: Cursor) -> PResult<Punct> { let (rest, ch) = punct_char(input)?; if ch == '\'' { - if ident_any(rest)?.0.starts_with("'") { + if ident_any(rest)?.0.starts_with_char('\'') { Err(Reject) } else { Ok((rest, Punct::new('\'', Spacing::Joint))) @@ -795,7 +908,7 @@ fn doc_comment<'a>(input: Cursor<'a>, trees: &mut TokenStreamBuilder) -> PResult #[cfg(span_locations)] let lo = input.off; let (rest, (comment, inner)) = doc_comment_contents(input)?; - let span = crate::Span::_new_stable(Span { + let span = crate::Span::_new_fallback(Span { #[cfg(span_locations)] lo, #[cfg(span_locations)] @@ -831,7 +944,7 @@ fn doc_comment<'a>(input: Cursor<'a>, trees: &mut TokenStreamBuilder) -> PResult bracketed.push_token_from_parser(TokenTree::Punct(equal)); bracketed.push_token_from_parser(TokenTree::Literal(literal)); let group = Group::new(Delimiter::Bracket, bracketed.build()); - let mut group = crate::Group::_new_stable(group); + let mut group = crate::Group::_new_fallback(group); group.set_span(span); trees.push_token_from_parser(TokenTree::Group(group)); @@ -848,7 +961,7 @@ fn doc_comment_contents(input: Cursor) -> PResult<(&str, bool)> { Ok((input, (&s[3..s.len() - 2], true))) } else if input.starts_with("///") { let input = input.advance(3); - if input.starts_with("/") { + if input.starts_with_char('/') { return Err(Reject); } let (input, s) = take_until_newline_or_eof(input); diff --git a/src/rcvec.rs b/src/rcvec.rs index 86ca7d8..37955af 100644 --- a/src/rcvec.rs +++ b/src/rcvec.rs @@ -1,7 +1,8 @@ +use alloc::rc::Rc; +use alloc::vec; use core::mem; +use core::panic::RefUnwindSafe; use core::slice; -use std::rc::Rc; -use std::vec; pub(crate) struct RcVec<T> { inner: Rc<Vec<T>>, @@ -52,7 +53,7 @@ impl<T> RcVec<T> { T: Clone, { let vec = if let Some(owned) = Rc::get_mut(&mut self.inner) { - mem::replace(owned, Vec::new()) + mem::take(owned) } else { Vec::clone(&self.inner) }; @@ -140,3 +141,5 @@ impl<T> Iterator for RcVecIntoIter<T> { self.inner.size_hint() } } + +impl<T> RefUnwindSafe for RcVec<T> where T: RefUnwindSafe {} diff --git a/src/wrapper.rs b/src/wrapper.rs index f5ec06b..860b6b7 100644 --- a/src/wrapper.rs +++ b/src/wrapper.rs @@ -3,7 +3,6 @@ use crate::detection::inside_proc_macro; use crate::location::LineColumn; use crate::{fallback, Delimiter, Punct, Spacing, TokenTree}; use core::fmt::{self, Debug, Display}; -use core::iter::FromIterator; use core::ops::RangeBounds; use core::str::FromStr; use std::panic; @@ -40,7 +39,7 @@ impl LexError { } fn mismatch() -> ! { - panic!("stable/nightly mismatch") + panic!("compiler/fallback mismatch") } impl DeferredTokenStream { @@ -286,15 +285,7 @@ impl Debug for LexError { impl Display for LexError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { - #[cfg(not(no_lexerror_display))] LexError::Compiler(e) => Display::fmt(e, f), - #[cfg(no_lexerror_display)] - LexError::Compiler(_e) => Display::fmt( - &fallback::LexError { - span: fallback::Span::call_site(), - }, - f, - ), LexError::Fallback(e) => Display::fmt(e, f), } } @@ -406,7 +397,6 @@ impl Span { } } - #[cfg(not(no_hygiene))] pub fn mixed_site() -> Self { if inside_proc_macro() { Span::Compiler(proc_macro::Span::mixed_site()) @@ -426,13 +416,7 @@ impl Span { pub fn resolved_at(&self, other: Span) -> Span { match (self, other) { - #[cfg(not(no_hygiene))] (Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(a.resolved_at(b)), - - // Name resolution affects semantics, but location is only cosmetic - #[cfg(no_hygiene)] - (Span::Compiler(_), Span::Compiler(_)) => other, - (Span::Fallback(a), Span::Fallback(b)) => Span::Fallback(a.resolved_at(b)), _ => mismatch(), } @@ -440,13 +424,7 @@ impl Span { pub fn located_at(&self, other: Span) -> Span { match (self, other) { - #[cfg(not(no_hygiene))] (Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(a.located_at(b)), - - // Name resolution affects semantics, but location is only cosmetic - #[cfg(no_hygiene)] - (Span::Compiler(_), Span::Compiler(_)) => *self, - (Span::Fallback(a), Span::Fallback(b)) => Span::Fallback(a.located_at(b)), _ => mismatch(), } @@ -470,12 +448,6 @@ impl Span { #[cfg(span_locations)] pub fn start(&self) -> LineColumn { match self { - #[cfg(proc_macro_span)] - Span::Compiler(s) => { - let proc_macro::LineColumn { line, column } = s.start(); - LineColumn { line, column } - } - #[cfg(not(proc_macro_span))] Span::Compiler(_) => LineColumn { line: 0, column: 0 }, Span::Fallback(s) => s.start(), } @@ -484,33 +456,11 @@ impl Span { #[cfg(span_locations)] pub fn end(&self) -> LineColumn { match self { - #[cfg(proc_macro_span)] - Span::Compiler(s) => { - let proc_macro::LineColumn { line, column } = s.end(); - LineColumn { line, column } - } - #[cfg(not(proc_macro_span))] Span::Compiler(_) => LineColumn { line: 0, column: 0 }, Span::Fallback(s) => s.end(), } } - #[cfg(super_unstable)] - pub fn before(&self) -> Span { - match self { - Span::Compiler(s) => Span::Compiler(s.before()), - Span::Fallback(s) => Span::Fallback(s.before()), - } - } - - #[cfg(super_unstable)] - pub fn after(&self) -> Span { - match self { - Span::Compiler(s) => Span::Compiler(s.after()), - Span::Fallback(s) => Span::Fallback(s.after()), - } - } - pub fn join(&self, other: Span) -> Option<Span> { let ret = match (self, other) { #[cfg(proc_macro_span)] @@ -530,6 +480,16 @@ impl Span { } } + pub fn source_text(&self) -> Option<String> { + match self { + #[cfg(not(no_source_text))] + Span::Compiler(s) => s.source_text(), + #[cfg(no_source_text)] + Span::Compiler(_) => None, + Span::Fallback(s) => s.source_text(), + } + } + fn unwrap_nightly(self) -> proc_macro::Span { match self { Span::Compiler(s) => s, @@ -620,20 +580,14 @@ impl Group { pub fn span_open(&self) -> Span { match self { - #[cfg(not(no_group_open_close))] Group::Compiler(g) => Span::Compiler(g.span_open()), - #[cfg(no_group_open_close)] - Group::Compiler(g) => Span::Compiler(g.span()), Group::Fallback(g) => Span::Fallback(g.span_open()), } } pub fn span_close(&self) -> Span { match self { - #[cfg(not(no_group_open_close))] Group::Compiler(g) => Span::Compiler(g.span_close()), - #[cfg(no_group_open_close)] - Group::Compiler(g) => Span::Compiler(g.span()), Group::Fallback(g) => Span::Fallback(g.span_close()), } } @@ -694,27 +648,7 @@ impl Ident { pub fn new_raw(string: &str, span: Span) -> Self { match span { - #[cfg(not(no_ident_new_raw))] Span::Compiler(s) => Ident::Compiler(proc_macro::Ident::new_raw(string, s)), - #[cfg(no_ident_new_raw)] - Span::Compiler(s) => { - let _ = proc_macro::Ident::new(string, s); - // At this point the un-r#-prefixed string is known to be a - // valid identifier. Try to produce a valid raw identifier by - // running the `TokenStream` parser, and unwrapping the first - // token as an `Ident`. - let raw_prefixed = format!("r#{}", string); - if let Ok(ts) = raw_prefixed.parse::<proc_macro::TokenStream>() { - let mut iter = ts.into_iter(); - if let (Some(proc_macro::TokenTree::Ident(mut id)), None) = - (iter.next(), iter.next()) - { - id.set_span(s); - return Ident::Compiler(id); - } - } - panic!("not allowed as a raw identifier: `{}`", raw_prefixed) - } Span::Fallback(s) => Ident::Fallback(fallback::Ident::new_raw(string, s)), } } @@ -816,7 +750,7 @@ macro_rules! unsuffixed_integers { impl Literal { pub unsafe fn from_str_unchecked(repr: &str) -> Self { if inside_proc_macro() { - Literal::Compiler(compiler_literal_from_str(repr).expect("invalid literal")) + Literal::Compiler(proc_macro::Literal::from_str(repr).expect("invalid literal")) } else { Literal::Fallback(fallback::Literal::from_str_unchecked(repr)) } @@ -939,7 +873,8 @@ impl FromStr for Literal { fn from_str(repr: &str) -> Result<Self, Self::Err> { if inside_proc_macro() { - compiler_literal_from_str(repr).map(Literal::Compiler) + let literal = proc_macro::Literal::from_str(repr)?; + Ok(Literal::Compiler(literal)) } else { let literal = fallback::Literal::from_str(repr)?; Ok(Literal::Fallback(literal)) @@ -947,24 +882,6 @@ impl FromStr for Literal { } } -fn compiler_literal_from_str(repr: &str) -> Result<proc_macro::Literal, LexError> { - #[cfg(not(no_literal_from_str))] - { - proc_macro::Literal::from_str(repr).map_err(LexError::Compiler) - } - #[cfg(no_literal_from_str)] - { - let tokens = proc_macro_parse(repr)?; - let mut iter = tokens.into_iter(); - if let (Some(proc_macro::TokenTree::Literal(literal)), None) = (iter.next(), iter.next()) { - if literal.to_string().len() == repr.len() { - return Ok(literal); - } - } - Err(LexError::call_site()) - } -} - impl Display for Literal { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { diff --git a/tests/marker.rs b/tests/marker.rs index 4fb2beb..d08fbfc 100644 --- a/tests/marker.rs +++ b/tests/marker.rs @@ -1,3 +1,5 @@ +#![allow(clippy::extra_unused_type_parameters)] + use proc_macro2::{ Delimiter, Group, Ident, LexError, Literal, Punct, Spacing, Span, TokenStream, TokenTree, }; @@ -60,7 +62,6 @@ mod semver_exempt { assert_impl!(SourceFile is not Send or Sync); } -#[cfg(not(no_libprocmacro_unwind_safe))] mod unwind_safe { use proc_macro2::{ Delimiter, Group, Ident, LexError, Literal, Punct, Spacing, Span, TokenStream, TokenTree, diff --git a/tests/test.rs b/tests/test.rs index e0af151..b75cd55 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -1,12 +1,12 @@ #![allow( clippy::assertions_on_result_states, clippy::items_after_statements, - clippy::non_ascii_literal + clippy::non_ascii_literal, + clippy::octal_escapes )] use proc_macro2::{Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree}; use std::iter; -use std::panic; use std::str::{self, FromStr}; #[test] @@ -89,24 +89,9 @@ fn lifetime_number() { } #[test] +#[should_panic(expected = r#""'a#" is not a valid Ident"#)] fn lifetime_invalid() { - let result = panic::catch_unwind(|| Ident::new("'a#", Span::call_site())); - match result { - Err(box_any) => { - let message = box_any.downcast_ref::<String>().unwrap(); - let expected1 = r#""\'a#" is not a valid Ident"#; // 1.31.0 .. 1.53.0 - let expected2 = r#""'a#" is not a valid Ident"#; // 1.53.0 .. - assert!( - message == expected1 || message == expected2, - "panic message does not match expected string\n\ - \x20 panic message: `{:?}`\n\ - \x20expected message: `{:?}`", - message, - expected2, - ); - } - Ok(_) => panic!("test did not panic as expected"), - } + Ident::new("'a#", Span::call_site()); } #[test] @@ -114,6 +99,13 @@ fn literal_string() { assert_eq!(Literal::string("foo").to_string(), "\"foo\""); assert_eq!(Literal::string("\"").to_string(), "\"\\\"\""); assert_eq!(Literal::string("didn't").to_string(), "\"didn't\""); + assert_eq!( + Literal::string("a\00b\07c\08d\0e\0").to_string(), + "\"a\\x000b\\x007c\\08d\\0e\\0\"", + ); + + "\"\\\r\n x\"".parse::<TokenStream>().unwrap(); + "\"\\\r\n \rx\"".parse::<TokenStream>().unwrap_err(); } #[test] @@ -147,6 +139,51 @@ fn literal_byte_string() { Literal::byte_string(b"\0\t\n\r\"\\2\x10").to_string(), "b\"\\0\\t\\n\\r\\\"\\\\2\\x10\"", ); + assert_eq!( + Literal::byte_string(b"a\00b\07c\08d\0e\0").to_string(), + "b\"a\\x000b\\x007c\\08d\\0e\\0\"", + ); + + "b\"\\\r\n x\"".parse::<TokenStream>().unwrap(); + "b\"\\\r\n \rx\"".parse::<TokenStream>().unwrap_err(); + "b\"\\\r\n \u{a0}x\"".parse::<TokenStream>().unwrap_err(); + "br\"\u{a0}\"".parse::<TokenStream>().unwrap_err(); +} + +#[test] +fn literal_c_string() { + let strings = r###" + c"hello\x80我叫\u{1F980}" // from the RFC + cr"\" + cr##"Hello "world"!"## + c"\t\n\r\"\\" + "###; + + let mut tokens = strings.parse::<TokenStream>().unwrap().into_iter(); + + for expected in &[ + r#"c"hello\x80我叫\u{1F980}""#, + r#"cr"\""#, + r###"cr##"Hello "world"!"##"###, + r#"c"\t\n\r\"\\""#, + ] { + match tokens.next().unwrap() { + TokenTree::Literal(literal) => { + assert_eq!(literal.to_string(), *expected); + } + unexpected => panic!("unexpected token: {:?}", unexpected), + } + } + + if let Some(unexpected) = tokens.next() { + panic!("unexpected token: {:?}", unexpected); + } + + for invalid in &[r#"c"\0""#, r#"c"\x00""#, r#"c"\u{0}""#, "c\"\0\""] { + if let Ok(unexpected) = invalid.parse::<TokenStream>() { + panic!("unexpected token: {:?}", unexpected); + } + } } #[test] @@ -265,6 +302,48 @@ fn literal_parse() { } #[test] +fn literal_span() { + let positive = "0.1".parse::<Literal>().unwrap(); + let negative = "-0.1".parse::<Literal>().unwrap(); + let subspan = positive.subspan(1..2); + + #[cfg(not(span_locations))] + { + let _ = negative; + assert!(subspan.is_none()); + } + + #[cfg(span_locations)] + { + assert_eq!(positive.span().start().column, 0); + assert_eq!(positive.span().end().column, 3); + assert_eq!(negative.span().start().column, 0); + assert_eq!(negative.span().end().column, 4); + assert_eq!(subspan.unwrap().source_text().unwrap(), "."); + } + + assert!(positive.subspan(1..4).is_none()); +} + +#[cfg(span_locations)] +#[test] +fn source_text() { + let input = " 𓀕 a z "; + let mut tokens = input + .parse::<proc_macro2::TokenStream>() + .unwrap() + .into_iter(); + + let first = tokens.next().unwrap(); + assert_eq!("𓀕", first.span().source_text().unwrap()); + + let second = tokens.next().unwrap(); + let third = tokens.next().unwrap(); + assert_eq!("z", third.span().source_text().unwrap()); + assert_eq!("a", second.span().source_text().unwrap()); +} + +#[test] fn roundtrip() { fn roundtrip(p: &str) { println!("parse: {}", p); @@ -603,8 +682,8 @@ fn non_ascii_tokens() { check_spans("/*** ábc */ x", &[(1, 12, 1, 13)]); check_spans(r#""abc""#, &[(1, 0, 1, 5)]); check_spans(r#""ábc""#, &[(1, 0, 1, 5)]); - check_spans(r###"r#"abc"#"###, &[(1, 0, 1, 8)]); - check_spans(r###"r#"ábc"#"###, &[(1, 0, 1, 8)]); + check_spans(r##"r#"abc"#"##, &[(1, 0, 1, 8)]); + check_spans(r##"r#"ábc"#"##, &[(1, 0, 1, 8)]); check_spans("r#\"a\nc\"#", &[(1, 0, 2, 3)]); check_spans("r#\"á\nc\"#", &[(1, 0, 2, 3)]); check_spans("'a'", &[(1, 0, 1, 3)]); @@ -624,7 +703,6 @@ fn non_ascii_tokens() { check_spans("ábc// foo", &[(1, 0, 1, 3)]); check_spans("ábć// foo", &[(1, 0, 1, 3)]); check_spans("b\"a\\\n c\"", &[(1, 0, 2, 3)]); - check_spans("b\"a\\\n\u{00a0}c\"", &[(1, 0, 2, 3)]); } #[cfg(span_locations)] @@ -656,6 +734,18 @@ fn check_spans_internal(ts: TokenStream, lines: &mut &[(usize, usize, usize, usi } #[test] +fn whitespace() { + // space, horizontal tab, vertical tab, form feed, carriage return, line + // feed, non-breaking space, left-to-right mark, right-to-left mark + let various_spaces = " \t\u{b}\u{c}\r\n\u{a0}\u{200e}\u{200f}"; + let tokens = various_spaces.parse::<TokenStream>().unwrap(); + assert_eq!(tokens.into_iter().count(), 0); + + let lone_carriage_returns = " \r \r\r\n "; + lone_carriage_returns.parse::<TokenStream>().unwrap(); +} + +#[test] fn byte_order_mark() { let string = "\u{feff}foo"; let tokens = string.parse::<TokenStream>().unwrap(); diff --git a/tests/test_fmt.rs b/tests/test_fmt.rs index 93dd19e..86a4c38 100644 --- a/tests/test_fmt.rs +++ b/tests/test_fmt.rs @@ -1,7 +1,7 @@ #![allow(clippy::from_iter_instead_of_collect)] use proc_macro2::{Delimiter, Group, Ident, Span, TokenStream, TokenTree}; -use std::iter::{self, FromIterator}; +use std::iter; #[test] fn test_fmt_group() { diff --git a/tests/test_size.rs b/tests/test_size.rs new file mode 100644 index 0000000..46e58db --- /dev/null +++ b/tests/test_size.rs @@ -0,0 +1,42 @@ +extern crate proc_macro; + +use std::mem; + +#[rustversion::attr(before(1.32), ignore)] +#[test] +fn test_proc_macro_span_size() { + assert_eq!(mem::size_of::<proc_macro::Span>(), 4); + assert_eq!(mem::size_of::<Option<proc_macro::Span>>(), 4); +} + +#[cfg_attr(not(all(not(wrap_proc_macro), not(span_locations))), ignore)] +#[test] +fn test_proc_macro2_fallback_span_size_without_locations() { + assert_eq!(mem::size_of::<proc_macro2::Span>(), 0); + assert_eq!(mem::size_of::<Option<proc_macro2::Span>>(), 1); +} + +#[cfg_attr(not(all(not(wrap_proc_macro), span_locations)), ignore)] +#[test] +fn test_proc_macro2_fallback_span_size_with_locations() { + assert_eq!(mem::size_of::<proc_macro2::Span>(), 8); + assert_eq!(mem::size_of::<Option<proc_macro2::Span>>(), 12); +} + +#[rustversion::attr(before(1.32), ignore)] +#[rustversion::attr( + since(1.32), + cfg_attr(not(all(wrap_proc_macro, not(span_locations))), ignore) +)] +#[test] +fn test_proc_macro2_wrapper_span_size_without_locations() { + assert_eq!(mem::size_of::<proc_macro2::Span>(), 4); + assert_eq!(mem::size_of::<Option<proc_macro2::Span>>(), 8); +} + +#[cfg_attr(not(all(wrap_proc_macro, span_locations)), ignore)] +#[test] +fn test_proc_macro2_wrapper_span_size_with_locations() { + assert_eq!(mem::size_of::<proc_macro2::Span>(), 12); + assert_eq!(mem::size_of::<Option<proc_macro2::Span>>(), 12); +} |