aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAndroid Build Coastguard Worker <android-build-coastguard-worker@google.com>2024-05-10 15:41:57 +0000
committerAndroid Build Coastguard Worker <android-build-coastguard-worker@google.com>2024-05-10 15:41:57 +0000
commit21a4167aa3286775e678cc7134f12078b13cbfdc (patch)
tree844ee2157e5b3f316f75f240afd2005305603718
parent766b5aa0719cb72bfd61deb2b818af0d13b19578 (diff)
parent21a62799aa52b66ce43dd836fbb2cbc84d1a8629 (diff)
downloadproc-macro2-21a4167aa3286775e678cc7134f12078b13cbfdc.tar.gz
Snap for 11819167 from 21a62799aa52b66ce43dd836fbb2cbc84d1a8629 to busytown-mac-infra-release
Change-Id: I519a929cc0f3ab158a775927c3dfd111d6b11c1f
-rw-r--r--.cargo_vcs_info.json2
-rw-r--r--.clippy.toml1
-rw-r--r--.github/workflows/ci.yml39
-rw-r--r--Android.bp158
-rw-r--r--BUILD59
-rw-r--r--Cargo.toml10
-rw-r--r--Cargo.toml.orig9
-rw-r--r--METADATA10
-rw-r--r--README.md4
-rw-r--r--build.rs74
-rw-r--r--cargo2android.json6
-rw-r--r--cargo2rulesmk.json3
-rw-r--r--cargo_embargo.json13
-rw-r--r--rules.mk22
-rw-r--r--rust-toolchain.toml2
-rw-r--r--src/extra.rs84
-rw-r--r--src/fallback.rs319
-rw-r--r--src/lib.rs84
-rw-r--r--src/marker.rs4
-rw-r--r--src/parse.rs315
-rw-r--r--src/rcvec.rs9
-rw-r--r--src/wrapper.rs111
-rw-r--r--tests/marker.rs3
-rw-r--r--tests/test.rs134
-rw-r--r--tests/test_fmt.rs2
-rw-r--r--tests/test_size.rs42
26 files changed, 1045 insertions, 474 deletions
diff --git a/.cargo_vcs_info.json b/.cargo_vcs_info.json
index 3b65b55..9eaf829 100644
--- a/.cargo_vcs_info.json
+++ b/.cargo_vcs_info.json
@@ -1,6 +1,6 @@
{
"git": {
- "sha1": "bc369f088f1b4067bc08848e8c6060b51a00e9df"
+ "sha1": "64b4608278be46fcc8d63ae1138da8cb600e258a"
},
"path_in_vcs": ""
} \ No newline at end of file
diff --git a/.clippy.toml b/.clippy.toml
deleted file mode 100644
index 3d30690..0000000
--- a/.clippy.toml
+++ /dev/null
@@ -1 +0,0 @@
-msrv = "1.31.0"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9d17e0c..296381c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -3,6 +3,7 @@ name: CI
on:
push:
pull_request:
+ workflow_dispatch:
schedule: [cron: "40 1 * * *"]
permissions:
@@ -23,13 +24,14 @@ jobs:
strategy:
fail-fast: false
matrix:
- rust: [1.31.0, stable, beta]
+ rust: [1.56.0, stable, beta]
timeout-minutes: 45
steps:
- uses: actions/checkout@v3
- uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{matrix.rust}}
+ components: rust-src
- run: cargo test
- run: cargo test --no-default-features
- run: cargo test --features span-locations
@@ -51,9 +53,13 @@ jobs:
steps:
- uses: actions/checkout@v3
- uses: dtolnay/rust-toolchain@nightly
+ with:
+ components: rust-src
+ - name: Enable type layout randomization
+ run: echo RUSTFLAGS=${RUSTFLAGS}\ -Zrandomize-layout >> $GITHUB_ENV
- run: cargo test
- run: cargo test --no-default-features
- - run: cargo test --no-default-features -- --ignored # run the ignored test to make sure the `proc-macro` feature is disabled
+ - run: cargo test --no-default-features --test features -- --ignored make_sure_no_proc_macro # run the ignored test to make sure the `proc-macro` feature is disabled
- run: cargo test --features span-locations
- run: cargo test --manifest-path tests/ui/Cargo.toml
- name: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test
@@ -68,7 +74,18 @@ jobs:
run: cargo test
env:
RUSTFLAGS: -Z allow-features= ${{env.RUSTFLAGS}}
- - run: cargo update -Z minimal-versions && cargo build
+
+ minimal:
+ name: Minimal versions
+ needs: pre_ci
+ if: needs.pre_ci.outputs.continue
+ runs-on: ubuntu-latest
+ timeout-minutes: 45
+ steps:
+ - uses: actions/checkout@v3
+ - uses: dtolnay/rust-toolchain@nightly
+ - run: cargo generate-lockfile -Z minimal-versions
+ - run: cargo check --locked
webassembly:
name: WebAssembly
@@ -81,6 +98,7 @@ jobs:
- uses: dtolnay/rust-toolchain@nightly
with:
target: wasm32-unknown-unknown
+ components: rust-src
- run: cargo test --target wasm32-unknown-unknown --no-run
fuzz:
@@ -92,8 +110,18 @@ jobs:
steps:
- uses: actions/checkout@v3
- uses: dtolnay/rust-toolchain@nightly
+ with:
+ components: rust-src
- uses: dtolnay/install@cargo-fuzz
- run: cargo fuzz check
+ - run: cargo check --no-default-features --features afl
+ working-directory: fuzz
+ - uses: dtolnay/install@honggfuzz
+ - run: sudo apt-get install binutils-dev libunwind-dev
+ continue-on-error: true # https://github.com/dtolnay/proc-macro2/issues/387
+ - run: cargo hfuzz build --no-default-features --features honggfuzz
+ working-directory: fuzz
+ continue-on-error: true # https://github.com/dtolnay/proc-macro2/issues/387
clippy:
name: Clippy
@@ -102,7 +130,9 @@ jobs:
timeout-minutes: 45
steps:
- uses: actions/checkout@v3
- - uses: dtolnay/rust-toolchain@clippy
+ - uses: dtolnay/rust-toolchain@nightly
+ with:
+ components: clippy, rust-src
- run: cargo clippy --tests -- -Dclippy::all -Dclippy::pedantic
- run: cargo clippy --tests --all-features -- -Dclippy::all -Dclippy::pedantic
@@ -115,6 +145,7 @@ jobs:
steps:
- uses: actions/checkout@v3
- uses: dtolnay/rust-toolchain@miri
+ - run: cargo miri setup
- run: cargo miri test
env:
MIRIFLAGS: -Zmiri-strict-provenance
diff --git a/Android.bp b/Android.bp
index e792ecb..fb14fd3 100644
--- a/Android.bp
+++ b/Android.bp
@@ -1,4 +1,4 @@
-// This file is generated by cargo2android.py --config cargo2android.json.
+// This file is generated by cargo_embargo.
// Do not modify this file as changes will be overridden on upgrade.
package {
@@ -41,9 +41,9 @@ rust_library_host {
name: "libproc_macro2",
crate_name: "proc_macro2",
cargo_env_compat: true,
- cargo_pkg_version: "1.0.51",
+ cargo_pkg_version: "1.0.69",
srcs: ["src/lib.rs"],
- edition: "2018",
+ edition: "2021",
features: [
"default",
"proc-macro",
@@ -52,23 +52,24 @@ rust_library_host {
cfgs: [
"proc_macro_span",
"span_locations",
- "use_proc_macro",
"wrap_proc_macro",
],
- rustlibs: [
- "libunicode_ident",
- ],
+ rustlibs: ["libunicode_ident"],
compile_multilib: "first",
}
-rust_defaults {
- name: "proc-macro2_test_defaults",
- crate_name: "proc_macro2",
+rust_test_host {
+ name: "proc-macro2_test_tests_comments",
+ crate_name: "comments",
cargo_env_compat: true,
- cargo_pkg_version: "1.0.51",
+ cargo_pkg_version: "1.0.69",
+ srcs: ["tests/comments.rs"],
test_suites: ["general-tests"],
auto_gen_config: true,
- edition: "2018",
+ test_options: {
+ unit_test: true,
+ },
+ edition: "2021",
features: [
"default",
"proc-macro",
@@ -77,7 +78,6 @@ rust_defaults {
cfgs: [
"proc_macro_span",
"span_locations",
- "use_proc_macro",
"wrap_proc_macro",
],
rustlibs: [
@@ -85,49 +85,155 @@ rust_defaults {
"libquote",
"libunicode_ident",
],
-}
-
-rust_test_host {
- name: "proc-macro2_test_tests_comments",
- defaults: ["proc-macro2_test_defaults"],
- srcs: ["tests/comments.rs"],
- test_options: {
- unit_test: true,
- },
+ proc_macros: ["librustversion"],
}
rust_test_host {
name: "proc-macro2_test_tests_features",
- defaults: ["proc-macro2_test_defaults"],
+ crate_name: "features",
+ cargo_env_compat: true,
+ cargo_pkg_version: "1.0.69",
srcs: ["tests/features.rs"],
+ test_suites: ["general-tests"],
+ auto_gen_config: true,
test_options: {
unit_test: true,
},
+ edition: "2021",
+ features: [
+ "default",
+ "proc-macro",
+ "span-locations",
+ ],
+ cfgs: [
+ "proc_macro_span",
+ "span_locations",
+ "wrap_proc_macro",
+ ],
+ rustlibs: [
+ "libproc_macro2",
+ "libquote",
+ "libunicode_ident",
+ ],
+ proc_macros: ["librustversion"],
}
rust_test_host {
name: "proc-macro2_test_tests_marker",
- defaults: ["proc-macro2_test_defaults"],
+ crate_name: "marker",
+ cargo_env_compat: true,
+ cargo_pkg_version: "1.0.69",
srcs: ["tests/marker.rs"],
+ test_suites: ["general-tests"],
+ auto_gen_config: true,
test_options: {
unit_test: true,
},
+ edition: "2021",
+ features: [
+ "default",
+ "proc-macro",
+ "span-locations",
+ ],
+ cfgs: [
+ "proc_macro_span",
+ "span_locations",
+ "wrap_proc_macro",
+ ],
+ rustlibs: [
+ "libproc_macro2",
+ "libquote",
+ "libunicode_ident",
+ ],
+ proc_macros: ["librustversion"],
}
rust_test_host {
name: "proc-macro2_test_tests_test",
- defaults: ["proc-macro2_test_defaults"],
+ crate_name: "test",
+ cargo_env_compat: true,
+ cargo_pkg_version: "1.0.69",
srcs: ["tests/test.rs"],
+ test_suites: ["general-tests"],
+ auto_gen_config: true,
test_options: {
unit_test: true,
},
+ edition: "2021",
+ features: [
+ "default",
+ "proc-macro",
+ "span-locations",
+ ],
+ cfgs: [
+ "proc_macro_span",
+ "span_locations",
+ "wrap_proc_macro",
+ ],
+ rustlibs: [
+ "libproc_macro2",
+ "libquote",
+ "libunicode_ident",
+ ],
+ proc_macros: ["librustversion"],
}
rust_test_host {
name: "proc-macro2_test_tests_test_fmt",
- defaults: ["proc-macro2_test_defaults"],
+ crate_name: "test_fmt",
+ cargo_env_compat: true,
+ cargo_pkg_version: "1.0.69",
srcs: ["tests/test_fmt.rs"],
+ test_suites: ["general-tests"],
+ auto_gen_config: true,
test_options: {
unit_test: true,
},
+ edition: "2021",
+ features: [
+ "default",
+ "proc-macro",
+ "span-locations",
+ ],
+ cfgs: [
+ "proc_macro_span",
+ "span_locations",
+ "wrap_proc_macro",
+ ],
+ rustlibs: [
+ "libproc_macro2",
+ "libquote",
+ "libunicode_ident",
+ ],
+ proc_macros: ["librustversion"],
+}
+
+rust_test_host {
+ name: "proc-macro2_test_tests_test_size",
+ crate_name: "test_size",
+ cargo_env_compat: true,
+ cargo_pkg_version: "1.0.69",
+ srcs: ["tests/test_size.rs"],
+ test_suites: ["general-tests"],
+ auto_gen_config: true,
+ test_options: {
+ unit_test: true,
+ },
+ edition: "2021",
+ features: [
+ "default",
+ "proc-macro",
+ "span-locations",
+ ],
+ cfgs: [
+ "proc_macro_span",
+ "span_locations",
+ "wrap_proc_macro",
+ ],
+ rustlibs: [
+ "libproc_macro2",
+ "libquote",
+ "libunicode_ident",
+ ],
+ proc_macros: ["librustversion"],
}
diff --git a/BUILD b/BUILD
new file mode 100644
index 0000000..7d96008
--- /dev/null
+++ b/BUILD
@@ -0,0 +1,59 @@
+load("@rules_license//rules:license.bzl", "license")
+load("@rules_license//rules:license_kind.bzl", "license_kind")
+load("@rules_rust//cargo:defs.bzl", "cargo_build_script")
+load("@rules_rust//rust:defs.bzl", "rust_library")
+
+package(
+ default_applicable_licenses = [":license"],
+ default_visibility = ["//visibility:public"],
+)
+
+license(
+ name = "license",
+ license_kinds = [
+ ":SPDX-license-identifier-Apache-2.0",
+ ":SPDX-license-identifier-MIT",
+ ],
+ license_text = "LICENSE-APACHE",
+ visibility = [":__subpackages__"],
+)
+
+license_kind(
+ name = "SPDX-license-identifier-Apache-2.0",
+ conditions = ["notice"],
+ url = "https://spdx.org/licenses/Apache-2.0.html",
+)
+
+license_kind(
+ name = "SPDX-license-identifier-MIT",
+ conditions = ["notice"],
+ url = "",
+)
+
+CRATE_FEATURES = [
+ "default",
+ "proc-macro",
+ "span-locations",
+]
+
+rust_library(
+ name = "proc-macro2",
+ srcs = glob(["**/*.rs"]),
+ crate_features = CRATE_FEATURES,
+ edition = "2021",
+ deps = [
+ ":proc-macro2_build_script",
+ # This should map to repo checked out from Android third party project
+ # "platform/external/rust/crates/unicode-ident".
+ "@unicode-ident",
+ ],
+)
+
+cargo_build_script(
+ name = "proc-macro2_build_script",
+ srcs = glob(["**/*.rs"]),
+ crate_features = CRATE_FEATURES,
+ crate_root = "build.rs",
+ edition = "2021",
+ visibility = ["//visibility:private"],
+)
diff --git a/Cargo.toml b/Cargo.toml
index 092933e..6f4ed46 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,10 +10,10 @@
# See Cargo.toml.orig for the original contents.
[package]
-edition = "2018"
-rust-version = "1.31"
+edition = "2021"
+rust-version = "1.56"
name = "proc-macro2"
-version = "1.0.51"
+version = "1.0.69"
authors = [
"David Tolnay <dtolnay@gmail.com>",
"Alex Crichton <alex@alexcrichton.com>",
@@ -40,6 +40,7 @@ rustdoc-args = [
"procmacro2_semver_exempt",
"--cfg",
"doc_cfg",
+ "--generate-link-to-definition",
]
targets = ["x86_64-unknown-linux-gnu"]
@@ -56,6 +57,9 @@ version = "1.0"
version = "1.0"
default_features = false
+[dev-dependencies.rustversion]
+version = "1"
+
[features]
default = ["proc-macro"]
nightly = []
diff --git a/Cargo.toml.orig b/Cargo.toml.orig
index d653e6f..ca5f7d6 100644
--- a/Cargo.toml.orig
+++ b/Cargo.toml.orig
@@ -1,20 +1,20 @@
[package]
name = "proc-macro2"
-version = "1.0.51" # remember to update html_root_url
+version = "1.0.69" # remember to update html_root_url
authors = ["David Tolnay <dtolnay@gmail.com>", "Alex Crichton <alex@alexcrichton.com>"]
autobenches = false
categories = ["development-tools::procedural-macro-helpers"]
description = "A substitute implementation of the compiler's `proc_macro` API to decouple token-based libraries from the procedural macro use case."
documentation = "https://docs.rs/proc-macro2"
-edition = "2018"
+edition = "2021"
keywords = ["macros", "syn"]
license = "MIT OR Apache-2.0"
repository = "https://github.com/dtolnay/proc-macro2"
-rust-version = "1.31"
+rust-version = "1.56"
[package.metadata.docs.rs]
rustc-args = ["--cfg", "procmacro2_semver_exempt"]
-rustdoc-args = ["--cfg", "procmacro2_semver_exempt", "--cfg", "doc_cfg"]
+rustdoc-args = ["--cfg", "procmacro2_semver_exempt", "--cfg", "doc_cfg", "--generate-link-to-definition"]
targets = ["x86_64-unknown-linux-gnu"]
[package.metadata.playground]
@@ -25,6 +25,7 @@ unicode-ident = "1.0"
[dev-dependencies]
quote = { version = "1.0", default_features = false }
+rustversion = "1"
[features]
proc-macro = []
diff --git a/METADATA b/METADATA
index 5d76bb6..34f90ff 100644
--- a/METADATA
+++ b/METADATA
@@ -1,6 +1,6 @@
# This project was upgraded with external_updater.
# Usage: tools/external_updater/updater.sh update rust/crates/proc-macro2
-# For more info, check https://cs.android.com/android/platform/superproject/+/master:tools/external_updater/README.md
+# For more info, check https://cs.android.com/android/platform/superproject/+/main:tools/external_updater/README.md
name: "proc-macro2"
description: "A substitute implementation of the compiler\'s `proc_macro` API to decouple token-based libraries from the procedural macro use case."
@@ -11,13 +11,13 @@ third_party {
}
url {
type: ARCHIVE
- value: "https://static.crates.io/crates/proc-macro2/proc-macro2-1.0.51.crate"
+ value: "https://static.crates.io/crates/proc-macro2/proc-macro2-1.0.66.crate"
}
- version: "1.0.51"
+ version: "1.0.66"
license_type: NOTICE
last_upgrade_date {
year: 2023
- month: 2
- day: 6
+ month: 8
+ day: 15
}
}
diff --git a/README.md b/README.md
index 131ba51..3a29ce8 100644
--- a/README.md
+++ b/README.md
@@ -52,7 +52,7 @@ pub fn my_derive(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
If parsing with [Syn], you'll use [`parse_macro_input!`] instead to propagate
parse errors correctly back to the compiler when parsing fails.
-[`parse_macro_input!`]: https://docs.rs/syn/1.0/syn/macro.parse_macro_input.html
+[`parse_macro_input!`]: https://docs.rs/syn/2.0/syn/macro.parse_macro_input.html
## Unstable features
@@ -62,7 +62,7 @@ proc-macro2 by default.
To opt into the additional APIs available in the most recent nightly compiler,
the `procmacro2_semver_exempt` config flag must be passed to rustc. We will
-polyfill those nightly-only APIs back to Rust 1.31.0. As these are unstable APIs
+polyfill those nightly-only APIs back to Rust 1.56.0. As these are unstable APIs
that track the nightly compiler, minor versions of proc-macro2 may make breaking
changes to them at any time.
diff --git a/build.rs b/build.rs
index b69d813..9f0fb51 100644
--- a/build.rs
+++ b/build.rs
@@ -1,11 +1,5 @@
// rustc-cfg emitted by the build script:
//
-// "use_proc_macro"
-// Link to extern crate proc_macro. Available on any compiler and any target
-// except wasm32. Requires "proc-macro" Cargo cfg to be enabled (default is
-// enabled). On wasm32 we never link to proc_macro even if "proc-macro" cfg
-// is enabled.
-//
// "wrap_proc_macro"
// Wrap types from libproc_macro rather than polyfilling the whole API.
// Enabled on rustc 1.29+ as long as procmacro2_semver_exempt is not set,
@@ -41,21 +35,17 @@
// 1.57+.
use std::env;
-use std::process::{self, Command};
+use std::process::Command;
use std::str;
+use std::u32;
fn main() {
println!("cargo:rerun-if-changed=build.rs");
- let version = match rustc_version() {
- Some(version) => version,
- None => return,
- };
-
- if version.minor < 31 {
- eprintln!("Minimum supported rustc version is 1.31");
- process::exit(1);
- }
+ let version = rustc_version().unwrap_or(RustcVersion {
+ minor: u32::MAX,
+ nightly: false,
+ });
let docs_rs = env::var_os("DOCS_RS").is_some();
let semver_exempt = cfg!(procmacro2_semver_exempt) || docs_rs;
@@ -68,53 +58,23 @@ fn main() {
println!("cargo:rustc-cfg=span_locations");
}
- if version.minor < 32 {
- println!("cargo:rustc-cfg=no_libprocmacro_unwind_safe");
- }
-
- if version.minor < 39 {
- println!("cargo:rustc-cfg=no_bind_by_move_pattern_guard");
- }
-
- if version.minor < 44 {
- println!("cargo:rustc-cfg=no_lexerror_display");
- }
-
- if version.minor < 45 {
- println!("cargo:rustc-cfg=no_hygiene");
- }
-
- if version.minor < 47 {
- println!("cargo:rustc-cfg=no_ident_new_raw");
- }
-
- if version.minor < 54 {
- println!("cargo:rustc-cfg=no_literal_from_str");
- }
-
- if version.minor < 55 {
- println!("cargo:rustc-cfg=no_group_open_close");
- }
-
if version.minor < 57 {
println!("cargo:rustc-cfg=no_is_available");
}
- let target = env::var("TARGET").unwrap();
- if !enable_use_proc_macro(&target) {
- return;
+ if version.minor < 66 {
+ println!("cargo:rustc-cfg=no_source_text");
}
- println!("cargo:rustc-cfg=use_proc_macro");
+ if !cfg!(feature = "proc-macro") {
+ return;
+ }
if version.nightly || !semver_exempt {
println!("cargo:rustc-cfg=wrap_proc_macro");
}
- if version.nightly
- && feature_allowed("proc_macro_span")
- && feature_allowed("proc_macro_span_shrink")
- {
+ if version.nightly && feature_allowed("proc_macro_span") {
println!("cargo:rustc-cfg=proc_macro_span");
}
@@ -123,16 +83,6 @@ fn main() {
}
}
-fn enable_use_proc_macro(target: &str) -> bool {
- // wasm targets don't have the `proc_macro` crate, disable this feature.
- if target.contains("wasm32") {
- return false;
- }
-
- // Otherwise, only enable it if our feature is actually enabled.
- cfg!(feature = "proc-macro")
-}
-
struct RustcVersion {
minor: u32,
nightly: bool,
diff --git a/cargo2android.json b/cargo2android.json
deleted file mode 100644
index 45a2954..0000000
--- a/cargo2android.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "features": "default,span-locations",
- "host-first-multilib": true,
- "run": true,
- "tests": true
-} \ No newline at end of file
diff --git a/cargo2rulesmk.json b/cargo2rulesmk.json
new file mode 100644
index 0000000..99d0b4b
--- /dev/null
+++ b/cargo2rulesmk.json
@@ -0,0 +1,3 @@
+{
+ "features": "default,span-locations"
+} \ No newline at end of file
diff --git a/cargo_embargo.json b/cargo_embargo.json
new file mode 100644
index 0000000..a64c9e4
--- /dev/null
+++ b/cargo_embargo.json
@@ -0,0 +1,13 @@
+{
+ "features": [
+ "default",
+ "span-locations"
+ ],
+ "package": {
+ "proc-macro2": {
+ "device_supported": false,
+ "host_first_multilib": true
+ }
+ },
+ "tests": true
+}
diff --git a/rules.mk b/rules.mk
new file mode 100644
index 0000000..39075a8
--- /dev/null
+++ b/rules.mk
@@ -0,0 +1,22 @@
+# This file is generated by cargo2rulesmk.py --run --config cargo2rulesmk.json.
+# Do not modify this file as changes will be overridden on upgrade.
+
+LOCAL_DIR := $(GET_LOCAL_DIR)
+MODULE := $(LOCAL_DIR)
+MODULE_CRATE_NAME := proc_macro2
+MODULE_SRCS := \
+ $(LOCAL_DIR)/src/lib.rs \
+
+MODULE_RUST_EDITION := 2021
+MODULE_RUSTFLAGS += \
+ --cfg 'feature="default"' \
+ --cfg 'feature="proc-macro"' \
+ --cfg 'feature="span-locations"' \
+ --cfg 'proc_macro_span' \
+ --cfg 'span_locations' \
+ --cfg 'wrap_proc_macro' \
+
+MODULE_LIBRARY_DEPS := \
+ external/rust/crates/unicode-ident \
+
+include make/library.mk
diff --git a/rust-toolchain.toml b/rust-toolchain.toml
new file mode 100644
index 0000000..20fe888
--- /dev/null
+++ b/rust-toolchain.toml
@@ -0,0 +1,2 @@
+[toolchain]
+components = ["rust-src"]
diff --git a/src/extra.rs b/src/extra.rs
new file mode 100644
index 0000000..4a69d46
--- /dev/null
+++ b/src/extra.rs
@@ -0,0 +1,84 @@
+//! Items which do not have a correspondence to any API in the proc_macro crate,
+//! but are necessary to include in proc-macro2.
+
+use crate::fallback;
+use crate::imp;
+use crate::marker::Marker;
+use crate::Span;
+use core::fmt::{self, Debug};
+
+/// An object that holds a [`Group`]'s `span_open()` and `span_close()` together
+/// (in a more compact representation than holding those 2 spans individually.
+///
+/// [`Group`]: crate::Group
+#[derive(Copy, Clone)]
+pub struct DelimSpan {
+ inner: DelimSpanEnum,
+ _marker: Marker,
+}
+
+#[derive(Copy, Clone)]
+enum DelimSpanEnum {
+ #[cfg(wrap_proc_macro)]
+ Compiler {
+ join: proc_macro::Span,
+ open: proc_macro::Span,
+ close: proc_macro::Span,
+ },
+ Fallback(fallback::Span),
+}
+
+impl DelimSpan {
+ pub(crate) fn new(group: &imp::Group) -> Self {
+ #[cfg(wrap_proc_macro)]
+ let inner = match group {
+ imp::Group::Compiler(group) => DelimSpanEnum::Compiler {
+ join: group.span(),
+ open: group.span_open(),
+ close: group.span_close(),
+ },
+ imp::Group::Fallback(group) => DelimSpanEnum::Fallback(group.span()),
+ };
+
+ #[cfg(not(wrap_proc_macro))]
+ let inner = DelimSpanEnum::Fallback(group.span());
+
+ DelimSpan {
+ inner,
+ _marker: Marker,
+ }
+ }
+
+ /// Returns a span covering the entire delimited group.
+ pub fn join(&self) -> Span {
+ match &self.inner {
+ #[cfg(wrap_proc_macro)]
+ DelimSpanEnum::Compiler { join, .. } => Span::_new(imp::Span::Compiler(*join)),
+ DelimSpanEnum::Fallback(span) => Span::_new_fallback(*span),
+ }
+ }
+
+ /// Returns a span for the opening punctuation of the group only.
+ pub fn open(&self) -> Span {
+ match &self.inner {
+ #[cfg(wrap_proc_macro)]
+ DelimSpanEnum::Compiler { open, .. } => Span::_new(imp::Span::Compiler(*open)),
+ DelimSpanEnum::Fallback(span) => Span::_new_fallback(span.first_byte()),
+ }
+ }
+
+ /// Returns a span for the closing punctuation of the group only.
+ pub fn close(&self) -> Span {
+ match &self.inner {
+ #[cfg(wrap_proc_macro)]
+ DelimSpanEnum::Compiler { close, .. } => Span::_new(imp::Span::Compiler(*close)),
+ DelimSpanEnum::Fallback(span) => Span::_new_fallback(span.last_byte()),
+ }
+ }
+}
+
+impl Debug for DelimSpan {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ Debug::fmt(&self.join(), f)
+ }
+}
diff --git a/src/fallback.rs b/src/fallback.rs
index 587395d..7f559cf 100644
--- a/src/fallback.rs
+++ b/src/fallback.rs
@@ -3,18 +3,17 @@ use crate::location::LineColumn;
use crate::parse::{self, Cursor};
use crate::rcvec::{RcVec, RcVecBuilder, RcVecIntoIter, RcVecMut};
use crate::{Delimiter, Spacing, TokenTree};
-#[cfg(span_locations)]
+#[cfg(all(span_locations, not(fuzzing)))]
+use alloc::collections::BTreeMap;
+#[cfg(all(span_locations, not(fuzzing)))]
use core::cell::RefCell;
#[cfg(span_locations)]
use core::cmp;
use core::fmt::{self, Debug, Display, Write};
-use core::iter::FromIterator;
use core::mem::ManuallyDrop;
use core::ops::RangeBounds;
use core::ptr;
use core::str::FromStr;
-#[cfg(procmacro2_semver_exempt)]
-use std::path::Path;
use std::path::PathBuf;
/// Force use of proc-macro2's fallback implementation of the API for now, even
@@ -73,7 +72,6 @@ impl TokenStream {
fn push_token_from_proc_macro(mut vec: RcVecMut<TokenTree>, token: TokenTree) {
// https://github.com/dtolnay/proc-macro2/issues/235
match token {
- #[cfg(not(no_bind_by_move_pattern_guard))]
TokenTree::Literal(crate::Literal {
#[cfg(wrap_proc_macro)]
inner: crate::imp::Literal::Fallback(literal),
@@ -83,20 +81,6 @@ fn push_token_from_proc_macro(mut vec: RcVecMut<TokenTree>, token: TokenTree) {
}) if literal.repr.starts_with('-') => {
push_negative_literal(vec, literal);
}
- #[cfg(no_bind_by_move_pattern_guard)]
- TokenTree::Literal(crate::Literal {
- #[cfg(wrap_proc_macro)]
- inner: crate::imp::Literal::Fallback(literal),
- #[cfg(not(wrap_proc_macro))]
- inner: literal,
- ..
- }) => {
- if literal.repr.starts_with('-') {
- push_negative_literal(vec, literal);
- } else {
- vec.push(TokenTree::Literal(crate::Literal::_new_stable(literal)));
- }
- }
_ => vec.push(token),
}
@@ -104,9 +88,9 @@ fn push_token_from_proc_macro(mut vec: RcVecMut<TokenTree>, token: TokenTree) {
fn push_negative_literal(mut vec: RcVecMut<TokenTree>, mut literal: Literal) {
literal.repr.remove(0);
let mut punct = crate::Punct::new('-', Spacing::Alone);
- punct.set_span(crate::Span::_new_stable(literal.span));
+ punct.set_span(crate::Span::_new_fallback(literal.span));
vec.push(TokenTree::Punct(punct));
- vec.push(TokenTree::Literal(crate::Literal::_new_stable(literal)));
+ vec.push(TokenTree::Literal(crate::Literal::_new_fallback(literal)));
}
}
@@ -162,11 +146,14 @@ impl TokenStreamBuilder {
#[cfg(span_locations)]
fn get_cursor(src: &str) -> Cursor {
+ #[cfg(fuzzing)]
+ return Cursor { rest: src, off: 1 };
+
// Create a dummy file & add it to the source map
+ #[cfg(not(fuzzing))]
SOURCE_MAP.with(|cm| {
let mut cm = cm.borrow_mut();
- let name = format!("<parsed string {}>", cm.files.len());
- let span = cm.add_file(&name, src);
+ let span = cm.add_file(src);
Cursor {
rest: src,
off: span.lo,
@@ -232,7 +219,7 @@ impl Debug for TokenStream {
}
}
-#[cfg(use_proc_macro)]
+#[cfg(feature = "proc-macro")]
impl From<proc_macro::TokenStream> for TokenStream {
fn from(inner: proc_macro::TokenStream) -> Self {
inner
@@ -242,7 +229,7 @@ impl From<proc_macro::TokenStream> for TokenStream {
}
}
-#[cfg(use_proc_macro)]
+#[cfg(feature = "proc-macro")]
impl From<TokenStream> for proc_macro::TokenStream {
fn from(inner: TokenStream) -> Self {
inner
@@ -320,7 +307,6 @@ impl SourceFile {
}
pub fn is_real(&self) -> bool {
- // XXX(nika): Support real files in the future?
false
}
}
@@ -334,29 +320,29 @@ impl Debug for SourceFile {
}
}
-#[cfg(span_locations)]
+#[cfg(all(span_locations, not(fuzzing)))]
thread_local! {
static SOURCE_MAP: RefCell<SourceMap> = RefCell::new(SourceMap {
- // NOTE: We start with a single dummy file which all call_site() and
- // def_site() spans reference.
+ // Start with a single dummy file which all call_site() and def_site()
+ // spans reference.
files: vec![FileInfo {
- #[cfg(procmacro2_semver_exempt)]
- name: "<unspecified>".to_owned(),
+ source_text: String::new(),
span: Span { lo: 0, hi: 0 },
lines: vec![0],
+ char_index_to_byte_offset: BTreeMap::new(),
}],
});
}
-#[cfg(span_locations)]
+#[cfg(all(span_locations, not(fuzzing)))]
struct FileInfo {
- #[cfg(procmacro2_semver_exempt)]
- name: String,
+ source_text: String,
span: Span,
lines: Vec<usize>,
+ char_index_to_byte_offset: BTreeMap<usize, usize>,
}
-#[cfg(span_locations)]
+#[cfg(all(span_locations, not(fuzzing)))]
impl FileInfo {
fn offset_line_column(&self, offset: usize) -> LineColumn {
assert!(self.span_within(Span {
@@ -379,11 +365,47 @@ impl FileInfo {
fn span_within(&self, span: Span) -> bool {
span.lo >= self.span.lo && span.hi <= self.span.hi
}
+
+ fn source_text(&mut self, span: Span) -> String {
+ let lo_char = (span.lo - self.span.lo) as usize;
+
+ // Look up offset of the largest already-computed char index that is
+ // less than or equal to the current requested one. We resume counting
+ // chars from that point.
+ let (&last_char_index, &last_byte_offset) = self
+ .char_index_to_byte_offset
+ .range(..=lo_char)
+ .next_back()
+ .unwrap_or((&0, &0));
+
+ let lo_byte = if last_char_index == lo_char {
+ last_byte_offset
+ } else {
+ let total_byte_offset = match self.source_text[last_byte_offset..]
+ .char_indices()
+ .nth(lo_char - last_char_index)
+ {
+ Some((additional_offset, _ch)) => last_byte_offset + additional_offset,
+ None => self.source_text.len(),
+ };
+ self.char_index_to_byte_offset
+ .insert(lo_char, total_byte_offset);
+ total_byte_offset
+ };
+
+ let trunc_lo = &self.source_text[lo_byte..];
+ let char_len = (span.hi - span.lo) as usize;
+ let source_text = match trunc_lo.char_indices().nth(char_len) {
+ Some((offset, _ch)) => &trunc_lo[..offset],
+ None => trunc_lo,
+ };
+ source_text.to_owned()
+ }
}
/// Computes the offsets of each line in the given source string
/// and the total number of characters
-#[cfg(span_locations)]
+#[cfg(all(span_locations, not(fuzzing)))]
fn lines_offsets(s: &str) -> (usize, Vec<usize>) {
let mut lines = vec![0];
let mut total = 0;
@@ -398,12 +420,12 @@ fn lines_offsets(s: &str) -> (usize, Vec<usize>) {
(total, lines)
}
-#[cfg(span_locations)]
+#[cfg(all(span_locations, not(fuzzing)))]
struct SourceMap {
files: Vec<FileInfo>,
}
-#[cfg(span_locations)]
+#[cfg(all(span_locations, not(fuzzing)))]
impl SourceMap {
fn next_start_pos(&self) -> u32 {
// Add 1 so there's always space between files.
@@ -413,35 +435,55 @@ impl SourceMap {
self.files.last().unwrap().span.hi + 1
}
- fn add_file(&mut self, name: &str, src: &str) -> Span {
+ fn add_file(&mut self, src: &str) -> Span {
let (len, lines) = lines_offsets(src);
let lo = self.next_start_pos();
- // XXX(nika): Should we bother doing a checked cast or checked add here?
let span = Span {
lo,
hi: lo + (len as u32),
};
self.files.push(FileInfo {
- #[cfg(procmacro2_semver_exempt)]
- name: name.to_owned(),
+ source_text: src.to_owned(),
span,
lines,
+ // Populated lazily by source_text().
+ char_index_to_byte_offset: BTreeMap::new(),
});
- #[cfg(not(procmacro2_semver_exempt))]
- let _ = name;
-
span
}
+ #[cfg(procmacro2_semver_exempt)]
+ fn filepath(&self, span: Span) -> PathBuf {
+ for (i, file) in self.files.iter().enumerate() {
+ if file.span_within(span) {
+ return PathBuf::from(if i == 0 {
+ "<unspecified>".to_owned()
+ } else {
+ format!("<parsed string {}>", i)
+ });
+ }
+ }
+ unreachable!("Invalid span with no related FileInfo!");
+ }
+
fn fileinfo(&self, span: Span) -> &FileInfo {
for file in &self.files {
if file.span_within(span) {
return file;
}
}
- panic!("Invalid span with no related FileInfo!");
+ unreachable!("Invalid span with no related FileInfo!");
+ }
+
+ fn fileinfo_mut(&mut self, span: Span) -> &mut FileInfo {
+ for file in &mut self.files {
+ if file.span_within(span) {
+ return file;
+ }
+ }
+ unreachable!("Invalid span with no related FileInfo!");
}
}
@@ -464,7 +506,6 @@ impl Span {
Span { lo: 0, hi: 0 }
}
- #[cfg(not(no_hygiene))]
pub fn mixed_site() -> Self {
Span::call_site()
}
@@ -487,17 +528,25 @@ impl Span {
#[cfg(procmacro2_semver_exempt)]
pub fn source_file(&self) -> SourceFile {
+ #[cfg(fuzzing)]
+ return SourceFile {
+ path: PathBuf::from("<unspecified>"),
+ };
+
+ #[cfg(not(fuzzing))]
SOURCE_MAP.with(|cm| {
let cm = cm.borrow();
- let fi = cm.fileinfo(*self);
- SourceFile {
- path: Path::new(&fi.name).to_owned(),
- }
+ let path = cm.filepath(*self);
+ SourceFile { path }
})
}
#[cfg(span_locations)]
pub fn start(&self) -> LineColumn {
+ #[cfg(fuzzing)]
+ return LineColumn { line: 0, column: 0 };
+
+ #[cfg(not(fuzzing))]
SOURCE_MAP.with(|cm| {
let cm = cm.borrow();
let fi = cm.fileinfo(*self);
@@ -507,6 +556,10 @@ impl Span {
#[cfg(span_locations)]
pub fn end(&self) -> LineColumn {
+ #[cfg(fuzzing)]
+ return LineColumn { line: 0, column: 0 };
+
+ #[cfg(not(fuzzing))]
SOURCE_MAP.with(|cm| {
let cm = cm.borrow();
let fi = cm.fileinfo(*self);
@@ -514,26 +567,6 @@ impl Span {
})
}
- #[cfg(procmacro2_semver_exempt)]
- pub fn before(&self) -> Span {
- Span {
- #[cfg(span_locations)]
- lo: self.lo,
- #[cfg(span_locations)]
- hi: self.lo,
- }
- }
-
- #[cfg(procmacro2_semver_exempt)]
- pub fn after(&self) -> Span {
- Span {
- #[cfg(span_locations)]
- lo: self.hi,
- #[cfg(span_locations)]
- hi: self.hi,
- }
- }
-
#[cfg(not(span_locations))]
pub fn join(&self, _other: Span) -> Option<Span> {
Some(Span {})
@@ -541,6 +574,13 @@ impl Span {
#[cfg(span_locations)]
pub fn join(&self, other: Span) -> Option<Span> {
+ #[cfg(fuzzing)]
+ return {
+ let _ = other;
+ None
+ };
+
+ #[cfg(not(fuzzing))]
SOURCE_MAP.with(|cm| {
let cm = cm.borrow();
// If `other` is not within the same FileInfo as us, return None.
@@ -555,12 +595,32 @@ impl Span {
}
#[cfg(not(span_locations))]
- fn first_byte(self) -> Self {
+ pub fn source_text(&self) -> Option<String> {
+ None
+ }
+
+ #[cfg(span_locations)]
+ pub fn source_text(&self) -> Option<String> {
+ #[cfg(fuzzing)]
+ return None;
+
+ #[cfg(not(fuzzing))]
+ {
+ if self.is_call_site() {
+ None
+ } else {
+ Some(SOURCE_MAP.with(|cm| cm.borrow_mut().fileinfo_mut(*self).source_text(*self)))
+ }
+ }
+ }
+
+ #[cfg(not(span_locations))]
+ pub(crate) fn first_byte(self) -> Self {
self
}
#[cfg(span_locations)]
- fn first_byte(self) -> Self {
+ pub(crate) fn first_byte(self) -> Self {
Span {
lo: self.lo,
hi: cmp::min(self.lo.saturating_add(1), self.hi),
@@ -568,17 +628,22 @@ impl Span {
}
#[cfg(not(span_locations))]
- fn last_byte(self) -> Self {
+ pub(crate) fn last_byte(self) -> Self {
self
}
#[cfg(span_locations)]
- fn last_byte(self) -> Self {
+ pub(crate) fn last_byte(self) -> Self {
Span {
lo: cmp::max(self.hi.saturating_sub(1), self.lo),
hi: self.hi,
}
}
+
+ #[cfg(span_locations)]
+ fn is_call_site(&self) -> bool {
+ self.lo == 0 && self.hi == 0
+ }
}
impl Debug for Span {
@@ -594,7 +659,7 @@ impl Debug for Span {
pub(crate) fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) {
#[cfg(span_locations)]
{
- if span.lo == 0 && span.hi == 0 {
+ if span.is_call_site() {
return;
}
}
@@ -730,7 +795,7 @@ fn validate_ident(string: &str, raw: bool) {
panic!("Ident is not allowed to be empty; use Option<Ident>");
}
- if string.bytes().all(|digit| digit >= b'0' && digit <= b'9') {
+ if string.bytes().all(|digit| b'0' <= digit && digit <= b'9') {
panic!("Ident cannot be a number; use Literal instead");
}
@@ -791,6 +856,7 @@ impl Display for Ident {
}
}
+#[allow(clippy::missing_fields_in_debug)]
impl Debug for Ident {
// Ident(proc_macro), Ident(r#union)
#[cfg(not(span_locations))]
@@ -899,12 +965,25 @@ impl Literal {
pub fn string(t: &str) -> Literal {
let mut repr = String::with_capacity(t.len() + 2);
repr.push('"');
- for c in t.chars() {
- if c == '\'' {
+ let mut chars = t.chars();
+ while let Some(ch) = chars.next() {
+ if ch == '\0' {
+ repr.push_str(
+ if chars
+ .as_str()
+ .starts_with(|next| '0' <= next && next <= '7')
+ {
+ // circumvent clippy::octal_escapes lint
+ "\\x00"
+ } else {
+ "\\0"
+ },
+ );
+ } else if ch == '\'' {
// escape_debug turns this into "\'" which is unnecessary.
- repr.push(c);
+ repr.push(ch);
} else {
- repr.extend(c.escape_debug());
+ repr.extend(ch.escape_debug());
}
}
repr.push('"');
@@ -926,16 +1005,21 @@ impl Literal {
pub fn byte_string(bytes: &[u8]) -> Literal {
let mut escaped = "b\"".to_string();
- for b in bytes {
+ let mut bytes = bytes.iter();
+ while let Some(&b) = bytes.next() {
#[allow(clippy::match_overlapping_arm)]
- match *b {
- b'\0' => escaped.push_str(r"\0"),
+ match b {
+ b'\0' => escaped.push_str(match bytes.as_slice().first() {
+ // circumvent clippy::octal_escapes lint
+ Some(b'0'..=b'7') => r"\x00",
+ _ => r"\0",
+ }),
b'\t' => escaped.push_str(r"\t"),
b'\n' => escaped.push_str(r"\n"),
b'\r' => escaped.push_str(r"\r"),
b'"' => escaped.push_str("\\\""),
b'\\' => escaped.push_str("\\\\"),
- b'\x20'..=b'\x7E' => escaped.push(*b as char),
+ b'\x20'..=b'\x7E' => escaped.push(b as char),
_ => {
let _ = write!(escaped, "\\x{:02X}", b);
}
@@ -953,28 +1037,75 @@ impl Literal {
self.span = span;
}
- pub fn subspan<R: RangeBounds<usize>>(&self, _range: R) -> Option<Span> {
- None
+ pub fn subspan<R: RangeBounds<usize>>(&self, range: R) -> Option<Span> {
+ #[cfg(not(span_locations))]
+ {
+ let _ = range;
+ None
+ }
+
+ #[cfg(span_locations)]
+ {
+ use core::ops::Bound;
+
+ let lo = match range.start_bound() {
+ Bound::Included(start) => {
+ let start = u32::try_from(*start).ok()?;
+ self.span.lo.checked_add(start)?
+ }
+ Bound::Excluded(start) => {
+ let start = u32::try_from(*start).ok()?;
+ self.span.lo.checked_add(start)?.checked_add(1)?
+ }
+ Bound::Unbounded => self.span.lo,
+ };
+ let hi = match range.end_bound() {
+ Bound::Included(end) => {
+ let end = u32::try_from(*end).ok()?;
+ self.span.lo.checked_add(end)?.checked_add(1)?
+ }
+ Bound::Excluded(end) => {
+ let end = u32::try_from(*end).ok()?;
+ self.span.lo.checked_add(end)?
+ }
+ Bound::Unbounded => self.span.hi,
+ };
+ if lo <= hi && hi <= self.span.hi {
+ Some(Span { lo, hi })
+ } else {
+ None
+ }
+ }
}
}
impl FromStr for Literal {
type Err = LexError;
- fn from_str(mut repr: &str) -> Result<Self, Self::Err> {
- let negative = repr.starts_with('-');
+ fn from_str(repr: &str) -> Result<Self, Self::Err> {
+ let mut cursor = get_cursor(repr);
+ #[cfg(span_locations)]
+ let lo = cursor.off;
+
+ let negative = cursor.starts_with_char('-');
if negative {
- repr = &repr[1..];
- if !repr.starts_with(|ch: char| ch.is_ascii_digit()) {
+ cursor = cursor.advance(1);
+ if !cursor.starts_with_fn(|ch| ch.is_ascii_digit()) {
return Err(LexError::call_site());
}
}
- let cursor = get_cursor(repr);
- if let Ok((_rest, mut literal)) = parse::literal(cursor) {
- if literal.repr.len() == repr.len() {
+
+ if let Ok((rest, mut literal)) = parse::literal(cursor) {
+ if rest.is_empty() {
if negative {
literal.repr.insert(0, '-');
}
+ literal.span = Span {
+ #[cfg(span_locations)]
+ lo,
+ #[cfg(span_locations)]
+ hi: rest.off,
+ };
return Ok(literal);
}
}
diff --git a/src/lib.rs b/src/lib.rs
index 261c167..56d986b 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -55,7 +55,7 @@
//! If parsing with [Syn], you'll use [`parse_macro_input!`] instead to
//! propagate parse errors correctly back to the compiler when parsing fails.
//!
-//! [`parse_macro_input!`]: https://docs.rs/syn/1.0/syn/macro.parse_macro_input.html
+//! [`parse_macro_input!`]: https://docs.rs/syn/2.0/syn/macro.parse_macro_input.html
//!
//! # Unstable features
//!
@@ -65,7 +65,7 @@
//!
//! To opt into the additional APIs available in the most recent nightly
//! compiler, the `procmacro2_semver_exempt` config flag must be passed to
-//! rustc. We will polyfill those nightly-only APIs back to Rust 1.31.0. As
+//! rustc. We will polyfill those nightly-only APIs back to Rust 1.56.0. As
//! these are unstable APIs that track the nightly compiler, minor versions of
//! proc-macro2 may make breaking changes to them at any time.
//!
@@ -86,11 +86,8 @@
//! a different thread.
// Proc-macro2 types in rustdoc of other crates get linked to here.
-#![doc(html_root_url = "https://docs.rs/proc-macro2/1.0.51")]
-#![cfg_attr(
- any(proc_macro_span, super_unstable),
- feature(proc_macro_span, proc_macro_span_shrink)
-)]
+#![doc(html_root_url = "https://docs.rs/proc-macro2/1.0.69")]
+#![cfg_attr(any(proc_macro_span, super_unstable), feature(proc_macro_span))]
#![cfg_attr(super_unstable, feature(proc_macro_def_site))]
#![cfg_attr(doc_cfg, feature(doc_cfg))]
#![allow(
@@ -98,9 +95,14 @@
clippy::cast_possible_truncation,
clippy::doc_markdown,
clippy::items_after_statements,
+ clippy::iter_without_into_iter,
+ clippy::let_underscore_untyped,
clippy::manual_assert,
+ clippy::manual_range_contains,
+ clippy::missing_safety_doc,
clippy::must_use_candidate,
clippy::needless_doctest_main,
+ clippy::new_without_default,
clippy::return_self_not_must_use,
clippy::shadow_unrelated,
clippy::trivially_copy_pass_by_ref,
@@ -118,7 +120,9 @@ compile_error! {"\
build script as well.
"}
-#[cfg(use_proc_macro)]
+extern crate alloc;
+
+#[cfg(feature = "proc-macro")]
extern crate proc_macro;
mod marker;
@@ -133,6 +137,8 @@ mod detection;
#[doc(hidden)]
pub mod fallback;
+pub mod extra;
+
#[cfg(not(wrap_proc_macro))]
use crate::fallback as imp;
#[path = "wrapper.rs"]
@@ -142,11 +148,11 @@ mod imp;
#[cfg(span_locations)]
mod location;
+use crate::extra::DelimSpan;
use crate::marker::Marker;
use core::cmp::Ordering;
use core::fmt::{self, Debug, Display};
use core::hash::{Hash, Hasher};
-use core::iter::FromIterator;
use core::ops::RangeBounds;
use core::str::FromStr;
use std::error::Error;
@@ -183,7 +189,7 @@ impl TokenStream {
}
}
- fn _new_stable(inner: fallback::TokenStream) -> Self {
+ fn _new_fallback(inner: fallback::TokenStream) -> Self {
TokenStream {
inner: inner.into(),
_marker: Marker,
@@ -229,14 +235,16 @@ impl FromStr for TokenStream {
}
}
-#[cfg(use_proc_macro)]
+#[cfg(feature = "proc-macro")]
+#[cfg_attr(doc_cfg, doc(cfg(feature = "proc-macro")))]
impl From<proc_macro::TokenStream> for TokenStream {
fn from(inner: proc_macro::TokenStream) -> Self {
TokenStream::_new(inner.into())
}
}
-#[cfg(use_proc_macro)]
+#[cfg(feature = "proc-macro")]
+#[cfg_attr(doc_cfg, doc(cfg(feature = "proc-macro")))]
impl From<TokenStream> for proc_macro::TokenStream {
fn from(inner: TokenStream) -> Self {
inner.inner.into()
@@ -377,7 +385,7 @@ impl Span {
}
}
- fn _new_stable(inner: fallback::Span) -> Self {
+ fn _new_fallback(inner: fallback::Span) -> Self {
Span {
inner: inner.into(),
_marker: Marker,
@@ -396,9 +404,6 @@ impl Span {
/// The span located at the invocation of the procedural macro, but with
/// local variables, labels, and `$crate` resolved at the definition site
/// of the macro. This is the same hygiene behavior as `macro_rules`.
- ///
- /// This function requires Rust 1.45 or later.
- #[cfg(not(no_hygiene))]
pub fn mixed_site() -> Self {
Span::_new(imp::Span::mixed_site())
}
@@ -485,24 +490,6 @@ impl Span {
self.inner.end()
}
- /// Creates an empty span pointing to directly before this span.
- ///
- /// This method is semver exempt and not exposed by default.
- #[cfg(all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)))]
- #[cfg_attr(doc_cfg, doc(cfg(procmacro2_semver_exempt)))]
- pub fn before(&self) -> Span {
- Span::_new(self.inner.before())
- }
-
- /// Creates an empty span pointing to directly after this span.
- ///
- /// This method is semver exempt and not exposed by default.
- #[cfg(all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)))]
- #[cfg_attr(doc_cfg, doc(cfg(procmacro2_semver_exempt)))]
- pub fn after(&self) -> Span {
- Span::_new(self.inner.after())
- }
-
/// Create a new span encompassing `self` and `other`.
///
/// Returns `None` if `self` and `other` are from different files.
@@ -524,6 +511,17 @@ impl Span {
pub fn eq(&self, other: &Span) -> bool {
self.inner.eq(&other.inner)
}
+
+ /// Returns the source text behind a span. This preserves the original
+ /// source code, including spaces and comments. It only returns a result if
+ /// the span corresponds to real source code.
+ ///
+ /// Note: The observable result of a macro should only rely on the tokens
+ /// and not on this source text. The result of this function is a best
+ /// effort to be used for diagnostics only.
+ pub fn source_text(&self) -> Option<String> {
+ self.inner.source_text()
+ }
}
/// Prints a span in a form convenient for debugging.
@@ -664,7 +662,7 @@ impl Group {
Group { inner }
}
- fn _new_stable(inner: fallback::Group) -> Self {
+ fn _new_fallback(inner: fallback::Group) -> Self {
Group {
inner: inner.into(),
}
@@ -681,7 +679,8 @@ impl Group {
}
}
- /// Returns the delimiter of this `Group`
+ /// Returns the punctuation used as the delimiter for this group: a set of
+ /// parentheses, square brackets, or curly braces.
pub fn delimiter(&self) -> Delimiter {
self.inner.delimiter()
}
@@ -725,6 +724,13 @@ impl Group {
Span::_new(self.inner.span_close())
}
+ /// Returns an object that holds this group's `span_open()` and
+ /// `span_close()` together (in a more compact representation than holding
+ /// those 2 spans individually).
+ pub fn delim_span(&self) -> DelimSpan {
+ DelimSpan::new(&self.inner)
+ }
+
/// Configures the span for this `Group`'s delimiters, but not its internal
/// tokens.
///
@@ -848,7 +854,7 @@ impl Debug for Punct {
/// Rust keywords. Use `input.call(Ident::parse_any)` when parsing to match the
/// behaviour of `Ident::new`.
///
-/// [`Parse`]: https://docs.rs/syn/1.0/syn/parse/trait.Parse.html
+/// [`Parse`]: https://docs.rs/syn/2.0/syn/parse/trait.Parse.html
///
/// # Examples
///
@@ -939,7 +945,7 @@ impl Ident {
/// Panics if the input string is neither a keyword nor a legal variable
/// name. If you are not sure whether the string contains an identifier and
/// need to handle an error case, use
- /// <a href="https://docs.rs/syn/1.0/syn/fn.parse_str.html"><code
+ /// <a href="https://docs.rs/syn/2.0/syn/fn.parse_str.html"><code
/// style="padding-right:0;">syn::parse_str</code></a><code
/// style="padding-left:0;">::&lt;Ident&gt;</code>
/// rather than `Ident::new`.
@@ -1081,7 +1087,7 @@ impl Literal {
}
}
- fn _new_stable(inner: fallback::Literal) -> Self {
+ fn _new_fallback(inner: fallback::Literal) -> Self {
Literal {
inner: inner.into(),
_marker: Marker,
diff --git a/src/marker.rs b/src/marker.rs
index 59fd096..e648dd2 100644
--- a/src/marker.rs
+++ b/src/marker.rs
@@ -1,6 +1,6 @@
+use alloc::rc::Rc;
use core::marker::PhantomData;
-use std::panic::{RefUnwindSafe, UnwindSafe};
-use std::rc::Rc;
+use core::panic::{RefUnwindSafe, UnwindSafe};
// Zero sized marker with the correct set of autotrait impls we want all proc
// macro types to have.
diff --git a/src/parse.rs b/src/parse.rs
index 2a87948..1430d73 100644
--- a/src/parse.rs
+++ b/src/parse.rs
@@ -27,7 +27,18 @@ impl<'a> Cursor<'a> {
self.rest.starts_with(s)
}
- fn is_empty(&self) -> bool {
+ pub fn starts_with_char(&self, ch: char) -> bool {
+ self.rest.starts_with(ch)
+ }
+
+ pub fn starts_with_fn<Pattern>(&self, f: Pattern) -> bool
+ where
+ Pattern: FnMut(char) -> bool,
+ {
+ self.rest.starts_with(f)
+ }
+
+ pub fn is_empty(&self) -> bool {
self.rest.is_empty()
}
@@ -97,7 +108,7 @@ fn skip_whitespace(input: Cursor) -> Cursor {
s = s.advance(1);
continue;
}
- b if b <= 0x7f => {}
+ b if b.is_ascii() => {}
_ => {
let ch = s.chars().next().unwrap();
if is_whitespace(ch) {
@@ -150,6 +161,10 @@ fn word_break(input: Cursor) -> Result<Cursor, Reject> {
}
}
+// Rustc's representation of a macro expansion error in expression position or
+// type position.
+const ERROR: &str = "(/*ERROR*/)";
+
pub(crate) fn token_stream(mut input: Cursor) -> Result<TokenStream, LexError> {
let mut trees = TokenStreamBuilder::new();
let mut stack = Vec::new();
@@ -181,7 +196,7 @@ pub(crate) fn token_stream(mut input: Cursor) -> Result<TokenStream, LexError> {
};
if let Some(open_delimiter) = match first {
- b'(' => Some(Delimiter::Parenthesis),
+ b'(' if !input.starts_with(ERROR) => Some(Delimiter::Parenthesis),
b'[' => Some(Delimiter::Bracket),
b'{' => Some(Delimiter::Brace),
_ => None,
@@ -217,13 +232,13 @@ pub(crate) fn token_stream(mut input: Cursor) -> Result<TokenStream, LexError> {
hi: input.off,
});
trees = outer;
- trees.push_token_from_parser(TokenTree::Group(crate::Group::_new_stable(g)));
+ trees.push_token_from_parser(TokenTree::Group(crate::Group::_new_fallback(g)));
} else {
let (rest, mut tt) = match leaf_token(input) {
Ok((rest, tt)) => (rest, tt),
Err(Reject) => return Err(lex_error(input)),
};
- tt.set_span(crate::Span::_new_stable(Span {
+ tt.set_span(crate::Span::_new_fallback(Span {
#[cfg(span_locations)]
lo,
#[cfg(span_locations)]
@@ -251,20 +266,26 @@ fn lex_error(cursor: Cursor) -> LexError {
fn leaf_token(input: Cursor) -> PResult<TokenTree> {
if let Ok((input, l)) = literal(input) {
// must be parsed before ident
- Ok((input, TokenTree::Literal(crate::Literal::_new_stable(l))))
+ Ok((input, TokenTree::Literal(crate::Literal::_new_fallback(l))))
} else if let Ok((input, p)) = punct(input) {
Ok((input, TokenTree::Punct(p)))
} else if let Ok((input, i)) = ident(input) {
Ok((input, TokenTree::Ident(i)))
+ } else if input.starts_with(ERROR) {
+ let rest = input.advance(ERROR.len());
+ let repr = crate::Literal::_new_fallback(Literal::_new(ERROR.to_owned()));
+ Ok((rest, TokenTree::Literal(repr)))
} else {
Err(Reject)
}
}
fn ident(input: Cursor) -> PResult<crate::Ident> {
- if ["r\"", "r#\"", "r##", "b\"", "b\'", "br\"", "br#"]
- .iter()
- .any(|prefix| input.starts_with(prefix))
+ if [
+ "r\"", "r#\"", "r##", "b\"", "b\'", "br\"", "br#", "c\"", "cr\"", "cr#",
+ ]
+ .iter()
+ .any(|prefix| input.starts_with(prefix))
{
Err(Reject)
} else {
@@ -322,6 +343,8 @@ fn literal_nocapture(input: Cursor) -> Result<Cursor, Reject> {
Ok(ok)
} else if let Ok(ok) = byte_string(input) {
Ok(ok)
+ } else if let Ok(ok) = c_string(input) {
+ Ok(ok)
} else if let Ok(ok) = byte(input) {
Ok(ok)
} else if let Ok(ok) = character(input) {
@@ -352,8 +375,8 @@ fn string(input: Cursor) -> Result<Cursor, Reject> {
}
}
-fn cooked_string(input: Cursor) -> Result<Cursor, Reject> {
- let mut chars = input.char_indices().peekable();
+fn cooked_string(mut input: Cursor) -> Result<Cursor, Reject> {
+ let mut chars = input.char_indices();
while let Some((i, ch)) = chars.next() {
match ch {
@@ -367,31 +390,16 @@ fn cooked_string(input: Cursor) -> Result<Cursor, Reject> {
},
'\\' => match chars.next() {
Some((_, 'x')) => {
- if !backslash_x_char(&mut chars) {
- break;
- }
+ backslash_x_char(&mut chars)?;
}
- Some((_, 'n')) | Some((_, 'r')) | Some((_, 't')) | Some((_, '\\'))
- | Some((_, '\'')) | Some((_, '"')) | Some((_, '0')) => {}
+ Some((_, 'n' | 'r' | 't' | '\\' | '\'' | '"' | '0')) => {}
Some((_, 'u')) => {
- if !backslash_u(&mut chars) {
- break;
- }
+ backslash_u(&mut chars)?;
}
- Some((_, ch @ '\n')) | Some((_, ch @ '\r')) => {
- let mut last = ch;
- loop {
- if last == '\r' && chars.next().map_or(true, |(_, ch)| ch != '\n') {
- return Err(Reject);
- }
- match chars.peek() {
- Some((_, ch)) if ch.is_whitespace() => {
- last = *ch;
- chars.next();
- }
- _ => break,
- }
- }
+ Some((newline, ch @ ('\n' | '\r'))) => {
+ input = input.advance(newline + 1);
+ trailing_backslash(&mut input, ch as u8)?;
+ chars = input.char_indices();
}
_ => break,
},
@@ -401,11 +409,30 @@ fn cooked_string(input: Cursor) -> Result<Cursor, Reject> {
Err(Reject)
}
+fn raw_string(input: Cursor) -> Result<Cursor, Reject> {
+ let (input, delimiter) = delimiter_of_raw_string(input)?;
+ let mut bytes = input.bytes().enumerate();
+ while let Some((i, byte)) = bytes.next() {
+ match byte {
+ b'"' if input.rest[i + 1..].starts_with(delimiter) => {
+ let rest = input.advance(i + 1 + delimiter.len());
+ return Ok(literal_suffix(rest));
+ }
+ b'\r' => match bytes.next() {
+ Some((_, b'\n')) => {}
+ _ => break,
+ },
+ _ => {}
+ }
+ }
+ Err(Reject)
+}
+
fn byte_string(input: Cursor) -> Result<Cursor, Reject> {
if let Ok(input) = input.parse("b\"") {
cooked_byte_string(input)
} else if let Ok(input) = input.parse("br") {
- raw_string(input)
+ raw_byte_string(input)
} else {
Err(Reject)
}
@@ -425,68 +452,125 @@ fn cooked_byte_string(mut input: Cursor) -> Result<Cursor, Reject> {
},
b'\\' => match bytes.next() {
Some((_, b'x')) => {
- if !backslash_x_byte(&mut bytes) {
- break;
- }
+ backslash_x_byte(&mut bytes)?;
}
- Some((_, b'n')) | Some((_, b'r')) | Some((_, b't')) | Some((_, b'\\'))
- | Some((_, b'0')) | Some((_, b'\'')) | Some((_, b'"')) => {}
- Some((newline, b @ b'\n')) | Some((newline, b @ b'\r')) => {
- let mut last = b as char;
- let rest = input.advance(newline + 1);
- let mut chars = rest.char_indices();
- loop {
- if last == '\r' && chars.next().map_or(true, |(_, ch)| ch != '\n') {
- return Err(Reject);
- }
- match chars.next() {
- Some((_, ch)) if ch.is_whitespace() => last = ch,
- Some((offset, _)) => {
- input = rest.advance(offset);
- bytes = input.bytes().enumerate();
- break;
- }
- None => return Err(Reject),
- }
- }
+ Some((_, b'n' | b'r' | b't' | b'\\' | b'0' | b'\'' | b'"')) => {}
+ Some((newline, b @ (b'\n' | b'\r'))) => {
+ input = input.advance(newline + 1);
+ trailing_backslash(&mut input, b)?;
+ bytes = input.bytes().enumerate();
}
_ => break,
},
- b if b < 0x80 => {}
+ b if b.is_ascii() => {}
_ => break,
}
}
Err(Reject)
}
-fn raw_string(input: Cursor) -> Result<Cursor, Reject> {
- let mut chars = input.char_indices();
- let mut n = 0;
- for (i, ch) in &mut chars {
- match ch {
- '"' => {
- n = i;
- break;
+fn delimiter_of_raw_string(input: Cursor) -> PResult<&str> {
+ for (i, byte) in input.bytes().enumerate() {
+ match byte {
+ b'"' => {
+ if i > 255 {
+ // https://github.com/rust-lang/rust/pull/95251
+ return Err(Reject);
+ }
+ return Ok((input.advance(i + 1), &input.rest[..i]));
}
- '#' => {}
- _ => return Err(Reject),
+ b'#' => {}
+ _ => break,
}
}
- if n > 255 {
- // https://github.com/rust-lang/rust/pull/95251
- return Err(Reject);
+ Err(Reject)
+}
+
+fn raw_byte_string(input: Cursor) -> Result<Cursor, Reject> {
+ let (input, delimiter) = delimiter_of_raw_string(input)?;
+ let mut bytes = input.bytes().enumerate();
+ while let Some((i, byte)) = bytes.next() {
+ match byte {
+ b'"' if input.rest[i + 1..].starts_with(delimiter) => {
+ let rest = input.advance(i + 1 + delimiter.len());
+ return Ok(literal_suffix(rest));
+ }
+ b'\r' => match bytes.next() {
+ Some((_, b'\n')) => {}
+ _ => break,
+ },
+ other => {
+ if !other.is_ascii() {
+ break;
+ }
+ }
+ }
}
+ Err(Reject)
+}
+
+fn c_string(input: Cursor) -> Result<Cursor, Reject> {
+ if let Ok(input) = input.parse("c\"") {
+ cooked_c_string(input)
+ } else if let Ok(input) = input.parse("cr") {
+ raw_c_string(input)
+ } else {
+ Err(Reject)
+ }
+}
+
+fn raw_c_string(input: Cursor) -> Result<Cursor, Reject> {
+ let (input, delimiter) = delimiter_of_raw_string(input)?;
+ let mut bytes = input.bytes().enumerate();
+ while let Some((i, byte)) = bytes.next() {
+ match byte {
+ b'"' if input.rest[i + 1..].starts_with(delimiter) => {
+ let rest = input.advance(i + 1 + delimiter.len());
+ return Ok(literal_suffix(rest));
+ }
+ b'\r' => match bytes.next() {
+ Some((_, b'\n')) => {}
+ _ => break,
+ },
+ b'\0' => break,
+ _ => {}
+ }
+ }
+ Err(Reject)
+}
+
+fn cooked_c_string(mut input: Cursor) -> Result<Cursor, Reject> {
+ let mut chars = input.char_indices();
+
while let Some((i, ch)) = chars.next() {
match ch {
- '"' if input.rest[i + 1..].starts_with(&input.rest[..n]) => {
- let rest = input.advance(i + 1 + n);
- return Ok(literal_suffix(rest));
+ '"' => {
+ let input = input.advance(i + 1);
+ return Ok(literal_suffix(input));
}
'\r' => match chars.next() {
Some((_, '\n')) => {}
_ => break,
},
- _ => {}
+ '\\' => match chars.next() {
+ Some((_, 'x')) => {
+ backslash_x_nonzero(&mut chars)?;
+ }
+ Some((_, 'n' | 'r' | 't' | '\\' | '\'' | '"')) => {}
+ Some((_, 'u')) => {
+ if backslash_u(&mut chars)? == '\0' {
+ break;
+ }
+ }
+ Some((newline, ch @ ('\n' | '\r'))) => {
+ input = input.advance(newline + 1);
+ trailing_backslash(&mut input, ch as u8)?;
+ chars = input.char_indices();
+ }
+ _ => break,
+ },
+ '\0' => break,
+ _ch => {}
}
}
Err(Reject)
@@ -497,9 +581,8 @@ fn byte(input: Cursor) -> Result<Cursor, Reject> {
let mut bytes = input.bytes().enumerate();
let ok = match bytes.next().map(|(_, b)| b) {
Some(b'\\') => match bytes.next().map(|(_, b)| b) {
- Some(b'x') => backslash_x_byte(&mut bytes),
- Some(b'n') | Some(b'r') | Some(b't') | Some(b'\\') | Some(b'0') | Some(b'\'')
- | Some(b'"') => true,
+ Some(b'x') => backslash_x_byte(&mut bytes).is_ok(),
+ Some(b'n' | b'r' | b't' | b'\\' | b'0' | b'\'' | b'"') => true,
_ => false,
},
b => b.is_some(),
@@ -520,11 +603,9 @@ fn character(input: Cursor) -> Result<Cursor, Reject> {
let mut chars = input.char_indices();
let ok = match chars.next().map(|(_, ch)| ch) {
Some('\\') => match chars.next().map(|(_, ch)| ch) {
- Some('x') => backslash_x_char(&mut chars),
- Some('u') => backslash_u(&mut chars),
- Some('n') | Some('r') | Some('t') | Some('\\') | Some('0') | Some('\'') | Some('"') => {
- true
- }
+ Some('x') => backslash_x_char(&mut chars).is_ok(),
+ Some('u') => backslash_u(&mut chars).is_ok(),
+ Some('n' | 'r' | 't' | '\\' | '0' | '\'' | '"') => true,
_ => false,
},
ch => ch.is_some(),
@@ -538,36 +619,49 @@ fn character(input: Cursor) -> Result<Cursor, Reject> {
}
macro_rules! next_ch {
- ($chars:ident @ $pat:pat $(| $rest:pat)*) => {
+ ($chars:ident @ $pat:pat) => {
match $chars.next() {
Some((_, ch)) => match ch {
- $pat $(| $rest)* => ch,
- _ => return false,
+ $pat => ch,
+ _ => return Err(Reject),
},
- None => return false,
+ None => return Err(Reject),
}
};
}
-fn backslash_x_char<I>(chars: &mut I) -> bool
+fn backslash_x_char<I>(chars: &mut I) -> Result<(), Reject>
where
I: Iterator<Item = (usize, char)>,
{
next_ch!(chars @ '0'..='7');
next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F');
- true
+ Ok(())
}
-fn backslash_x_byte<I>(chars: &mut I) -> bool
+fn backslash_x_byte<I>(chars: &mut I) -> Result<(), Reject>
where
I: Iterator<Item = (usize, u8)>,
{
next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F');
next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F');
- true
+ Ok(())
}
-fn backslash_u<I>(chars: &mut I) -> bool
+fn backslash_x_nonzero<I>(chars: &mut I) -> Result<(), Reject>
+where
+ I: Iterator<Item = (usize, char)>,
+{
+ let first = next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F');
+ let second = next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F');
+ if first == '0' && second == '0' {
+ Err(Reject)
+ } else {
+ Ok(())
+ }
+}
+
+fn backslash_u<I>(chars: &mut I) -> Result<char, Reject>
where
I: Iterator<Item = (usize, char)>,
{
@@ -580,17 +674,36 @@ where
'a'..='f' => 10 + ch as u8 - b'a',
'A'..='F' => 10 + ch as u8 - b'A',
'_' if len > 0 => continue,
- '}' if len > 0 => return char::from_u32(value).is_some(),
- _ => return false,
+ '}' if len > 0 => return char::from_u32(value).ok_or(Reject),
+ _ => break,
};
if len == 6 {
- return false;
+ break;
}
value *= 0x10;
value += u32::from(digit);
len += 1;
}
- false
+ Err(Reject)
+}
+
+fn trailing_backslash(input: &mut Cursor, mut last: u8) -> Result<(), Reject> {
+ let mut whitespace = input.bytes().enumerate();
+ loop {
+ if last == b'\r' && whitespace.next().map_or(true, |(_, b)| b != b'\n') {
+ return Err(Reject);
+ }
+ match whitespace.next() {
+ Some((_, b @ (b' ' | b'\t' | b'\n' | b'\r'))) => {
+ last = b;
+ }
+ Some((offset, _)) => {
+ *input = input.advance(offset);
+ return Ok(());
+ }
+ None => return Err(Reject),
+ }
+ }
}
fn float(input: Cursor) -> Result<Cursor, Reject> {
@@ -606,7 +719,7 @@ fn float(input: Cursor) -> Result<Cursor, Reject> {
fn float_digits(input: Cursor) -> Result<Cursor, Reject> {
let mut chars = input.chars().peekable();
match chars.next() {
- Some(ch) if ch >= '0' && ch <= '9' => {}
+ Some(ch) if '0' <= ch && ch <= '9' => {}
_ => return Err(Reject),
}
@@ -756,7 +869,7 @@ fn digits(mut input: Cursor) -> Result<Cursor, Reject> {
fn punct(input: Cursor) -> PResult<Punct> {
let (rest, ch) = punct_char(input)?;
if ch == '\'' {
- if ident_any(rest)?.0.starts_with("'") {
+ if ident_any(rest)?.0.starts_with_char('\'') {
Err(Reject)
} else {
Ok((rest, Punct::new('\'', Spacing::Joint)))
@@ -795,7 +908,7 @@ fn doc_comment<'a>(input: Cursor<'a>, trees: &mut TokenStreamBuilder) -> PResult
#[cfg(span_locations)]
let lo = input.off;
let (rest, (comment, inner)) = doc_comment_contents(input)?;
- let span = crate::Span::_new_stable(Span {
+ let span = crate::Span::_new_fallback(Span {
#[cfg(span_locations)]
lo,
#[cfg(span_locations)]
@@ -831,7 +944,7 @@ fn doc_comment<'a>(input: Cursor<'a>, trees: &mut TokenStreamBuilder) -> PResult
bracketed.push_token_from_parser(TokenTree::Punct(equal));
bracketed.push_token_from_parser(TokenTree::Literal(literal));
let group = Group::new(Delimiter::Bracket, bracketed.build());
- let mut group = crate::Group::_new_stable(group);
+ let mut group = crate::Group::_new_fallback(group);
group.set_span(span);
trees.push_token_from_parser(TokenTree::Group(group));
@@ -848,7 +961,7 @@ fn doc_comment_contents(input: Cursor) -> PResult<(&str, bool)> {
Ok((input, (&s[3..s.len() - 2], true)))
} else if input.starts_with("///") {
let input = input.advance(3);
- if input.starts_with("/") {
+ if input.starts_with_char('/') {
return Err(Reject);
}
let (input, s) = take_until_newline_or_eof(input);
diff --git a/src/rcvec.rs b/src/rcvec.rs
index 86ca7d8..37955af 100644
--- a/src/rcvec.rs
+++ b/src/rcvec.rs
@@ -1,7 +1,8 @@
+use alloc::rc::Rc;
+use alloc::vec;
use core::mem;
+use core::panic::RefUnwindSafe;
use core::slice;
-use std::rc::Rc;
-use std::vec;
pub(crate) struct RcVec<T> {
inner: Rc<Vec<T>>,
@@ -52,7 +53,7 @@ impl<T> RcVec<T> {
T: Clone,
{
let vec = if let Some(owned) = Rc::get_mut(&mut self.inner) {
- mem::replace(owned, Vec::new())
+ mem::take(owned)
} else {
Vec::clone(&self.inner)
};
@@ -140,3 +141,5 @@ impl<T> Iterator for RcVecIntoIter<T> {
self.inner.size_hint()
}
}
+
+impl<T> RefUnwindSafe for RcVec<T> where T: RefUnwindSafe {}
diff --git a/src/wrapper.rs b/src/wrapper.rs
index f5ec06b..860b6b7 100644
--- a/src/wrapper.rs
+++ b/src/wrapper.rs
@@ -3,7 +3,6 @@ use crate::detection::inside_proc_macro;
use crate::location::LineColumn;
use crate::{fallback, Delimiter, Punct, Spacing, TokenTree};
use core::fmt::{self, Debug, Display};
-use core::iter::FromIterator;
use core::ops::RangeBounds;
use core::str::FromStr;
use std::panic;
@@ -40,7 +39,7 @@ impl LexError {
}
fn mismatch() -> ! {
- panic!("stable/nightly mismatch")
+ panic!("compiler/fallback mismatch")
}
impl DeferredTokenStream {
@@ -286,15 +285,7 @@ impl Debug for LexError {
impl Display for LexError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
- #[cfg(not(no_lexerror_display))]
LexError::Compiler(e) => Display::fmt(e, f),
- #[cfg(no_lexerror_display)]
- LexError::Compiler(_e) => Display::fmt(
- &fallback::LexError {
- span: fallback::Span::call_site(),
- },
- f,
- ),
LexError::Fallback(e) => Display::fmt(e, f),
}
}
@@ -406,7 +397,6 @@ impl Span {
}
}
- #[cfg(not(no_hygiene))]
pub fn mixed_site() -> Self {
if inside_proc_macro() {
Span::Compiler(proc_macro::Span::mixed_site())
@@ -426,13 +416,7 @@ impl Span {
pub fn resolved_at(&self, other: Span) -> Span {
match (self, other) {
- #[cfg(not(no_hygiene))]
(Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(a.resolved_at(b)),
-
- // Name resolution affects semantics, but location is only cosmetic
- #[cfg(no_hygiene)]
- (Span::Compiler(_), Span::Compiler(_)) => other,
-
(Span::Fallback(a), Span::Fallback(b)) => Span::Fallback(a.resolved_at(b)),
_ => mismatch(),
}
@@ -440,13 +424,7 @@ impl Span {
pub fn located_at(&self, other: Span) -> Span {
match (self, other) {
- #[cfg(not(no_hygiene))]
(Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(a.located_at(b)),
-
- // Name resolution affects semantics, but location is only cosmetic
- #[cfg(no_hygiene)]
- (Span::Compiler(_), Span::Compiler(_)) => *self,
-
(Span::Fallback(a), Span::Fallback(b)) => Span::Fallback(a.located_at(b)),
_ => mismatch(),
}
@@ -470,12 +448,6 @@ impl Span {
#[cfg(span_locations)]
pub fn start(&self) -> LineColumn {
match self {
- #[cfg(proc_macro_span)]
- Span::Compiler(s) => {
- let proc_macro::LineColumn { line, column } = s.start();
- LineColumn { line, column }
- }
- #[cfg(not(proc_macro_span))]
Span::Compiler(_) => LineColumn { line: 0, column: 0 },
Span::Fallback(s) => s.start(),
}
@@ -484,33 +456,11 @@ impl Span {
#[cfg(span_locations)]
pub fn end(&self) -> LineColumn {
match self {
- #[cfg(proc_macro_span)]
- Span::Compiler(s) => {
- let proc_macro::LineColumn { line, column } = s.end();
- LineColumn { line, column }
- }
- #[cfg(not(proc_macro_span))]
Span::Compiler(_) => LineColumn { line: 0, column: 0 },
Span::Fallback(s) => s.end(),
}
}
- #[cfg(super_unstable)]
- pub fn before(&self) -> Span {
- match self {
- Span::Compiler(s) => Span::Compiler(s.before()),
- Span::Fallback(s) => Span::Fallback(s.before()),
- }
- }
-
- #[cfg(super_unstable)]
- pub fn after(&self) -> Span {
- match self {
- Span::Compiler(s) => Span::Compiler(s.after()),
- Span::Fallback(s) => Span::Fallback(s.after()),
- }
- }
-
pub fn join(&self, other: Span) -> Option<Span> {
let ret = match (self, other) {
#[cfg(proc_macro_span)]
@@ -530,6 +480,16 @@ impl Span {
}
}
+ pub fn source_text(&self) -> Option<String> {
+ match self {
+ #[cfg(not(no_source_text))]
+ Span::Compiler(s) => s.source_text(),
+ #[cfg(no_source_text)]
+ Span::Compiler(_) => None,
+ Span::Fallback(s) => s.source_text(),
+ }
+ }
+
fn unwrap_nightly(self) -> proc_macro::Span {
match self {
Span::Compiler(s) => s,
@@ -620,20 +580,14 @@ impl Group {
pub fn span_open(&self) -> Span {
match self {
- #[cfg(not(no_group_open_close))]
Group::Compiler(g) => Span::Compiler(g.span_open()),
- #[cfg(no_group_open_close)]
- Group::Compiler(g) => Span::Compiler(g.span()),
Group::Fallback(g) => Span::Fallback(g.span_open()),
}
}
pub fn span_close(&self) -> Span {
match self {
- #[cfg(not(no_group_open_close))]
Group::Compiler(g) => Span::Compiler(g.span_close()),
- #[cfg(no_group_open_close)]
- Group::Compiler(g) => Span::Compiler(g.span()),
Group::Fallback(g) => Span::Fallback(g.span_close()),
}
}
@@ -694,27 +648,7 @@ impl Ident {
pub fn new_raw(string: &str, span: Span) -> Self {
match span {
- #[cfg(not(no_ident_new_raw))]
Span::Compiler(s) => Ident::Compiler(proc_macro::Ident::new_raw(string, s)),
- #[cfg(no_ident_new_raw)]
- Span::Compiler(s) => {
- let _ = proc_macro::Ident::new(string, s);
- // At this point the un-r#-prefixed string is known to be a
- // valid identifier. Try to produce a valid raw identifier by
- // running the `TokenStream` parser, and unwrapping the first
- // token as an `Ident`.
- let raw_prefixed = format!("r#{}", string);
- if let Ok(ts) = raw_prefixed.parse::<proc_macro::TokenStream>() {
- let mut iter = ts.into_iter();
- if let (Some(proc_macro::TokenTree::Ident(mut id)), None) =
- (iter.next(), iter.next())
- {
- id.set_span(s);
- return Ident::Compiler(id);
- }
- }
- panic!("not allowed as a raw identifier: `{}`", raw_prefixed)
- }
Span::Fallback(s) => Ident::Fallback(fallback::Ident::new_raw(string, s)),
}
}
@@ -816,7 +750,7 @@ macro_rules! unsuffixed_integers {
impl Literal {
pub unsafe fn from_str_unchecked(repr: &str) -> Self {
if inside_proc_macro() {
- Literal::Compiler(compiler_literal_from_str(repr).expect("invalid literal"))
+ Literal::Compiler(proc_macro::Literal::from_str(repr).expect("invalid literal"))
} else {
Literal::Fallback(fallback::Literal::from_str_unchecked(repr))
}
@@ -939,7 +873,8 @@ impl FromStr for Literal {
fn from_str(repr: &str) -> Result<Self, Self::Err> {
if inside_proc_macro() {
- compiler_literal_from_str(repr).map(Literal::Compiler)
+ let literal = proc_macro::Literal::from_str(repr)?;
+ Ok(Literal::Compiler(literal))
} else {
let literal = fallback::Literal::from_str(repr)?;
Ok(Literal::Fallback(literal))
@@ -947,24 +882,6 @@ impl FromStr for Literal {
}
}
-fn compiler_literal_from_str(repr: &str) -> Result<proc_macro::Literal, LexError> {
- #[cfg(not(no_literal_from_str))]
- {
- proc_macro::Literal::from_str(repr).map_err(LexError::Compiler)
- }
- #[cfg(no_literal_from_str)]
- {
- let tokens = proc_macro_parse(repr)?;
- let mut iter = tokens.into_iter();
- if let (Some(proc_macro::TokenTree::Literal(literal)), None) = (iter.next(), iter.next()) {
- if literal.to_string().len() == repr.len() {
- return Ok(literal);
- }
- }
- Err(LexError::call_site())
- }
-}
-
impl Display for Literal {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
diff --git a/tests/marker.rs b/tests/marker.rs
index 4fb2beb..d08fbfc 100644
--- a/tests/marker.rs
+++ b/tests/marker.rs
@@ -1,3 +1,5 @@
+#![allow(clippy::extra_unused_type_parameters)]
+
use proc_macro2::{
Delimiter, Group, Ident, LexError, Literal, Punct, Spacing, Span, TokenStream, TokenTree,
};
@@ -60,7 +62,6 @@ mod semver_exempt {
assert_impl!(SourceFile is not Send or Sync);
}
-#[cfg(not(no_libprocmacro_unwind_safe))]
mod unwind_safe {
use proc_macro2::{
Delimiter, Group, Ident, LexError, Literal, Punct, Spacing, Span, TokenStream, TokenTree,
diff --git a/tests/test.rs b/tests/test.rs
index e0af151..b75cd55 100644
--- a/tests/test.rs
+++ b/tests/test.rs
@@ -1,12 +1,12 @@
#![allow(
clippy::assertions_on_result_states,
clippy::items_after_statements,
- clippy::non_ascii_literal
+ clippy::non_ascii_literal,
+ clippy::octal_escapes
)]
use proc_macro2::{Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
use std::iter;
-use std::panic;
use std::str::{self, FromStr};
#[test]
@@ -89,24 +89,9 @@ fn lifetime_number() {
}
#[test]
+#[should_panic(expected = r#""'a#" is not a valid Ident"#)]
fn lifetime_invalid() {
- let result = panic::catch_unwind(|| Ident::new("'a#", Span::call_site()));
- match result {
- Err(box_any) => {
- let message = box_any.downcast_ref::<String>().unwrap();
- let expected1 = r#""\'a#" is not a valid Ident"#; // 1.31.0 .. 1.53.0
- let expected2 = r#""'a#" is not a valid Ident"#; // 1.53.0 ..
- assert!(
- message == expected1 || message == expected2,
- "panic message does not match expected string\n\
- \x20 panic message: `{:?}`\n\
- \x20expected message: `{:?}`",
- message,
- expected2,
- );
- }
- Ok(_) => panic!("test did not panic as expected"),
- }
+ Ident::new("'a#", Span::call_site());
}
#[test]
@@ -114,6 +99,13 @@ fn literal_string() {
assert_eq!(Literal::string("foo").to_string(), "\"foo\"");
assert_eq!(Literal::string("\"").to_string(), "\"\\\"\"");
assert_eq!(Literal::string("didn't").to_string(), "\"didn't\"");
+ assert_eq!(
+ Literal::string("a\00b\07c\08d\0e\0").to_string(),
+ "\"a\\x000b\\x007c\\08d\\0e\\0\"",
+ );
+
+ "\"\\\r\n x\"".parse::<TokenStream>().unwrap();
+ "\"\\\r\n \rx\"".parse::<TokenStream>().unwrap_err();
}
#[test]
@@ -147,6 +139,51 @@ fn literal_byte_string() {
Literal::byte_string(b"\0\t\n\r\"\\2\x10").to_string(),
"b\"\\0\\t\\n\\r\\\"\\\\2\\x10\"",
);
+ assert_eq!(
+ Literal::byte_string(b"a\00b\07c\08d\0e\0").to_string(),
+ "b\"a\\x000b\\x007c\\08d\\0e\\0\"",
+ );
+
+ "b\"\\\r\n x\"".parse::<TokenStream>().unwrap();
+ "b\"\\\r\n \rx\"".parse::<TokenStream>().unwrap_err();
+ "b\"\\\r\n \u{a0}x\"".parse::<TokenStream>().unwrap_err();
+ "br\"\u{a0}\"".parse::<TokenStream>().unwrap_err();
+}
+
+#[test]
+fn literal_c_string() {
+ let strings = r###"
+ c"hello\x80我叫\u{1F980}" // from the RFC
+ cr"\"
+ cr##"Hello "world"!"##
+ c"\t\n\r\"\\"
+ "###;
+
+ let mut tokens = strings.parse::<TokenStream>().unwrap().into_iter();
+
+ for expected in &[
+ r#"c"hello\x80我叫\u{1F980}""#,
+ r#"cr"\""#,
+ r###"cr##"Hello "world"!"##"###,
+ r#"c"\t\n\r\"\\""#,
+ ] {
+ match tokens.next().unwrap() {
+ TokenTree::Literal(literal) => {
+ assert_eq!(literal.to_string(), *expected);
+ }
+ unexpected => panic!("unexpected token: {:?}", unexpected),
+ }
+ }
+
+ if let Some(unexpected) = tokens.next() {
+ panic!("unexpected token: {:?}", unexpected);
+ }
+
+ for invalid in &[r#"c"\0""#, r#"c"\x00""#, r#"c"\u{0}""#, "c\"\0\""] {
+ if let Ok(unexpected) = invalid.parse::<TokenStream>() {
+ panic!("unexpected token: {:?}", unexpected);
+ }
+ }
}
#[test]
@@ -265,6 +302,48 @@ fn literal_parse() {
}
#[test]
+fn literal_span() {
+ let positive = "0.1".parse::<Literal>().unwrap();
+ let negative = "-0.1".parse::<Literal>().unwrap();
+ let subspan = positive.subspan(1..2);
+
+ #[cfg(not(span_locations))]
+ {
+ let _ = negative;
+ assert!(subspan.is_none());
+ }
+
+ #[cfg(span_locations)]
+ {
+ assert_eq!(positive.span().start().column, 0);
+ assert_eq!(positive.span().end().column, 3);
+ assert_eq!(negative.span().start().column, 0);
+ assert_eq!(negative.span().end().column, 4);
+ assert_eq!(subspan.unwrap().source_text().unwrap(), ".");
+ }
+
+ assert!(positive.subspan(1..4).is_none());
+}
+
+#[cfg(span_locations)]
+#[test]
+fn source_text() {
+ let input = " 𓀕 a z ";
+ let mut tokens = input
+ .parse::<proc_macro2::TokenStream>()
+ .unwrap()
+ .into_iter();
+
+ let first = tokens.next().unwrap();
+ assert_eq!("𓀕", first.span().source_text().unwrap());
+
+ let second = tokens.next().unwrap();
+ let third = tokens.next().unwrap();
+ assert_eq!("z", third.span().source_text().unwrap());
+ assert_eq!("a", second.span().source_text().unwrap());
+}
+
+#[test]
fn roundtrip() {
fn roundtrip(p: &str) {
println!("parse: {}", p);
@@ -603,8 +682,8 @@ fn non_ascii_tokens() {
check_spans("/*** ábc */ x", &[(1, 12, 1, 13)]);
check_spans(r#""abc""#, &[(1, 0, 1, 5)]);
check_spans(r#""ábc""#, &[(1, 0, 1, 5)]);
- check_spans(r###"r#"abc"#"###, &[(1, 0, 1, 8)]);
- check_spans(r###"r#"ábc"#"###, &[(1, 0, 1, 8)]);
+ check_spans(r##"r#"abc"#"##, &[(1, 0, 1, 8)]);
+ check_spans(r##"r#"ábc"#"##, &[(1, 0, 1, 8)]);
check_spans("r#\"a\nc\"#", &[(1, 0, 2, 3)]);
check_spans("r#\"á\nc\"#", &[(1, 0, 2, 3)]);
check_spans("'a'", &[(1, 0, 1, 3)]);
@@ -624,7 +703,6 @@ fn non_ascii_tokens() {
check_spans("ábc// foo", &[(1, 0, 1, 3)]);
check_spans("ábć// foo", &[(1, 0, 1, 3)]);
check_spans("b\"a\\\n c\"", &[(1, 0, 2, 3)]);
- check_spans("b\"a\\\n\u{00a0}c\"", &[(1, 0, 2, 3)]);
}
#[cfg(span_locations)]
@@ -656,6 +734,18 @@ fn check_spans_internal(ts: TokenStream, lines: &mut &[(usize, usize, usize, usi
}
#[test]
+fn whitespace() {
+ // space, horizontal tab, vertical tab, form feed, carriage return, line
+ // feed, non-breaking space, left-to-right mark, right-to-left mark
+ let various_spaces = " \t\u{b}\u{c}\r\n\u{a0}\u{200e}\u{200f}";
+ let tokens = various_spaces.parse::<TokenStream>().unwrap();
+ assert_eq!(tokens.into_iter().count(), 0);
+
+ let lone_carriage_returns = " \r \r\r\n ";
+ lone_carriage_returns.parse::<TokenStream>().unwrap();
+}
+
+#[test]
fn byte_order_mark() {
let string = "\u{feff}foo";
let tokens = string.parse::<TokenStream>().unwrap();
diff --git a/tests/test_fmt.rs b/tests/test_fmt.rs
index 93dd19e..86a4c38 100644
--- a/tests/test_fmt.rs
+++ b/tests/test_fmt.rs
@@ -1,7 +1,7 @@
#![allow(clippy::from_iter_instead_of_collect)]
use proc_macro2::{Delimiter, Group, Ident, Span, TokenStream, TokenTree};
-use std::iter::{self, FromIterator};
+use std::iter;
#[test]
fn test_fmt_group() {
diff --git a/tests/test_size.rs b/tests/test_size.rs
new file mode 100644
index 0000000..46e58db
--- /dev/null
+++ b/tests/test_size.rs
@@ -0,0 +1,42 @@
+extern crate proc_macro;
+
+use std::mem;
+
+#[rustversion::attr(before(1.32), ignore)]
+#[test]
+fn test_proc_macro_span_size() {
+ assert_eq!(mem::size_of::<proc_macro::Span>(), 4);
+ assert_eq!(mem::size_of::<Option<proc_macro::Span>>(), 4);
+}
+
+#[cfg_attr(not(all(not(wrap_proc_macro), not(span_locations))), ignore)]
+#[test]
+fn test_proc_macro2_fallback_span_size_without_locations() {
+ assert_eq!(mem::size_of::<proc_macro2::Span>(), 0);
+ assert_eq!(mem::size_of::<Option<proc_macro2::Span>>(), 1);
+}
+
+#[cfg_attr(not(all(not(wrap_proc_macro), span_locations)), ignore)]
+#[test]
+fn test_proc_macro2_fallback_span_size_with_locations() {
+ assert_eq!(mem::size_of::<proc_macro2::Span>(), 8);
+ assert_eq!(mem::size_of::<Option<proc_macro2::Span>>(), 12);
+}
+
+#[rustversion::attr(before(1.32), ignore)]
+#[rustversion::attr(
+ since(1.32),
+ cfg_attr(not(all(wrap_proc_macro, not(span_locations))), ignore)
+)]
+#[test]
+fn test_proc_macro2_wrapper_span_size_without_locations() {
+ assert_eq!(mem::size_of::<proc_macro2::Span>(), 4);
+ assert_eq!(mem::size_of::<Option<proc_macro2::Span>>(), 8);
+}
+
+#[cfg_attr(not(all(wrap_proc_macro, span_locations)), ignore)]
+#[test]
+fn test_proc_macro2_wrapper_span_size_with_locations() {
+ assert_eq!(mem::size_of::<proc_macro2::Span>(), 12);
+ assert_eq!(mem::size_of::<Option<proc_macro2::Span>>(), 12);
+}