diff options
author | Chih-Hung Hsieh <chh@google.com> | 2020-10-26 22:45:07 +0000 |
---|---|---|
committer | Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com> | 2020-10-26 22:45:07 +0000 |
commit | 24ea1276c63491bd19e06aa0939cb34a7b7bd121 (patch) | |
tree | c65f0c49ad462a62aff46c4f7d200cdd1423c384 | |
parent | c6ca8f74f323c74aa90729d37e00babda3131f8c (diff) | |
parent | 44526bd183933881e036ccf1f2c8751a0a56c406 (diff) | |
download | paste-24ea1276c63491bd19e06aa0939cb34a7b7bd121.tar.gz |
Upgrade rust/crates/paste to 1.0.2 am: 44526bd183
Original change: https://android-review.googlesource.com/c/platform/external/rust/crates/paste/+/1474896
Change-Id: Icd89093e921841c548cc90bc226c40c6e9b912fd
-rw-r--r-- | .cargo_vcs_info.json | 2 | ||||
-rw-r--r-- | Cargo.toml | 2 | ||||
-rw-r--r-- | Cargo.toml.orig | 2 | ||||
-rw-r--r-- | METADATA | 8 | ||||
-rw-r--r-- | src/doc.rs | 72 | ||||
-rw-r--r-- | src/lib.rs | 262 | ||||
-rw-r--r-- | src/segment.rs | 233 | ||||
-rw-r--r-- | tests/test_doc.rs | 10 |
8 files changed, 317 insertions, 274 deletions
diff --git a/.cargo_vcs_info.json b/.cargo_vcs_info.json index 18adbe1..8b0f57b 100644 --- a/.cargo_vcs_info.json +++ b/.cargo_vcs_info.json @@ -1,5 +1,5 @@ { "git": { - "sha1": "ead8998a76e6b28a0ade8574490e18f7bb52877b" + "sha1": "6a5265f7a937412fb1da72fb72fd32bbaffecebc" } } @@ -13,7 +13,7 @@ [package] edition = "2018" name = "paste" -version = "1.0.1" +version = "1.0.2" authors = ["David Tolnay <dtolnay@gmail.com>"] description = "Macros for all your token pasting needs" readme = "README.md" diff --git a/Cargo.toml.orig b/Cargo.toml.orig index 5365865..42aee90 100644 --- a/Cargo.toml.orig +++ b/Cargo.toml.orig @@ -1,6 +1,6 @@ [package] name = "paste" -version = "1.0.1" +version = "1.0.2" authors = ["David Tolnay <dtolnay@gmail.com>"] edition = "2018" license = "MIT OR Apache-2.0" @@ -7,13 +7,13 @@ third_party { } url { type: ARCHIVE - value: "https://static.crates.io/crates/paste/paste-1.0.1.crate" + value: "https://static.crates.io/crates/paste/paste-1.0.2.crate" } - version: "1.0.1" + version: "1.0.2" license_type: NOTICE last_upgrade_date { year: 2020 - month: 9 - day: 15 + month: 10 + day: 26 } } @@ -1,5 +1,8 @@ +use crate::error::Result; +use crate::segment::{self, Segment}; use proc_macro::{Delimiter, Span, TokenStream, TokenTree}; use std::iter; +use std::mem; use std::str::FromStr; pub fn is_pasted_doc(input: &TokenStream) -> bool { @@ -26,16 +29,33 @@ pub fn is_pasted_doc(input: &TokenStream) -> bool { state == State::Rest } -pub fn do_paste_doc(attr: &TokenStream, span: Span) -> TokenStream { +pub fn do_paste_doc(attr: &TokenStream, span: Span) -> Result<TokenStream> { let mut expanded = TokenStream::new(); - let mut tokens = attr.clone().into_iter(); + let mut tokens = attr.clone().into_iter().peekable(); expanded.extend(tokens.by_ref().take(2)); // `doc =` - let mut lit = String::new(); - lit.push('"'); - for token in tokens { - lit += &escaped_string_value(&token).unwrap(); + let mut segments = segment::parse(&mut tokens)?; + + for segment in &mut segments { + if let Segment::String(string) = segment { + if let Some(open_quote) = string.value.find('"') { + if open_quote == 0 { + string.value.truncate(string.value.len() - 1); + string.value.remove(0); + } else { + let begin = open_quote + 1; + let end = string.value.rfind('"').unwrap(); + let raw_string = mem::replace(&mut string.value, String::new()); + for ch in raw_string[begin..end].chars() { + string.value.extend(ch.escape_default()); + } + } + } + } } + + let mut lit = segment::paste(&segments)?; + lit.insert(0, '"'); lit.push('"'); let mut lit = TokenStream::from_str(&lit) @@ -45,48 +65,26 @@ pub fn do_paste_doc(attr: &TokenStream, span: Span) -> TokenStream { .unwrap(); lit.set_span(span); expanded.extend(iter::once(lit)); - expanded + Ok(expanded) } fn is_stringlike(token: &TokenTree) -> bool { - escaped_string_value(token).is_some() -} - -fn escaped_string_value(token: &TokenTree) -> Option<String> { match token { - TokenTree::Ident(ident) => Some(ident.to_string()), + TokenTree::Ident(_) => true, TokenTree::Literal(literal) => { - let mut repr = literal.to_string(); - if repr.starts_with('b') || repr.starts_with('\'') { - None - } else if repr.starts_with('"') { - repr.truncate(repr.len() - 1); - repr.remove(0); - Some(repr) - } else if repr.starts_with('r') { - let begin = repr.find('"').unwrap() + 1; - let end = repr.rfind('"').unwrap(); - let mut escaped = String::new(); - for ch in repr[begin..end].chars() { - escaped.extend(ch.escape_default()); - } - Some(escaped) - } else { - Some(repr) - } + let repr = literal.to_string(); + !repr.starts_with('b') && !repr.starts_with('\'') } TokenTree::Group(group) => { if group.delimiter() != Delimiter::None { - return None; + return false; } let mut inner = group.stream().into_iter(); - let first = inner.next()?; - if inner.next().is_none() { - escaped_string_value(&first) - } else { - None + match inner.next() { + Some(first) => inner.next().is_none() && is_stringlike(&first), + None => false, } } - TokenTree::Punct(_) => None, + TokenTree::Punct(punct) => punct.as_char() == '\'' || punct.as_char() == ':', } } @@ -142,13 +142,13 @@ extern crate proc_macro; mod doc; mod error; +mod segment; use crate::doc::{do_paste_doc, is_pasted_doc}; use crate::error::{Error, Result}; -use proc_macro::{ - token_stream, Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree, -}; -use std::iter::{self, FromIterator, Peekable}; +use crate::segment::Segment; +use proc_macro::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree}; +use std::iter; use std::panic; #[proc_macro] @@ -199,8 +199,9 @@ fn expand(input: TokenStream, contains_paste: &mut bool) -> Result<TokenStream> let span = group.span(); if delimiter == Delimiter::Bracket && is_paste_operation(&content) { let segments = parse_bracket_as_segments(content, span)?; - let pasted = paste_segments(span, &segments)?; - expanded.extend(pasted); + let pasted = segment::paste(&segments)?; + let tokens = pasted_to_tokens(pasted, span)?; + expanded.extend(tokens); *contains_paste = true; } else if delimiter == Delimiter::None && is_flat_group(&content) { expanded.extend(content); @@ -209,7 +210,7 @@ fn expand(input: TokenStream, contains_paste: &mut bool) -> Result<TokenStream> && (lookbehind == Lookbehind::Pound || lookbehind == Lookbehind::PoundBang) && is_pasted_doc(&content) { - let pasted = do_paste_doc(&content, span); + let pasted = do_paste_doc(&content, span)?; let mut group = Group::new(delimiter, pasted); group.set_span(span); expanded.extend(iter::once(TokenTree::Group(group))); @@ -302,22 +303,6 @@ fn is_flat_group(input: &TokenStream) -> bool { state == State::Ident || state == State::Literal || state == State::Lifetime } -struct LitStr { - value: String, - span: Span, -} - -struct Colon { - span: Span, -} - -enum Segment { - String(String), - Apostrophe(Span), - Env(LitStr), - Modifier(Colon, Ident), -} - fn is_paste_operation(input: &TokenStream) -> bool { let mut tokens = input.clone().into_iter(); @@ -347,7 +332,7 @@ fn parse_bracket_as_segments(input: TokenStream, scope: Span) -> Result<Vec<Segm None => return Err(Error::new(scope, "expected `[< ... >]`")), } - let segments = parse_segments(&mut tokens, scope)?; + let mut segments = segment::parse(&mut tokens)?; match &tokens.next() { Some(TokenTree::Punct(punct)) if punct.as_char() == '>' => {} @@ -355,218 +340,39 @@ fn parse_bracket_as_segments(input: TokenStream, scope: Span) -> Result<Vec<Segm None => return Err(Error::new(scope, "expected `[< ... >]`")), } - match tokens.next() { - Some(unexpected) => Err(Error::new( + if let Some(unexpected) = tokens.next() { + return Err(Error::new( unexpected.span(), "unexpected input, expected `[< ... >]`", - )), - None => Ok(segments), + )); } -} -fn parse_segments( - tokens: &mut Peekable<token_stream::IntoIter>, - scope: Span, -) -> Result<Vec<Segment>> { - let mut segments = Vec::new(); - while match tokens.peek() { - None => false, - Some(TokenTree::Punct(punct)) => punct.as_char() != '>', - Some(_) => true, - } { - match tokens.next().unwrap() { - TokenTree::Ident(ident) => { - let mut fragment = ident.to_string(); - if fragment.starts_with("r#") { - fragment = fragment.split_off(2); - } - if fragment == "env" - && match tokens.peek() { - Some(TokenTree::Punct(punct)) => punct.as_char() == '!', - _ => false, - } - { - tokens.next().unwrap(); // `!` - let expect_group = tokens.next(); - let parenthesized = match &expect_group { - Some(TokenTree::Group(group)) - if group.delimiter() == Delimiter::Parenthesis => - { - group - } - Some(wrong) => return Err(Error::new(wrong.span(), "expected `(`")), - None => return Err(Error::new(scope, "expected `(` after `env!`")), - }; - let mut inner = parenthesized.stream().into_iter(); - let lit = match inner.next() { - Some(TokenTree::Literal(lit)) => lit, - Some(wrong) => { - return Err(Error::new(wrong.span(), "expected string literal")) - } - None => { - return Err(Error::new2( - ident.span(), - parenthesized.span(), - "expected string literal as argument to env! macro", - )) - } - }; - let lit_string = lit.to_string(); - if lit_string.starts_with('"') - && lit_string.ends_with('"') - && lit_string.len() >= 2 - { - // TODO: maybe handle escape sequences in the string if - // someone has a use case. - segments.push(Segment::Env(LitStr { - value: lit_string[1..lit_string.len() - 1].to_owned(), - span: lit.span(), - })); - } else { - return Err(Error::new(lit.span(), "expected string literal")); - } - if let Some(unexpected) = inner.next() { - return Err(Error::new( - unexpected.span(), - "unexpected token in env! macro", - )); - } - } else { - segments.push(Segment::String(fragment)); - } - } - TokenTree::Literal(lit) => { - let mut lit_string = lit.to_string(); - if lit_string.contains(&['#', '\\', '.', '+'][..]) { - return Err(Error::new(lit.span(), "unsupported literal")); - } - lit_string = lit_string - .replace('"', "") - .replace('\'', "") - .replace('-', "_"); - segments.push(Segment::String(lit_string)); - } - TokenTree::Punct(punct) => match punct.as_char() { - '_' => segments.push(Segment::String("_".to_owned())), - '\'' => segments.push(Segment::Apostrophe(punct.span())), - ':' => { - let colon = Colon { span: punct.span() }; - let ident = match tokens.next() { - Some(TokenTree::Ident(ident)) => ident, - wrong => { - let span = wrong.as_ref().map_or(scope, TokenTree::span); - return Err(Error::new(span, "expected identifier after `:`")); - } - }; - segments.push(Segment::Modifier(colon, ident)); - } - _ => return Err(Error::new(punct.span(), "unexpected punct")), - }, - TokenTree::Group(group) => { - if group.delimiter() == Delimiter::None { - let mut inner = group.stream().into_iter().peekable(); - let nested = parse_segments(&mut inner, group.span())?; - if let Some(unexpected) = inner.next() { - return Err(Error::new(unexpected.span(), "unexpected token")); - } - segments.extend(nested); - } else { - return Err(Error::new(group.span(), "unexpected token")); - } + for segment in &mut segments { + if let Segment::String(string) = segment { + if string.value.contains(&['#', '\\', '.', '+'][..]) { + return Err(Error::new(string.span, "unsupported literal")); } + string.value = string + .value + .replace('"', "") + .replace('\'', "") + .replace('-', "_"); } } + Ok(segments) } -fn paste_segments(span: Span, segments: &[Segment]) -> Result<TokenStream> { - let mut evaluated = Vec::new(); - let mut is_lifetime = false; +fn pasted_to_tokens(mut pasted: String, span: Span) -> Result<TokenStream> { + let mut tokens = TokenStream::new(); - for segment in segments { - match segment { - Segment::String(segment) => { - evaluated.push(segment.clone()); - } - Segment::Apostrophe(span) => { - if is_lifetime { - return Err(Error::new(*span, "unexpected lifetime")); - } - is_lifetime = true; - } - Segment::Env(var) => { - let resolved = match std::env::var(&var.value) { - Ok(resolved) => resolved, - Err(_) => { - return Err(Error::new( - var.span, - &format!("no such env var: {:?}", var.value), - )); - } - }; - let resolved = resolved.replace('-', "_"); - evaluated.push(resolved); - } - Segment::Modifier(colon, ident) => { - let last = match evaluated.pop() { - Some(last) => last, - None => { - return Err(Error::new2(colon.span, ident.span(), "unexpected modifier")) - } - }; - match ident.to_string().as_str() { - "lower" => { - evaluated.push(last.to_lowercase()); - } - "upper" => { - evaluated.push(last.to_uppercase()); - } - "snake" => { - let mut acc = String::new(); - let mut prev = '_'; - for ch in last.chars() { - if ch.is_uppercase() && prev != '_' { - acc.push('_'); - } - acc.push(ch); - prev = ch; - } - evaluated.push(acc.to_lowercase()); - } - "camel" => { - let mut acc = String::new(); - let mut prev = '_'; - for ch in last.chars() { - if ch != '_' { - if prev == '_' { - for chu in ch.to_uppercase() { - acc.push(chu); - } - } else if prev.is_uppercase() { - for chl in ch.to_lowercase() { - acc.push(chl); - } - } else { - acc.push(ch); - } - } - prev = ch; - } - evaluated.push(acc); - } - _ => { - return Err(Error::new2( - colon.span, - ident.span(), - "unsupported modifier", - )); - } - } - } - } + if pasted.starts_with('\'') { + let mut apostrophe = TokenTree::Punct(Punct::new('\'', Spacing::Joint)); + apostrophe.set_span(span); + tokens.extend(iter::once(apostrophe)); + pasted.remove(0); } - let pasted = evaluated.into_iter().collect::<String>(); let ident = match panic::catch_unwind(|| Ident::new(&pasted, span)) { Ok(ident) => TokenTree::Ident(ident), Err(_) => { @@ -576,11 +382,7 @@ fn paste_segments(span: Span, segments: &[Segment]) -> Result<TokenStream> { )); } }; - let tokens = if is_lifetime { - let apostrophe = TokenTree::Punct(Punct::new('\'', Spacing::Joint)); - vec![apostrophe, ident] - } else { - vec![ident] - }; - Ok(TokenStream::from_iter(tokens)) + + tokens.extend(iter::once(ident)); + Ok(tokens) } diff --git a/src/segment.rs b/src/segment.rs new file mode 100644 index 0000000..592a047 --- /dev/null +++ b/src/segment.rs @@ -0,0 +1,233 @@ +use crate::error::{Error, Result}; +use proc_macro::{token_stream, Delimiter, Ident, Span, TokenTree}; +use std::iter::Peekable; + +pub(crate) enum Segment { + String(LitStr), + Apostrophe(Span), + Env(LitStr), + Modifier(Colon, Ident), +} + +pub(crate) struct LitStr { + pub value: String, + pub span: Span, +} + +pub(crate) struct Colon { + pub span: Span, +} + +pub(crate) fn parse(tokens: &mut Peekable<token_stream::IntoIter>) -> Result<Vec<Segment>> { + let mut segments = Vec::new(); + while match tokens.peek() { + None => false, + Some(TokenTree::Punct(punct)) => punct.as_char() != '>', + Some(_) => true, + } { + match tokens.next().unwrap() { + TokenTree::Ident(ident) => { + let mut fragment = ident.to_string(); + if fragment.starts_with("r#") { + fragment = fragment.split_off(2); + } + if fragment == "env" + && match tokens.peek() { + Some(TokenTree::Punct(punct)) => punct.as_char() == '!', + _ => false, + } + { + let bang = tokens.next().unwrap(); // `!` + let expect_group = tokens.next(); + let parenthesized = match &expect_group { + Some(TokenTree::Group(group)) + if group.delimiter() == Delimiter::Parenthesis => + { + group + } + Some(wrong) => return Err(Error::new(wrong.span(), "expected `(`")), + None => { + return Err(Error::new2( + ident.span(), + bang.span(), + "expected `(` after `env!`", + )); + } + }; + let mut inner = parenthesized.stream().into_iter(); + let lit = match inner.next() { + Some(TokenTree::Literal(lit)) => lit, + Some(wrong) => { + return Err(Error::new(wrong.span(), "expected string literal")) + } + None => { + return Err(Error::new2( + ident.span(), + parenthesized.span(), + "expected string literal as argument to env! macro", + )) + } + }; + let lit_string = lit.to_string(); + if lit_string.starts_with('"') + && lit_string.ends_with('"') + && lit_string.len() >= 2 + { + // TODO: maybe handle escape sequences in the string if + // someone has a use case. + segments.push(Segment::Env(LitStr { + value: lit_string[1..lit_string.len() - 1].to_owned(), + span: lit.span(), + })); + } else { + return Err(Error::new(lit.span(), "expected string literal")); + } + if let Some(unexpected) = inner.next() { + return Err(Error::new( + unexpected.span(), + "unexpected token in env! macro", + )); + } + } else { + segments.push(Segment::String(LitStr { + value: fragment, + span: ident.span(), + })); + } + } + TokenTree::Literal(lit) => { + segments.push(Segment::String(LitStr { + value: lit.to_string(), + span: lit.span(), + })); + } + TokenTree::Punct(punct) => match punct.as_char() { + '_' => segments.push(Segment::String(LitStr { + value: "_".to_owned(), + span: punct.span(), + })), + '\'' => segments.push(Segment::Apostrophe(punct.span())), + ':' => { + let colon_span = punct.span(); + let colon = Colon { span: colon_span }; + let ident = match tokens.next() { + Some(TokenTree::Ident(ident)) => ident, + wrong => { + let span = wrong.as_ref().map_or(colon_span, TokenTree::span); + return Err(Error::new(span, "expected identifier after `:`")); + } + }; + segments.push(Segment::Modifier(colon, ident)); + } + _ => return Err(Error::new(punct.span(), "unexpected punct")), + }, + TokenTree::Group(group) => { + if group.delimiter() == Delimiter::None { + let mut inner = group.stream().into_iter().peekable(); + let nested = parse(&mut inner)?; + if let Some(unexpected) = inner.next() { + return Err(Error::new(unexpected.span(), "unexpected token")); + } + segments.extend(nested); + } else { + return Err(Error::new(group.span(), "unexpected token")); + } + } + } + } + Ok(segments) +} + +pub(crate) fn paste(segments: &[Segment]) -> Result<String> { + let mut evaluated = Vec::new(); + let mut is_lifetime = false; + + for segment in segments { + match segment { + Segment::String(segment) => { + evaluated.push(segment.value.clone()); + } + Segment::Apostrophe(span) => { + if is_lifetime { + return Err(Error::new(*span, "unexpected lifetime")); + } + is_lifetime = true; + } + Segment::Env(var) => { + let resolved = match std::env::var(&var.value) { + Ok(resolved) => resolved, + Err(_) => { + return Err(Error::new( + var.span, + &format!("no such env var: {:?}", var.value), + )); + } + }; + let resolved = resolved.replace('-', "_"); + evaluated.push(resolved); + } + Segment::Modifier(colon, ident) => { + let last = match evaluated.pop() { + Some(last) => last, + None => { + return Err(Error::new2(colon.span, ident.span(), "unexpected modifier")) + } + }; + match ident.to_string().as_str() { + "lower" => { + evaluated.push(last.to_lowercase()); + } + "upper" => { + evaluated.push(last.to_uppercase()); + } + "snake" => { + let mut acc = String::new(); + let mut prev = '_'; + for ch in last.chars() { + if ch.is_uppercase() && prev != '_' { + acc.push('_'); + } + acc.push(ch); + prev = ch; + } + evaluated.push(acc.to_lowercase()); + } + "camel" => { + let mut acc = String::new(); + let mut prev = '_'; + for ch in last.chars() { + if ch != '_' { + if prev == '_' { + for chu in ch.to_uppercase() { + acc.push(chu); + } + } else if prev.is_uppercase() { + for chl in ch.to_lowercase() { + acc.push(chl); + } + } else { + acc.push(ch); + } + } + prev = ch; + } + evaluated.push(acc); + } + _ => { + return Err(Error::new2( + colon.span, + ident.span(), + "unsupported modifier", + )); + } + } + } + } + } + + let mut pasted = evaluated.into_iter().collect::<String>(); + if is_lifetime { + pasted.insert(0, '\''); + } + Ok(pasted) +} diff --git a/tests/test_doc.rs b/tests/test_doc.rs index 96fe3a0..1ceaf23 100644 --- a/tests/test_doc.rs +++ b/tests/test_doc.rs @@ -42,3 +42,13 @@ fn test_literals() { let expected = "int=0x1 bool=true float=0.01"; assert_eq!(doc, expected); } + +#[test] +fn test_case() { + let doc = paste! { + get_doc!(#[doc = "HTTP " get:upper "!"]) + }; + + let expected = "HTTP GET!"; + assert_eq!(doc, expected); +} |