From 16b5ac111c1299a4ba9324104eb1cba22d54383b Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 29 Aug 2025 14:26:44 +1000 Subject: [PATCH 1/2] Remove very outdated comment about token streams. They're now just an `Arc>`. No ropes, no views, nothing like that. --- compiler/rustc_ast/src/tokenstream.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index f4f35a4d2ee07..f9bac42ada38f 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -3,15 +3,6 @@ //! `TokenStream`s represent syntactic objects before they are converted into ASTs. //! A `TokenStream` is, roughly speaking, a sequence of [`TokenTree`]s, //! which are themselves a single [`Token`] or a `Delimited` subsequence of tokens. -//! -//! ## Ownership -//! -//! `TokenStream`s are persistent data structures constructed as ropes with reference -//! counted-children. In general, this means that calling an operation on a `TokenStream` -//! (such as `slice`) produces an entirely new `TokenStream` from the borrowed reference to -//! the original. This essentially coerces `TokenStream`s into "views" of their subparts, -//! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking -//! ownership of the original. use std::borrow::Cow; use std::ops::Range; From 364a3be579bbc6a9a06b3d32e84748382e1001ae Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 29 Aug 2025 14:25:40 +1000 Subject: [PATCH 2/2] Put `TokenStream` stuff in a sensible order. I.e. the type definition, then a single inherent `impl` block, then the trait `impl` blocks. The lack of sensible ordering here has bugged me for some time. --- compiler/rustc_ast/src/tokenstream.rs | 132 +++++++++++++------------- 1 file changed, 65 insertions(+), 67 deletions(-) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index f9bac42ada38f..a5d8fbfac612a 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -90,17 +90,6 @@ impl TokenTree { } } -impl HashStable for TokenStream -where - CTX: crate::HashStableContext, -{ - fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { - for sub_tt in self.iter() { - sub_tt.hash_stable(hcx, hasher); - } - } -} - /// A lazy version of [`AttrTokenStream`], which defers creation of an actual /// `AttrTokenStream` until it is needed. #[derive(Clone)] @@ -547,10 +536,6 @@ pub struct AttrsTarget { pub tokens: LazyAttrTokenStream, } -/// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s. -#[derive(Clone, Debug, Default, Encodable, Decodable)] -pub struct TokenStream(pub(crate) Arc>); - /// Indicates whether a token can join with the following token to form a /// compound token. Used for conversions to `proc_macro::Spacing`. Also used to /// guide pretty-printing, which is where the `JointHidden` value (which isn't @@ -611,58 +596,9 @@ pub enum Spacing { JointHidden, } -impl TokenStream { - /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream` - /// separating the two arguments with a comma for diagnostic suggestions. - pub fn add_comma(&self) -> Option<(TokenStream, Span)> { - // Used to suggest if a user writes `foo!(a b);` - let mut suggestion = None; - let mut iter = self.0.iter().enumerate().peekable(); - while let Some((pos, ts)) = iter.next() { - if let Some((_, next)) = iter.peek() { - let sp = match (&ts, &next) { - (_, TokenTree::Token(Token { kind: token::Comma, .. }, _)) => continue, - ( - TokenTree::Token(token_left, Spacing::Alone), - TokenTree::Token(token_right, _), - ) if (token_left.is_non_reserved_ident() || token_left.is_lit()) - && (token_right.is_non_reserved_ident() || token_right.is_lit()) => - { - token_left.span - } - (TokenTree::Delimited(sp, ..), _) => sp.entire(), - _ => continue, - }; - let sp = sp.shrink_to_hi(); - let comma = TokenTree::token_alone(token::Comma, sp); - suggestion = Some((pos, comma, sp)); - } - } - if let Some((pos, comma, sp)) = suggestion { - let mut new_stream = Vec::with_capacity(self.0.len() + 1); - let parts = self.0.split_at(pos + 1); - new_stream.extend_from_slice(parts.0); - new_stream.push(comma); - new_stream.extend_from_slice(parts.1); - return Some((TokenStream::new(new_stream), sp)); - } - None - } -} - -impl FromIterator for TokenStream { - fn from_iter>(iter: I) -> Self { - TokenStream::new(iter.into_iter().collect::>()) - } -} - -impl Eq for TokenStream {} - -impl PartialEq for TokenStream { - fn eq(&self, other: &TokenStream) -> bool { - self.iter().eq(other.iter()) - } -} +/// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s. +#[derive(Clone, Debug, Default, Encodable, Decodable)] +pub struct TokenStream(pub(crate) Arc>); impl TokenStream { pub fn new(tts: Vec) -> TokenStream { @@ -838,6 +774,68 @@ impl TokenStream { } } } + + /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream` + /// separating the two arguments with a comma for diagnostic suggestions. + pub fn add_comma(&self) -> Option<(TokenStream, Span)> { + // Used to suggest if a user writes `foo!(a b);` + let mut suggestion = None; + let mut iter = self.0.iter().enumerate().peekable(); + while let Some((pos, ts)) = iter.next() { + if let Some((_, next)) = iter.peek() { + let sp = match (&ts, &next) { + (_, TokenTree::Token(Token { kind: token::Comma, .. }, _)) => continue, + ( + TokenTree::Token(token_left, Spacing::Alone), + TokenTree::Token(token_right, _), + ) if (token_left.is_non_reserved_ident() || token_left.is_lit()) + && (token_right.is_non_reserved_ident() || token_right.is_lit()) => + { + token_left.span + } + (TokenTree::Delimited(sp, ..), _) => sp.entire(), + _ => continue, + }; + let sp = sp.shrink_to_hi(); + let comma = TokenTree::token_alone(token::Comma, sp); + suggestion = Some((pos, comma, sp)); + } + } + if let Some((pos, comma, sp)) = suggestion { + let mut new_stream = Vec::with_capacity(self.0.len() + 1); + let parts = self.0.split_at(pos + 1); + new_stream.extend_from_slice(parts.0); + new_stream.push(comma); + new_stream.extend_from_slice(parts.1); + return Some((TokenStream::new(new_stream), sp)); + } + None + } +} + +impl PartialEq for TokenStream { + fn eq(&self, other: &TokenStream) -> bool { + self.iter().eq(other.iter()) + } +} + +impl Eq for TokenStream {} + +impl FromIterator for TokenStream { + fn from_iter>(iter: I) -> Self { + TokenStream::new(iter.into_iter().collect::>()) + } +} + +impl HashStable for TokenStream +where + CTX: crate::HashStableContext, +{ + fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { + for sub_tt in self.iter() { + sub_tt.hash_stable(hcx, hasher); + } + } } #[derive(Clone)]