rustc_parse/
lib.rs

1//! The main parser interface.
2
3// tidy-alphabetical-start
4#![allow(internal_features)]
5#![allow(rustc::diagnostic_outside_of_impl)]
6#![allow(rustc::untranslatable_diagnostic)]
7#![feature(assert_matches)]
8#![feature(box_patterns)]
9#![feature(debug_closure_helpers)]
10#![feature(if_let_guard)]
11#![feature(iter_intersperse)]
12#![recursion_limit = "256"]
13// tidy-alphabetical-end
14
15use std::path::{Path, PathBuf};
16use std::str::Utf8Error;
17use std::sync::Arc;
18
19use rustc_ast as ast;
20use rustc_ast::tokenstream::TokenStream;
21use rustc_ast::{AttrItem, Attribute, MetaItemInner, token};
22use rustc_ast_pretty::pprust;
23use rustc_errors::{Diag, EmissionGuarantee, FatalError, PResult, pluralize};
24use rustc_session::parse::ParseSess;
25use rustc_span::source_map::SourceMap;
26use rustc_span::{FileName, SourceFile, Span};
27pub use unicode_normalization::UNICODE_VERSION as UNICODE_NORMALIZATION_VERSION;
28
29pub const MACRO_ARGUMENTS: Option<&str> = Some("macro arguments");
30
31#[macro_use]
32pub mod parser;
33use parser::Parser;
34pub mod lexer;
35pub mod validate_attr;
36
37mod errors;
38
39rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
40
41// Unwrap the result if `Ok`, otherwise emit the diagnostics and abort.
42pub fn unwrap_or_emit_fatal<T>(expr: Result<T, Vec<Diag<'_>>>) -> T {
43    match expr {
44        Ok(expr) => expr,
45        Err(errs) => {
46            for err in errs {
47                err.emit();
48            }
49            FatalError.raise()
50        }
51    }
52}
53
54/// Creates a new parser from a source string. On failure, the errors must be consumed via
55/// `unwrap_or_emit_fatal`, `emit`, `cancel`, etc., otherwise a panic will occur when they are
56/// dropped.
57pub fn new_parser_from_source_str(
58    psess: &ParseSess,
59    name: FileName,
60    source: String,
61) -> Result<Parser<'_>, Vec<Diag<'_>>> {
62    let source_file = psess.source_map().new_source_file(name, source);
63    new_parser_from_source_file(psess, source_file)
64}
65
66/// Creates a new parser from a filename. On failure, the errors must be consumed via
67/// `unwrap_or_emit_fatal`, `emit`, `cancel`, etc., otherwise a panic will occur when they are
68/// dropped.
69///
70/// If a span is given, that is used on an error as the source of the problem.
71pub fn new_parser_from_file<'a>(
72    psess: &'a ParseSess,
73    path: &Path,
74    sp: Option<Span>,
75) -> Result<Parser<'a>, Vec<Diag<'a>>> {
76    let sm = psess.source_map();
77    let source_file = sm.load_file(path).unwrap_or_else(|e| {
78        let msg = format!("couldn't read `{}`: {}", path.display(), e);
79        let mut err = psess.dcx().struct_fatal(msg);
80        if let Ok(contents) = std::fs::read(path)
81            && let Err(utf8err) = String::from_utf8(contents.clone())
82        {
83            utf8_error(
84                sm,
85                &path.display().to_string(),
86                sp,
87                &mut err,
88                utf8err.utf8_error(),
89                &contents,
90            );
91        }
92        if let Some(sp) = sp {
93            err.span(sp);
94        }
95        err.emit();
96    });
97    new_parser_from_source_file(psess, source_file)
98}
99
100pub fn utf8_error<E: EmissionGuarantee>(
101    sm: &SourceMap,
102    path: &str,
103    sp: Option<Span>,
104    err: &mut Diag<'_, E>,
105    utf8err: Utf8Error,
106    contents: &[u8],
107) {
108    // The file exists, but it wasn't valid UTF-8.
109    let start = utf8err.valid_up_to();
110    let note = format!("invalid utf-8 at byte `{start}`");
111    let msg = if let Some(len) = utf8err.error_len() {
112        format!(
113            "byte{s} `{bytes}` {are} not valid utf-8",
114            bytes = if len == 1 {
115                format!("{:?}", contents[start])
116            } else {
117                format!("{:?}", &contents[start..start + len])
118            },
119            s = pluralize!(len),
120            are = if len == 1 { "is" } else { "are" },
121        )
122    } else {
123        note.clone()
124    };
125    let contents = String::from_utf8_lossy(contents).to_string();
126    let source = sm.new_source_file(PathBuf::from(path).into(), contents);
127    let span = Span::with_root_ctxt(
128        source.normalized_byte_pos(start as u32),
129        source.normalized_byte_pos(start as u32),
130    );
131    if span.is_dummy() {
132        err.note(note);
133    } else {
134        if sp.is_some() {
135            err.span_note(span, msg);
136        } else {
137            err.span(span);
138            err.span_label(span, msg);
139        }
140    }
141}
142
143/// Given a session and a `source_file`, return a parser. Returns any buffered errors from lexing
144/// the initial token stream.
145fn new_parser_from_source_file(
146    psess: &ParseSess,
147    source_file: Arc<SourceFile>,
148) -> Result<Parser<'_>, Vec<Diag<'_>>> {
149    let end_pos = source_file.end_position();
150    let stream = source_file_to_stream(psess, source_file, None)?;
151    let mut parser = Parser::new(psess, stream, None);
152    if parser.token == token::Eof {
153        parser.token.span = Span::new(end_pos, end_pos, parser.token.span.ctxt(), None);
154    }
155    Ok(parser)
156}
157
158pub fn source_str_to_stream(
159    psess: &ParseSess,
160    name: FileName,
161    source: String,
162    override_span: Option<Span>,
163) -> Result<TokenStream, Vec<Diag<'_>>> {
164    let source_file = psess.source_map().new_source_file(name, source);
165    source_file_to_stream(psess, source_file, override_span)
166}
167
168/// Given a source file, produces a sequence of token trees. Returns any buffered errors from
169/// parsing the token stream.
170fn source_file_to_stream<'psess>(
171    psess: &'psess ParseSess,
172    source_file: Arc<SourceFile>,
173    override_span: Option<Span>,
174) -> Result<TokenStream, Vec<Diag<'psess>>> {
175    let src = source_file.src.as_ref().unwrap_or_else(|| {
176        psess.dcx().bug(format!(
177            "cannot lex `source_file` without source: {}",
178            psess.source_map().filename_for_diagnostics(&source_file.name)
179        ));
180    });
181
182    lexer::lex_token_trees(psess, src.as_str(), source_file.start_pos, override_span)
183}
184
185/// Runs the given subparser `f` on the tokens of the given `attr`'s item.
186pub fn parse_in<'a, T>(
187    psess: &'a ParseSess,
188    tts: TokenStream,
189    name: &'static str,
190    mut f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
191) -> PResult<'a, T> {
192    let mut parser = Parser::new(psess, tts, Some(name));
193    let result = f(&mut parser)?;
194    if parser.token != token::Eof {
195        parser.unexpected()?;
196    }
197    Ok(result)
198}
199
200pub fn fake_token_stream_for_item(psess: &ParseSess, item: &ast::Item) -> TokenStream {
201    let source = pprust::item_to_string(item);
202    let filename = FileName::macro_expansion_source_code(&source);
203    unwrap_or_emit_fatal(source_str_to_stream(psess, filename, source, Some(item.span)))
204}
205
206pub fn fake_token_stream_for_crate(psess: &ParseSess, krate: &ast::Crate) -> TokenStream {
207    let source = pprust::crate_to_string_for_macros(krate);
208    let filename = FileName::macro_expansion_source_code(&source);
209    unwrap_or_emit_fatal(source_str_to_stream(
210        psess,
211        filename,
212        source,
213        Some(krate.spans.inner_span),
214    ))
215}
216
217pub fn parse_cfg_attr(
218    cfg_attr: &Attribute,
219    psess: &ParseSess,
220) -> Option<(MetaItemInner, Vec<(AttrItem, Span)>)> {
221    const CFG_ATTR_GRAMMAR_HELP: &str = "#[cfg_attr(condition, attribute, other_attribute, ...)]";
222    const CFG_ATTR_NOTE_REF: &str = "for more information, visit \
223        <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>";
224
225    match cfg_attr.get_normal_item().args {
226        ast::AttrArgs::Delimited(ast::DelimArgs { dspan, delim, ref tokens })
227            if !tokens.is_empty() =>
228        {
229            crate::validate_attr::check_cfg_attr_bad_delim(psess, dspan, delim);
230            match parse_in(psess, tokens.clone(), "`cfg_attr` input", |p| p.parse_cfg_attr()) {
231                Ok(r) => return Some(r),
232                Err(e) => {
233                    e.with_help(format!("the valid syntax is `{CFG_ATTR_GRAMMAR_HELP}`"))
234                        .with_note(CFG_ATTR_NOTE_REF)
235                        .emit();
236                }
237            }
238        }
239        _ => {
240            psess.dcx().emit_err(errors::MalformedCfgAttr {
241                span: cfg_attr.span,
242                sugg: CFG_ATTR_GRAMMAR_HELP,
243            });
244        }
245    }
246    None
247}