clang 22.0.0git
DependencyDirectivesScanner.cpp
Go to the documentation of this file.
1//===- DependencyDirectivesScanner.cpp ------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This is the interface for scanning header and source files to get the
11/// minimum necessary preprocessor directives for evaluating includes. It
12/// reduces the source down to #define, #include, #import, @import, and any
13/// conditional preprocessor logic that contains one of those.
14///
15//===----------------------------------------------------------------------===//
16
21#include "clang/Lex/Lexer.h"
22#include "clang/Lex/Pragma.h"
23#include "llvm/ADT/ScopeExit.h"
24#include "llvm/ADT/SmallString.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/StringSwitch.h"
27#include <optional>
28
29using namespace clang;
31using namespace llvm;
32
33namespace {
34
35struct DirectiveWithTokens {
37 unsigned NumTokens;
38
39 DirectiveWithTokens(DirectiveKind Kind, unsigned NumTokens)
40 : Kind(Kind), NumTokens(NumTokens) {}
41};
42
43/// Does an efficient "scan" of the sources to detect the presence of
44/// preprocessor (or module import) directives and collects the raw lexed tokens
45/// for those directives so that the \p Lexer can "replay" them when the file is
46/// included.
47///
48/// Note that the behavior of the raw lexer is affected by the language mode,
49/// while at this point we want to do a scan and collect tokens once,
50/// irrespective of the language mode that the file will get included in. To
51/// compensate for that the \p Lexer, while "replaying", will adjust a token
52/// where appropriate, when it could affect the preprocessor's state.
53/// For example in a directive like
54///
55/// \code
56/// #if __has_cpp_attribute(clang::fallthrough)
57/// \endcode
58///
59/// The preprocessor needs to see '::' as 'tok::coloncolon' instead of 2
60/// 'tok::colon'. The \p Lexer will adjust if it sees consecutive 'tok::colon'
61/// while in C++ mode.
62struct Scanner {
63 Scanner(StringRef Input,
65 DiagnosticsEngine *Diags, SourceLocation InputSourceLoc)
66 : Input(Input), Tokens(Tokens), Diags(Diags),
67 InputSourceLoc(InputSourceLoc), LangOpts(getLangOptsForDepScanning()),
68 TheLexer(InputSourceLoc, LangOpts, Input.begin(), Input.begin(),
69 Input.end()) {}
70
71 static LangOptions getLangOptsForDepScanning() {
72 LangOptions LangOpts;
73 // Set the lexer to use 'tok::at' for '@', instead of 'tok::unknown'.
74 LangOpts.ObjC = true;
75 LangOpts.LineComment = true;
76 LangOpts.RawStringLiterals = true;
77 // FIXME: we do not enable C11 or C++11, so we are missing u/u8/U"".
78 return LangOpts;
79 }
80
81 /// Lex the provided source and emit the directive tokens.
82 ///
83 /// \returns True on error.
84 bool scan(SmallVectorImpl<Directive> &Directives);
85
86 friend bool clang::scanInputForCXX20ModulesUsage(StringRef Source);
87
88private:
89 /// Lexes next token and advances \p First and the \p Lexer.
91 lexToken(const char *&First, const char *const End);
92
94 lexIncludeFilename(const char *&First, const char *const End);
95
96 void skipLine(const char *&First, const char *const End);
97 void skipDirective(StringRef Name, const char *&First, const char *const End);
98
99 /// Returns the spelling of a string literal or identifier after performing
100 /// any processing needed to handle \c clang::Token::NeedsCleaning.
101 StringRef cleanStringIfNeeded(const dependency_directives_scan::Token &Tok);
102
103 /// Lexes next token and if it is identifier returns its string, otherwise
104 /// it skips the current line and returns \p std::nullopt.
105 ///
106 /// In any case (whatever the token kind) \p First and the \p Lexer will
107 /// advance beyond the token.
108 [[nodiscard]] std::optional<StringRef>
109 tryLexIdentifierOrSkipLine(const char *&First, const char *const End);
110
111 /// Used when it is certain that next token is an identifier.
112 [[nodiscard]] StringRef lexIdentifier(const char *&First,
113 const char *const End);
114
115 /// Lexes next token and returns true iff it is an identifier that matches \p
116 /// Id, otherwise it skips the current line and returns false.
117 ///
118 /// In any case (whatever the token kind) \p First and the \p Lexer will
119 /// advance beyond the token.
120 [[nodiscard]] bool isNextIdentifierOrSkipLine(StringRef Id,
121 const char *&First,
122 const char *const End);
123
124 /// Lexes next token and returns true iff it matches the kind \p K.
125 /// Otherwise it skips the current line and returns false.
126 ///
127 /// In any case (whatever the token kind) \p First and the \p Lexer will
128 /// advance beyond the token.
129 [[nodiscard]] bool isNextTokenOrSkipLine(tok::TokenKind K, const char *&First,
130 const char *const End);
131
132 /// Lexes next token and if it is string literal, returns its string.
133 /// Otherwise, it skips the current line and returns \p std::nullopt.
134 ///
135 /// In any case (whatever the token kind) \p First and the \p Lexer will
136 /// advance beyond the token.
137 [[nodiscard]] std::optional<StringRef>
138 tryLexStringLiteralOrSkipLine(const char *&First, const char *const End);
139
140 [[nodiscard]] bool scanImpl(const char *First, const char *const End);
141 [[nodiscard]] bool lexPPLine(const char *&First, const char *const End);
142 [[nodiscard]] bool lexAt(const char *&First, const char *const End);
143 [[nodiscard]] bool lexModule(const char *&First, const char *const End);
144 [[nodiscard]] bool lexDefine(const char *HashLoc, const char *&First,
145 const char *const End);
146 [[nodiscard]] bool lexPragma(const char *&First, const char *const End);
147 [[nodiscard]] bool lex_Pragma(const char *&First, const char *const End);
148 [[nodiscard]] bool lexEndif(const char *&First, const char *const End);
149 [[nodiscard]] bool lexDefault(DirectiveKind Kind, const char *&First,
150 const char *const End);
151 [[nodiscard]] bool lexModuleDirectiveBody(DirectiveKind Kind,
152 const char *&First,
153 const char *const End);
154 void lexPPDirectiveBody(const char *&First, const char *const End);
155
156 DirectiveWithTokens &pushDirective(DirectiveKind Kind) {
157 Tokens.append(CurDirToks);
158 DirsWithToks.emplace_back(Kind, CurDirToks.size());
159 CurDirToks.clear();
160 return DirsWithToks.back();
161 }
162 void popDirective() {
163 Tokens.pop_back_n(DirsWithToks.pop_back_val().NumTokens);
164 }
165 DirectiveKind topDirective() const {
166 return DirsWithToks.empty() ? pp_none : DirsWithToks.back().Kind;
167 }
168
169 unsigned getOffsetAt(const char *CurPtr) const {
170 return CurPtr - Input.data();
171 }
172
173 /// Reports a diagnostic if the diagnostic engine is provided. Always returns
174 /// true at the end.
175 bool reportError(const char *CurPtr, unsigned Err);
176
177 StringMap<char> SplitIds;
178 StringRef Input;
180 DiagnosticsEngine *Diags;
181 SourceLocation InputSourceLoc;
182
183 const char *LastTokenPtr = nullptr;
184 /// Keeps track of the tokens for the currently lexed directive. Once a
185 /// directive is fully lexed and "committed" then the tokens get appended to
186 /// \p Tokens and \p CurDirToks is cleared for the next directive.
188 /// The directives that were lexed along with the number of tokens that each
189 /// directive contains. The tokens of all the directives are kept in \p Tokens
190 /// vector, in the same order as the directives order in \p DirsWithToks.
192 LangOptions LangOpts;
193 Lexer TheLexer;
194};
195
196} // end anonymous namespace
197
198bool Scanner::reportError(const char *CurPtr, unsigned Err) {
199 if (!Diags)
200 return true;
201 assert(CurPtr >= Input.data() && "invalid buffer ptr");
202 Diags->Report(InputSourceLoc.getLocWithOffset(getOffsetAt(CurPtr)), Err);
203 return true;
204}
205
206static void skipOverSpaces(const char *&First, const char *const End) {
207 while (First != End && isHorizontalWhitespace(*First))
208 ++First;
209}
210
211// Move back by one character, skipping escaped newlines (backslash + \n)
212static char previousChar(const char *First, const char *&Current) {
213 assert(Current > First);
214 --Current;
215 while (Current > First && isVerticalWhitespace(*Current)) {
216 // Check if the previous character is a backslash
217 if (Current > First && *(Current - 1) == '\\') {
218 // Use Lexer's getEscapedNewLineSize to get the size of the escaped
219 // newline
220 unsigned EscapeSize = Lexer::getEscapedNewLineSize(Current);
221 if (EscapeSize > 0) {
222 // Skip back over the entire escaped newline sequence (backslash +
223 // newline)
224 Current -= (1 + EscapeSize);
225 } else {
226 break;
227 }
228 } else {
229 break;
230 }
231 }
232 return *Current;
233}
234
235[[nodiscard]] static bool isRawStringLiteral(const char *First,
236 const char *Current) {
237 assert(First <= Current);
238
239 // Check if we can even back up.
240 if (*Current != '"' || First == Current)
241 return false;
242
243 // Check for an "R".
244 if (previousChar(First, Current) != 'R')
245 return false;
246 if (First == Current ||
248 return true;
249
250 // Check for a prefix of "u", "U", or "L".
251 if (*Current == 'u' || *Current == 'U' || *Current == 'L')
252 return First == Current ||
254
255 // Check for a prefix of "u8".
256 if (*Current != '8' || First == Current ||
257 previousChar(First, Current) != 'u')
258 return false;
259 return First == Current ||
261}
262
263static void skipRawString(const char *&First, const char *const End) {
264 assert(First[0] == '"');
265
266 const char *Last = ++First;
267 while (Last != End && *Last != '(')
268 ++Last;
269 if (Last == End) {
270 First = Last; // Hit the end... just give up.
271 return;
272 }
273
274 StringRef Terminator(First, Last - First);
275 for (;;) {
276 // Move First to just past the next ")".
277 First = Last;
278 while (First != End && *First != ')')
279 ++First;
280 if (First == End)
281 return;
282 ++First;
283
284 // Look ahead for the terminator sequence.
285 Last = First;
286 while (Last != End && size_t(Last - First) < Terminator.size() &&
287 Terminator[Last - First] == *Last)
288 ++Last;
289
290 // Check if we hit it (or the end of the file).
291 if (Last == End) {
292 First = Last;
293 return;
294 }
295 if (size_t(Last - First) < Terminator.size())
296 continue;
297 if (*Last != '"')
298 continue;
299 First = Last + 1;
300 return;
301 }
302}
303
304// Returns the length of EOL, either 0 (no end-of-line), 1 (\n) or 2 (\r\n)
305static unsigned isEOL(const char *First, const char *const End) {
306 if (First == End)
307 return 0;
308 if (End - First > 1 && isVerticalWhitespace(First[0]) &&
309 isVerticalWhitespace(First[1]) && First[0] != First[1])
310 return 2;
311 return !!isVerticalWhitespace(First[0]);
312}
313
314static void skipString(const char *&First, const char *const End) {
315 assert(*First == '\'' || *First == '"' || *First == '<');
316 const char Terminator = *First == '<' ? '>' : *First;
317 for (++First; First != End && *First != Terminator; ++First) {
318 // String and character literals don't extend past the end of the line.
320 return;
321 if (*First != '\\')
322 continue;
323 // Skip past backslash to the next character. This ensures that the
324 // character right after it is skipped as well, which matters if it's
325 // the terminator.
326 if (++First == End)
327 return;
328 if (!isWhitespace(*First))
329 continue;
330 // Whitespace after the backslash might indicate a line continuation.
331 const char *FirstAfterBackslashPastSpace = First;
332 skipOverSpaces(FirstAfterBackslashPastSpace, End);
333 if (unsigned NLSize = isEOL(FirstAfterBackslashPastSpace, End)) {
334 // Advance the character pointer to the next line for the next
335 // iteration.
336 First = FirstAfterBackslashPastSpace + NLSize - 1;
337 }
338 }
339 if (First != End)
340 ++First; // Finish off the string.
341}
342
343// Returns the length of the skipped newline
344static unsigned skipNewline(const char *&First, const char *End) {
345 if (First == End)
346 return 0;
347 assert(isVerticalWhitespace(*First));
348 unsigned Len = isEOL(First, End);
349 assert(Len && "expected newline");
350 First += Len;
351 return Len;
352}
353
354static void skipToNewlineRaw(const char *&First, const char *const End) {
355 for (;;) {
356 if (First == End)
357 return;
358
359 unsigned Len = isEOL(First, End);
360 if (Len)
361 return;
362
363 char LastNonWhitespace = ' ';
364 do {
366 LastNonWhitespace = *First;
367 if (++First == End)
368 return;
369 Len = isEOL(First, End);
370 } while (!Len);
371
372 if (LastNonWhitespace != '\\')
373 return;
374
375 First += Len;
376 // Keep skipping lines...
377 }
378}
379
380static void skipLineComment(const char *&First, const char *const End) {
381 assert(First[0] == '/' && First[1] == '/');
382 First += 2;
384}
385
386static void skipBlockComment(const char *&First, const char *const End) {
387 assert(First[0] == '/' && First[1] == '*');
388 if (End - First < 4) {
389 First = End;
390 return;
391 }
392 for (First += 3; First != End; ++First)
393 if (First[-1] == '*' && First[0] == '/') {
394 ++First;
395 return;
396 }
397}
398
399/// \returns True if the current single quotation mark character is a C++14
400/// digit separator.
401static bool isQuoteCppDigitSeparator(const char *const Start,
402 const char *const Cur,
403 const char *const End) {
404 assert(*Cur == '\'' && "expected quotation character");
405 // skipLine called in places where we don't expect a valid number
406 // body before `start` on the same line, so always return false at the start.
407 if (Start == Cur)
408 return false;
409 // The previous character must be a valid PP number character.
410 // Make sure that the L, u, U, u8 prefixes don't get marked as a
411 // separator though.
412 char Prev = *(Cur - 1);
413 if (Prev == 'L' || Prev == 'U' || Prev == 'u')
414 return false;
415 if (Prev == '8' && (Cur - 1 != Start) && *(Cur - 2) == 'u')
416 return false;
417 if (!isPreprocessingNumberBody(Prev))
418 return false;
419 // The next character should be a valid identifier body character.
420 return (Cur + 1) < End && isAsciiIdentifierContinue(*(Cur + 1));
421}
422
423void Scanner::skipLine(const char *&First, const char *const End) {
424 for (;;) {
425 assert(First <= End);
426 if (First == End)
427 return;
428
430 skipNewline(First, End);
431 return;
432 }
433 const char *Start = First;
434 // Use `LastNonWhitespace`to track if a line-continuation has ever been seen
435 // before a new-line character:
436 char LastNonWhitespace = ' ';
437 while (First != End && !isVerticalWhitespace(*First)) {
438 // Iterate over strings correctly to avoid comments and newlines.
439 if (*First == '"' ||
440 (*First == '\'' && !isQuoteCppDigitSeparator(Start, First, End))) {
441 LastTokenPtr = First;
442 if (isRawStringLiteral(Start, First))
443 skipRawString(First, End);
444 else
445 skipString(First, End);
446 continue;
447 }
448
449 // Continue on the same line if an EOL is preceded with backslash
450 if (First + 1 < End && *First == '\\') {
451 if (unsigned Len = isEOL(First + 1, End)) {
452 First += 1 + Len;
453 continue;
454 }
455 }
456
457 // Iterate over comments correctly.
458 if (*First != '/' || End - First < 2) {
459 LastTokenPtr = First;
460 if (!isWhitespace(*First))
461 LastNonWhitespace = *First;
462 ++First;
463 continue;
464 }
465
466 if (First[1] == '/') {
467 // "//...".
469 continue;
470 }
471
472 if (First[1] != '*') {
473 LastTokenPtr = First;
474 if (!isWhitespace(*First))
475 LastNonWhitespace = *First;
476 ++First;
477 continue;
478 }
479
480 // "/*...*/".
482 }
483 if (First == End)
484 return;
485
486 // Skip over the newline.
487 skipNewline(First, End);
488
489 if (LastNonWhitespace != '\\')
490 break;
491 }
492}
493
494void Scanner::skipDirective(StringRef Name, const char *&First,
495 const char *const End) {
496 if (llvm::StringSwitch<bool>(Name)
497 .Case("warning", true)
498 .Case("error", true)
499 .Default(false))
500 // Do not process quotes or comments.
502 else
503 skipLine(First, End);
504}
505
506static void skipWhitespace(const char *&First, const char *const End) {
507 for (;;) {
508 assert(First <= End);
509 skipOverSpaces(First, End);
510
511 if (End - First < 2)
512 return;
513
514 if (*First == '\\') {
515 const char *Ptr = First + 1;
516 while (Ptr < End && isHorizontalWhitespace(*Ptr))
517 ++Ptr;
518 if (Ptr != End && isVerticalWhitespace(*Ptr)) {
519 skipNewline(Ptr, End);
520 First = Ptr;
521 continue;
522 }
523 return;
524 }
525
526 // Check for a non-comment character.
527 if (First[0] != '/')
528 return;
529
530 // "// ...".
531 if (First[1] == '/') {
533 return;
534 }
535
536 // Cannot be a comment.
537 if (First[1] != '*')
538 return;
539
540 // "/*...*/".
542 }
543}
544
545bool Scanner::lexModuleDirectiveBody(DirectiveKind Kind, const char *&First,
546 const char *const End) {
547 const char *DirectiveLoc = Input.data() + CurDirToks.front().Offset;
548 for (;;) {
549 // Keep a copy of the First char incase it needs to be reset.
550 const char *Previous = First;
551 const dependency_directives_scan::Token &Tok = lexToken(First, End);
552 if ((Tok.is(tok::hash) || Tok.is(tok::at)) &&
554 CurDirToks.pop_back();
555 First = Previous;
556 return false;
557 }
558 if (Tok.is(tok::eof))
559 return reportError(
560 DirectiveLoc,
561 diag::err_dep_source_scanner_missing_semi_after_at_import);
562 if (Tok.is(tok::semi))
563 break;
564 }
565
566 const auto &Tok = lexToken(First, End);
567 pushDirective(Kind);
568 if (Tok.is(tok::eof) || Tok.is(tok::eod))
569 return false;
570 return reportError(DirectiveLoc,
571 diag::err_dep_source_scanner_unexpected_tokens_at_import);
572}
573
574dependency_directives_scan::Token &Scanner::lexToken(const char *&First,
575 const char *const End) {
576 clang::Token Tok;
577 TheLexer.LexFromRawLexer(Tok);
578 First = Input.data() + TheLexer.getCurrentBufferOffset();
579 assert(First <= End);
580
581 unsigned Offset = TheLexer.getCurrentBufferOffset() - Tok.getLength();
582 CurDirToks.emplace_back(Offset, Tok.getLength(), Tok.getKind(),
583 Tok.getFlags());
584 return CurDirToks.back();
585}
586
588Scanner::lexIncludeFilename(const char *&First, const char *const End) {
589 clang::Token Tok;
590 TheLexer.LexIncludeFilename(Tok);
591 First = Input.data() + TheLexer.getCurrentBufferOffset();
592 assert(First <= End);
593
594 unsigned Offset = TheLexer.getCurrentBufferOffset() - Tok.getLength();
595 CurDirToks.emplace_back(Offset, Tok.getLength(), Tok.getKind(),
596 Tok.getFlags());
597 return CurDirToks.back();
598}
599
600void Scanner::lexPPDirectiveBody(const char *&First, const char *const End) {
601 while (true) {
602 const dependency_directives_scan::Token &Tok = lexToken(First, End);
603 if (Tok.is(tok::eod) || Tok.is(tok::eof))
604 break;
605 }
606}
607
608StringRef
609Scanner::cleanStringIfNeeded(const dependency_directives_scan::Token &Tok) {
610 bool NeedsCleaning = Tok.Flags & clang::Token::NeedsCleaning;
611 if (LLVM_LIKELY(!NeedsCleaning))
612 return Input.slice(Tok.Offset, Tok.getEnd());
613
614 SmallString<64> Spelling;
615 Spelling.resize(Tok.Length);
616
617 // FIXME: C++11 raw string literals need special handling (see getSpellingSlow
618 // in the Lexer). Currently we cannot see them due to our LangOpts.
619
620 unsigned SpellingLength = 0;
621 const char *BufPtr = Input.begin() + Tok.Offset;
622 const char *AfterIdent = Input.begin() + Tok.getEnd();
623 while (BufPtr < AfterIdent) {
624 auto [Char, Size] = Lexer::getCharAndSizeNoWarn(BufPtr, LangOpts);
625 Spelling[SpellingLength++] = Char;
626 BufPtr += Size;
627 }
628
629 return SplitIds.try_emplace(StringRef(Spelling.begin(), SpellingLength), 0)
630 .first->first();
631}
632
633std::optional<StringRef>
634Scanner::tryLexIdentifierOrSkipLine(const char *&First, const char *const End) {
635 const dependency_directives_scan::Token &Tok = lexToken(First, End);
636 if (Tok.isNot(tok::raw_identifier)) {
637 if (!Tok.is(tok::eod))
638 skipLine(First, End);
639 return std::nullopt;
640 }
641
642 return cleanStringIfNeeded(Tok);
643}
644
645StringRef Scanner::lexIdentifier(const char *&First, const char *const End) {
646 std::optional<StringRef> Id = tryLexIdentifierOrSkipLine(First, End);
647 assert(Id && "expected identifier token");
648 return *Id;
649}
650
651bool Scanner::isNextIdentifierOrSkipLine(StringRef Id, const char *&First,
652 const char *const End) {
653 if (std::optional<StringRef> FoundId =
654 tryLexIdentifierOrSkipLine(First, End)) {
655 if (*FoundId == Id)
656 return true;
657 skipLine(First, End);
658 }
659 return false;
660}
661
662bool Scanner::isNextTokenOrSkipLine(tok::TokenKind K, const char *&First,
663 const char *const End) {
664 const dependency_directives_scan::Token &Tok = lexToken(First, End);
665 if (Tok.is(K))
666 return true;
667 skipLine(First, End);
668 return false;
669}
670
671std::optional<StringRef>
672Scanner::tryLexStringLiteralOrSkipLine(const char *&First,
673 const char *const End) {
674 const dependency_directives_scan::Token &Tok = lexToken(First, End);
675 if (!tok::isStringLiteral(Tok.Kind)) {
676 if (!Tok.is(tok::eod))
677 skipLine(First, End);
678 return std::nullopt;
679 }
680
681 return cleanStringIfNeeded(Tok);
682}
683
684bool Scanner::lexAt(const char *&First, const char *const End) {
685 // Handle "@import".
686
687 // Lex '@'.
688 const dependency_directives_scan::Token &AtTok = lexToken(First, End);
689 assert(AtTok.is(tok::at));
690 (void)AtTok;
691
692 if (!isNextIdentifierOrSkipLine("import", First, End))
693 return false;
694 return lexModuleDirectiveBody(decl_at_import, First, End);
695}
696
697bool Scanner::lexModule(const char *&First, const char *const End) {
698 StringRef Id = lexIdentifier(First, End);
699 bool Export = false;
700 if (Id == "export") {
701 Export = true;
702 std::optional<StringRef> NextId = tryLexIdentifierOrSkipLine(First, End);
703 if (!NextId)
704 return false;
705 Id = *NextId;
706 }
707
708 if (Id != "module" && Id != "import") {
709 skipLine(First, End);
710 return false;
711 }
712
713 skipWhitespace(First, End);
714
715 // Ignore this as a module directive if the next character can't be part of
716 // an import.
717
718 switch (*First) {
719 case ':': {
720 // `module :` is never the start of a valid module declaration.
721 if (Id == "module") {
722 skipLine(First, End);
723 return false;
724 }
725 // A module partition starts with exactly one ':'. If we have '::', this is
726 // a scope resolution instead and shouldn't be recognized as a directive
727 // per P1857R3.
728 if (First + 1 != End && First[1] == ':') {
729 skipLine(First, End);
730 return false;
731 }
732 // `import:(type)name` is a valid ObjC method decl, so check one more token.
733 (void)lexToken(First, End);
734 if (!tryLexIdentifierOrSkipLine(First, End))
735 return false;
736 break;
737 }
738 case ';': {
739 // Handle the global module fragment `module;`.
740 if (Id == "module" && !Export)
741 break;
742 skipLine(First, End);
743 return false;
744 }
745 case '<':
746 case '"':
747 break;
748 default:
750 skipLine(First, End);
751 return false;
752 }
753 }
754
755 TheLexer.seek(getOffsetAt(First), /*IsAtStartOfLine*/ false);
756
758 if (Id == "module")
760 else
762
763 return lexModuleDirectiveBody(Kind, First, End);
764}
765
766bool Scanner::lex_Pragma(const char *&First, const char *const End) {
767 if (!isNextTokenOrSkipLine(tok::l_paren, First, End))
768 return false;
769
770 std::optional<StringRef> Str = tryLexStringLiteralOrSkipLine(First, End);
771
772 if (!Str || !isNextTokenOrSkipLine(tok::r_paren, First, End))
773 return false;
774
775 SmallString<64> Buffer(*Str);
776 prepare_PragmaString(Buffer);
777
778 // Use a new scanner instance since the tokens will be inside the allocated
779 // string. We should already have captured all the relevant tokens in the
780 // current scanner.
782 const char *Begin = Buffer.c_str();
783 Scanner PragmaScanner{StringRef(Begin, Buffer.size()), DiscardTokens, Diags,
784 InputSourceLoc};
785
786 PragmaScanner.TheLexer.setParsingPreprocessorDirective(true);
787 if (PragmaScanner.lexPragma(Begin, Buffer.end()))
788 return true;
789
790 DirectiveKind K = PragmaScanner.topDirective();
791 if (K == pp_none) {
792 skipLine(First, End);
793 return false;
794 }
795
796 assert(Begin == Buffer.end());
797 pushDirective(K);
798 return false;
799}
800
801bool Scanner::lexPragma(const char *&First, const char *const End) {
802 std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(First, End);
803 if (!FoundId)
804 return false;
805
806 StringRef Id = *FoundId;
807 auto Kind = llvm::StringSwitch<DirectiveKind>(Id)
808 .Case("once", pp_pragma_once)
809 .Case("push_macro", pp_pragma_push_macro)
810 .Case("pop_macro", pp_pragma_pop_macro)
811 .Case("include_alias", pp_pragma_include_alias)
812 .Default(pp_none);
813 if (Kind != pp_none) {
814 lexPPDirectiveBody(First, End);
815 pushDirective(Kind);
816 return false;
817 }
818
819 if (Id != "clang") {
820 skipLine(First, End);
821 return false;
822 }
823
824 FoundId = tryLexIdentifierOrSkipLine(First, End);
825 if (!FoundId)
826 return false;
827 Id = *FoundId;
828
829 // #pragma clang system_header
830 if (Id == "system_header") {
831 lexPPDirectiveBody(First, End);
832 pushDirective(pp_pragma_system_header);
833 return false;
834 }
835
836 if (Id != "module") {
837 skipLine(First, End);
838 return false;
839 }
840
841 // #pragma clang module.
842 if (!isNextIdentifierOrSkipLine("import", First, End))
843 return false;
844
845 // #pragma clang module import.
846 lexPPDirectiveBody(First, End);
847 pushDirective(pp_pragma_import);
848 return false;
849}
850
851bool Scanner::lexEndif(const char *&First, const char *const End) {
852 // Strip out "#else" if it's empty.
853 if (topDirective() == pp_else)
854 popDirective();
855
856 // If "#ifdef" is empty, strip it and skip the "#endif".
857 //
858 // FIXME: Once/if Clang starts disallowing __has_include in macro expansions,
859 // we can skip empty `#if` and `#elif` blocks as well after scanning for a
860 // literal __has_include in the condition. Even without that rule we could
861 // drop the tokens if we scan for identifiers in the condition and find none.
862 if (topDirective() == pp_ifdef || topDirective() == pp_ifndef) {
863 popDirective();
864 skipLine(First, End);
865 return false;
866 }
867
868 return lexDefault(pp_endif, First, End);
869}
870
871bool Scanner::lexDefault(DirectiveKind Kind, const char *&First,
872 const char *const End) {
873 lexPPDirectiveBody(First, End);
874 pushDirective(Kind);
875 return false;
876}
877
878static bool isStartOfRelevantLine(char First) {
879 switch (First) {
880 case '#':
881 case '@':
882 case 'i':
883 case 'e':
884 case 'm':
885 case '_':
886 return true;
887 }
888 return false;
889}
890
891bool Scanner::lexPPLine(const char *&First, const char *const End) {
892 assert(First != End);
893
894 skipWhitespace(First, End);
895 assert(First <= End);
896 if (First == End)
897 return false;
898
900 skipLine(First, End);
901 assert(First <= End);
902 return false;
903 }
904
905 LastTokenPtr = First;
906
907 TheLexer.seek(getOffsetAt(First), /*IsAtStartOfLine*/ true);
908
909 auto ScEx1 = make_scope_exit([&]() {
910 /// Clear Scanner's CurDirToks before returning, in case we didn't push a
911 /// new directive.
912 CurDirToks.clear();
913 });
914
915 if (*First == '_') {
916 if (isNextIdentifierOrSkipLine("_Pragma", First, End))
917 return lex_Pragma(First, End);
918 return false;
919 }
920
921 // Handle preprocessing directives.
922
923 TheLexer.setParsingPreprocessorDirective(true);
924 auto ScEx2 = make_scope_exit(
925 [&]() { TheLexer.setParsingPreprocessorDirective(false); });
926
927 // Handle "@import".
928 if (*First == '@')
929 return lexAt(First, End);
930
931 // Handle module directives for C++20 modules.
932 if (*First == 'i' || *First == 'e' || *First == 'm')
933 return lexModule(First, End);
934
935 // Lex '#'.
936 const dependency_directives_scan::Token &HashTok = lexToken(First, End);
937 if (HashTok.is(tok::hashhash)) {
938 // A \p tok::hashhash at this location is passed by the preprocessor to the
939 // parser to interpret, like any other token. So for dependency scanning
940 // skip it like a normal token not affecting the preprocessor.
941 skipLine(First, End);
942 assert(First <= End);
943 return false;
944 }
945 assert(HashTok.is(tok::hash));
946 (void)HashTok;
947
948 std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(First, End);
949 if (!FoundId)
950 return false;
951
952 StringRef Id = *FoundId;
953
954 if (Id == "pragma")
955 return lexPragma(First, End);
956
957 auto Kind = llvm::StringSwitch<DirectiveKind>(Id)
958 .Case("include", pp_include)
959 .Case("__include_macros", pp___include_macros)
960 .Case("define", pp_define)
961 .Case("undef", pp_undef)
962 .Case("import", pp_import)
963 .Case("include_next", pp_include_next)
964 .Case("if", pp_if)
965 .Case("ifdef", pp_ifdef)
966 .Case("ifndef", pp_ifndef)
967 .Case("elif", pp_elif)
968 .Case("elifdef", pp_elifdef)
969 .Case("elifndef", pp_elifndef)
970 .Case("else", pp_else)
971 .Case("endif", pp_endif)
972 .Default(pp_none);
973 if (Kind == pp_none) {
974 skipDirective(Id, First, End);
975 return false;
976 }
977
978 if (Kind == pp_endif)
979 return lexEndif(First, End);
980
981 switch (Kind) {
982 case pp_include:
984 case pp_include_next:
985 case pp_import:
986 // Ignore missing filenames in include or import directives.
987 if (lexIncludeFilename(First, End).is(tok::eod)) {
988 return false;
989 }
990 break;
991 default:
992 break;
993 }
994
995 // Everything else.
996 return lexDefault(Kind, First, End);
997}
998
999static void skipUTF8ByteOrderMark(const char *&First, const char *const End) {
1000 if ((End - First) >= 3 && First[0] == '\xef' && First[1] == '\xbb' &&
1001 First[2] == '\xbf')
1002 First += 3;
1003}
1004
1005bool Scanner::scanImpl(const char *First, const char *const End) {
1007 while (First != End)
1008 if (lexPPLine(First, End))
1009 return true;
1010 return false;
1011}
1012
1013bool Scanner::scan(SmallVectorImpl<Directive> &Directives) {
1014 bool Error = scanImpl(Input.begin(), Input.end());
1015
1016 if (!Error) {
1017 // Add an EOF on success.
1018 if (LastTokenPtr &&
1019 (Tokens.empty() || LastTokenPtr > Input.begin() + Tokens.back().Offset))
1020 pushDirective(tokens_present_before_eof);
1021 pushDirective(pp_eof);
1022 }
1023
1024 ArrayRef<dependency_directives_scan::Token> RemainingTokens = Tokens;
1025 for (const DirectiveWithTokens &DirWithToks : DirsWithToks) {
1026 assert(RemainingTokens.size() >= DirWithToks.NumTokens);
1027 Directives.emplace_back(DirWithToks.Kind,
1028 RemainingTokens.take_front(DirWithToks.NumTokens));
1029 RemainingTokens = RemainingTokens.drop_front(DirWithToks.NumTokens);
1030 }
1031 assert(RemainingTokens.empty());
1032
1033 return Error;
1034}
1035
1039 SourceLocation InputSourceLoc) {
1040 return Scanner(Input, Tokens, Diags, InputSourceLoc).scan(Directives);
1041}
1042
1044 StringRef Source,
1046 llvm::raw_ostream &OS) {
1047 // Add a space separator where it is convenient for testing purposes.
1048 auto needsSpaceSeparator =
1049 [](tok::TokenKind Prev,
1050 const dependency_directives_scan::Token &Tok) -> bool {
1051 if (Prev == Tok.Kind)
1052 return !Tok.isOneOf(tok::l_paren, tok::r_paren, tok::l_square,
1053 tok::r_square);
1054 if (Prev == tok::raw_identifier &&
1055 Tok.isOneOf(tok::hash, tok::numeric_constant, tok::string_literal,
1056 tok::char_constant, tok::header_name))
1057 return true;
1058 if (Prev == tok::r_paren &&
1059 Tok.isOneOf(tok::raw_identifier, tok::hash, tok::string_literal,
1060 tok::char_constant, tok::unknown))
1061 return true;
1062 if (Prev == tok::comma &&
1063 Tok.isOneOf(tok::l_paren, tok::string_literal, tok::less))
1064 return true;
1065 return false;
1066 };
1067
1068 for (const dependency_directives_scan::Directive &Directive : Directives) {
1070 OS << "<TokBeforeEOF>";
1071 std::optional<tok::TokenKind> PrevTokenKind;
1073 if (PrevTokenKind && needsSpaceSeparator(*PrevTokenKind, Tok))
1074 OS << ' ';
1075 PrevTokenKind = Tok.Kind;
1076 OS << Source.slice(Tok.Offset, Tok.getEnd());
1077 }
1078 }
1079}
1080
1082 const char *const End) {
1083 assert(First <= End);
1084 while (First != End) {
1085 if (*First == '#') {
1086 ++First;
1087 skipToNewlineRaw(First, End);
1088 }
1089 skipWhitespace(First, End);
1090 if (const auto Len = isEOL(First, End)) {
1091 First += Len;
1092 continue;
1093 }
1094 break;
1095 }
1096}
1097
1099 const char *First = Source.begin();
1100 const char *const End = Source.end();
1102 if (First == End)
1103 return false;
1104
1105 // Check if the next token can even be a module directive before creating a
1106 // full lexer.
1107 if (!(*First == 'i' || *First == 'e' || *First == 'm'))
1108 return false;
1109
1111 Scanner S(StringRef(First, End - First), Tokens, nullptr, SourceLocation());
1112 S.TheLexer.setParsingPreprocessorDirective(true);
1113 if (S.lexModule(First, End))
1114 return false;
1115 auto IsCXXNamedModuleDirective = [](const DirectiveWithTokens &D) {
1116 switch (D.Kind) {
1121 return true;
1122 default:
1123 return false;
1124 }
1125 };
1126 return llvm::any_of(S.DirsWithToks, IsCXXNamedModuleDirective);
1127}
Defines the Diagnostic-related interfaces.
const Decl * D
static void skipBlockComment(const char *&First, const char *const End)
static void skipRawString(const char *&First, const char *const End)
static void skipString(const char *&First, const char *const End)
static bool isStartOfRelevantLine(char First)
static void skipWhitespace(const char *&First, const char *const End)
static bool isRawStringLiteral(const char *First, const char *Current)
static void skipUntilMaybeCXX20ModuleDirective(const char *&First, const char *const End)
static void skipOverSpaces(const char *&First, const char *const End)
static unsigned isEOL(const char *First, const char *const End)
static char previousChar(const char *First, const char *&Current)
static void skipToNewlineRaw(const char *&First, const char *const End)
static unsigned skipNewline(const char *&First, const char *End)
static void skipUTF8ByteOrderMark(const char *&First, const char *const End)
static void skipLineComment(const char *&First, const char *const End)
static bool isQuoteCppDigitSeparator(const char *const Start, const char *const Cur, const char *const End)
This is the interface for scanning header and source files to get the minimum necessary preprocessor ...
uint32_t Id
Definition: SemaARM.cpp:1179
SourceLocation Begin
StateNode * Previous
Kind
Lists the kind of concrete classes of Decl.
Definition: DeclBase.h:89
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:231
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:434
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens.
Definition: Lexer.h:78
static unsigned getEscapedNewLineSize(const char *P)
getEscapedNewLineSize - Return the size of the specified escaped newline, or 0 if it is not an escape...
Definition: Lexer.cpp:1276
static SizedChar getCharAndSizeNoWarn(const char *Ptr, const LangOptions &LangOpts)
getCharAndSizeNoWarn - Like the getCharAndSize method, but does not ever emit a warning.
Definition: Lexer.h:604
Encodes a location in the source.
Token - This structure provides full information about a lexed token.
Definition: Token.h:36
unsigned getFlags() const
Return the internal represtation of the flags.
Definition: Token.h:264
unsigned getLength() const
Definition: Token.h:137
tok::TokenKind getKind() const
Definition: Token.h:97
@ StartOfLine
Definition: Token.h:75
@ NeedsCleaning
Definition: Token.h:80
DirectiveKind
Represents the kind of preprocessor directive or a module declaration that is tracked by the scanner ...
@ tokens_present_before_eof
Indicates that there are tokens present between the last scanned directive and eof.
bool isStringLiteral(TokenKind K)
Return true if this is a C or C++ string-literal (or C++11 user-defined-string-literal) token.
Definition: TokenKinds.h:89
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:25
The JSON file list parser is used to communicate input to InstallAPI.
LLVM_READONLY bool isVerticalWhitespace(unsigned char c)
Returns true if this character is vertical ASCII whitespace: '\n', '\r'.
Definition: CharInfo.h:99
LLVM_READONLY bool isAsciiIdentifierContinue(unsigned char c)
Definition: CharInfo.h:61
void printDependencyDirectivesAsSource(StringRef Source, ArrayRef< dependency_directives_scan::Directive > Directives, llvm::raw_ostream &OS)
Print the previously scanned dependency directives as minimized source text.
bool scanInputForCXX20ModulesUsage(StringRef Source)
Scan an input source buffer for C++20 named module usage.
LLVM_READONLY bool isHorizontalWhitespace(unsigned char c)
Returns true if this character is horizontal ASCII whitespace: ' ', '\t', '\f', '\v'.
Definition: CharInfo.h:91
bool scanSourceForDependencyDirectives(StringRef Input, SmallVectorImpl< dependency_directives_scan::Token > &Tokens, SmallVectorImpl< dependency_directives_scan::Directive > &Directives, DiagnosticsEngine *Diags=nullptr, SourceLocation InputSourceLoc=SourceLocation())
Scan the input for the preprocessor directives that might have an effect on the dependencies for a co...
LLVM_READONLY bool isWhitespace(unsigned char c)
Return true if this character is horizontal or vertical ASCII whitespace: ' ', '\t',...
Definition: CharInfo.h:108
LLVM_READONLY bool isPreprocessingNumberBody(unsigned char c)
Return true if this is the body character of a C preprocessing number, which is [a-zA-Z0-9_.
Definition: CharInfo.h:168
void prepare_PragmaString(SmallVectorImpl< char > &StrVal)
Destringize a _Pragma("") string according to C11 6.10.9.1: "The string literal is destringized by de...
Definition: Pragma.cpp:302
Diagnostic wrappers for TextAPI types for error reporting.
Definition: Dominators.h:30
Represents a directive that's lexed as part of the dependency directives scanning.
Token lexed as part of dependency directive scanning.
unsigned Offset
Offset into the original source input.