clang 22.0.0git
Preprocessor.h
Go to the documentation of this file.
1//===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Defines the clang::Preprocessor interface.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15#define LLVM_CLANG_LEX_PREPROCESSOR_H
16
20#include "clang/Basic/LLVM.h"
22#include "clang/Basic/Module.h"
27#include "clang/Lex/Lexer.h"
28#include "clang/Lex/MacroInfo.h"
30#include "clang/Lex/ModuleMap.h"
33#include "clang/Lex/Token.h"
36#include "llvm/ADT/APSInt.h"
37#include "llvm/ADT/ArrayRef.h"
38#include "llvm/ADT/DenseMap.h"
39#include "llvm/ADT/FoldingSet.h"
40#include "llvm/ADT/FunctionExtras.h"
41#include "llvm/ADT/PointerUnion.h"
42#include "llvm/ADT/STLExtras.h"
43#include "llvm/ADT/SmallPtrSet.h"
44#include "llvm/ADT/SmallVector.h"
45#include "llvm/ADT/StringRef.h"
46#include "llvm/ADT/TinyPtrVector.h"
47#include "llvm/ADT/iterator_range.h"
48#include "llvm/Support/Allocator.h"
49#include "llvm/Support/Casting.h"
50#include "llvm/Support/Registry.h"
51#include <cassert>
52#include <cstddef>
53#include <cstdint>
54#include <map>
55#include <memory>
56#include <optional>
57#include <string>
58#include <utility>
59#include <vector>
60
61namespace llvm {
62
63template<unsigned InternalLen> class SmallString;
64
65} // namespace llvm
66
67namespace clang {
68
69class CodeCompletionHandler;
70class CommentHandler;
71class DirectoryEntry;
72class EmptylineHandler;
73class ExternalPreprocessorSource;
74class FileEntry;
75class FileManager;
76class HeaderSearch;
77class MacroArgs;
78class PragmaHandler;
79class PragmaNamespace;
80class PreprocessingRecord;
81class PreprocessorLexer;
82class PreprocessorOptions;
83class ScratchBuffer;
84class TargetInfo;
85class NoTrivialPPDirectiveTracer;
86
87namespace Builtin {
88class Context;
89}
90
91/// Stores token information for comparing actual tokens with
92/// predefined values. Only handles simple tokens and identifiers.
94 tok::TokenKind Kind;
96
97public:
98 TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
99 assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
100 assert(Kind != tok::identifier &&
101 "Identifiers should be created by TokenValue(IdentifierInfo *)");
102 assert(!tok::isLiteral(Kind) && "Literals are not supported.");
103 assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
104 }
105
106 TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
107
108 bool operator==(const Token &Tok) const {
109 return Tok.getKind() == Kind &&
110 (!II || II == Tok.getIdentifierInfo());
111 }
112};
113
114/// Context in which macro name is used.
116 // other than #define or #undef
118
119 // macro name specified in #define
121
122 // macro name specified in #undef
123 MU_Undef = 2
125
126enum class EmbedResult {
127 Invalid = -1, // Parsing error occurred.
128 NotFound = 0, // Corresponds to __STDC_EMBED_NOT_FOUND__
129 Found = 1, // Corresponds to __STDC_EMBED_FOUND__
130 Empty = 2, // Corresponds to __STDC_EMBED_EMPTY__
131};
132
136 std::uint64_t Version;
137};
138
139/// Engages in a tight little dance with the lexer to efficiently
140/// preprocess tokens.
141///
142/// Lexers know only about tokens within a single source file, and don't
143/// know anything about preprocessor-level issues like the \#include stack,
144/// token expansion, etc.
148
149 llvm::unique_function<void(const clang::Token &)> OnToken;
150 /// Functor for getting the dependency preprocessor directives of a file.
151 ///
152 /// These are directives derived from a special form of lexing where the
153 /// source input is scanned for the preprocessor directives that might have an
154 /// effect on the dependencies for a compilation unit.
155 DependencyDirectivesGetter *GetDependencyDirectives = nullptr;
156 const PreprocessorOptions &PPOpts;
157 DiagnosticsEngine *Diags;
158 const LangOptions &LangOpts;
159 const TargetInfo *Target = nullptr;
160 const TargetInfo *AuxTarget = nullptr;
161 FileManager &FileMgr;
162 SourceManager &SourceMgr;
163 std::unique_ptr<ScratchBuffer> ScratchBuf;
164 HeaderSearch &HeaderInfo;
165 ModuleLoader &TheModuleLoader;
166
167 /// External source of macros.
169
170 /// A BumpPtrAllocator object used to quickly allocate and release
171 /// objects internal to the Preprocessor.
172 llvm::BumpPtrAllocator BP;
173
174 /// Identifiers for builtin macros and other builtins.
175 IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__
176 IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__
177 IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
178 IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
179 IdentifierInfo *Ident__FILE_NAME__; // __FILE_NAME__
180 IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
181 IdentifierInfo *Ident__COUNTER__; // __COUNTER__
182 IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma
183 IdentifierInfo *Ident__identifier; // __identifier
184 IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
185 IdentifierInfo *Ident__VA_OPT__; // __VA_OPT__
186 IdentifierInfo *Ident__has_feature; // __has_feature
187 IdentifierInfo *Ident__has_extension; // __has_extension
188 IdentifierInfo *Ident__has_builtin; // __has_builtin
189 IdentifierInfo *Ident__has_constexpr_builtin; // __has_constexpr_builtin
190 IdentifierInfo *Ident__has_attribute; // __has_attribute
191 IdentifierInfo *Ident__has_embed; // __has_embed
192 IdentifierInfo *Ident__has_include; // __has_include
193 IdentifierInfo *Ident__has_include_next; // __has_include_next
194 IdentifierInfo *Ident__has_warning; // __has_warning
195 IdentifierInfo *Ident__is_identifier; // __is_identifier
196 IdentifierInfo *Ident__building_module; // __building_module
197 IdentifierInfo *Ident__MODULE__; // __MODULE__
198 IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute
199 IdentifierInfo *Ident__has_c_attribute; // __has_c_attribute
200 IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute
201 IdentifierInfo *Ident__is_target_arch; // __is_target_arch
202 IdentifierInfo *Ident__is_target_vendor; // __is_target_vendor
203 IdentifierInfo *Ident__is_target_os; // __is_target_os
204 IdentifierInfo *Ident__is_target_environment; // __is_target_environment
205 IdentifierInfo *Ident__is_target_variant_os;
206 IdentifierInfo *Ident__is_target_variant_environment;
207 IdentifierInfo *Ident__FLT_EVAL_METHOD__; // __FLT_EVAL_METHOD
208
209 // Weak, only valid (and set) while InMacroArgs is true.
210 Token* ArgMacro;
211
212 SourceLocation DATELoc, TIMELoc;
213
214 // FEM_UnsetOnCommandLine means that an explicit evaluation method was
215 // not specified on the command line. The target is queried to set the
216 // default evaluation method.
217 LangOptions::FPEvalMethodKind CurrentFPEvalMethod =
219
220 // The most recent pragma location where the floating point evaluation
221 // method was modified. This is used to determine whether the
222 // 'pragma clang fp eval_method' was used whithin the current scope.
223 SourceLocation LastFPEvalPragmaLocation;
224
225 LangOptions::FPEvalMethodKind TUFPEvalMethod =
227
228 // Next __COUNTER__ value, starts at 0.
229 unsigned CounterValue = 0;
230
231 enum {
232 /// Maximum depth of \#includes.
233 MaxAllowedIncludeStackDepth = 200
234 };
235
236 // State that is set before the preprocessor begins.
237 bool KeepComments : 1;
238 bool KeepMacroComments : 1;
239 bool SuppressIncludeNotFoundError : 1;
240
241 // State that changes while the preprocessor runs:
242 bool InMacroArgs : 1; // True if parsing fn macro invocation args.
243
244 /// Whether the preprocessor owns the header search object.
245 bool OwnsHeaderSearch : 1;
246
247 /// True if macro expansion is disabled.
248 bool DisableMacroExpansion : 1;
249
250 /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
251 /// when parsing preprocessor directives.
252 bool MacroExpansionInDirectivesOverride : 1;
253
254 class ResetMacroExpansionHelper;
255
256 /// Whether we have already loaded macros from the external source.
257 mutable bool ReadMacrosFromExternalSource : 1;
258
259 /// True if pragmas are enabled.
260 bool PragmasEnabled : 1;
261
262 /// True if the current build action is a preprocessing action.
263 bool PreprocessedOutput : 1;
264
265 /// True if we are currently preprocessing a #if or #elif directive
266 bool ParsingIfOrElifDirective;
267
268 /// True if we are pre-expanding macro arguments.
269 bool InMacroArgPreExpansion;
270
271 /// Mapping/lookup information for all identifiers in
272 /// the program, including program keywords.
273 mutable IdentifierTable Identifiers;
274
275 /// This table contains all the selectors in the program.
276 ///
277 /// Unlike IdentifierTable above, this table *isn't* populated by the
278 /// preprocessor. It is declared/expanded here because its role/lifetime is
279 /// conceptually similar to the IdentifierTable. In addition, the current
280 /// control flow (in clang::ParseAST()), make it convenient to put here.
281 ///
282 /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
283 /// the lifetime of the preprocessor.
284 SelectorTable Selectors;
285
286 /// Information about builtins.
287 std::unique_ptr<Builtin::Context> BuiltinInfo;
288
289 /// Tracks all of the pragmas that the client registered
290 /// with this preprocessor.
291 std::unique_ptr<PragmaNamespace> PragmaHandlers;
292
293 /// Pragma handlers of the original source is stored here during the
294 /// parsing of a model file.
295 std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
296
297 /// Tracks all of the comment handlers that the client registered
298 /// with this preprocessor.
299 std::vector<CommentHandler *> CommentHandlers;
300
301 /// Empty line handler.
302 EmptylineHandler *Emptyline = nullptr;
303
304 /// True to avoid tearing down the lexer etc on EOF
305 bool IncrementalProcessing = false;
306
307public:
308 /// The kind of translation unit we are processing.
310
311 /// Returns a pointer into the given file's buffer that's guaranteed
312 /// to be between tokens. The returned pointer is always before \p Start.
313 /// The maximum distance betweenthe returned pointer and \p Start is
314 /// limited by a constant value, but also an implementation detail.
315 /// If no such check point exists, \c nullptr is returned.
316 const char *getCheckPoint(FileID FID, const char *Start) const;
317
318private:
319 /// The code-completion handler.
320 CodeCompletionHandler *CodeComplete = nullptr;
321
322 /// The file that we're performing code-completion for, if any.
323 const FileEntry *CodeCompletionFile = nullptr;
324
325 /// The offset in file for the code-completion point.
326 unsigned CodeCompletionOffset = 0;
327
328 /// The location for the code-completion point. This gets instantiated
329 /// when the CodeCompletionFile gets \#include'ed for preprocessing.
330 SourceLocation CodeCompletionLoc;
331
332 /// The start location for the file of the code-completion point.
333 ///
334 /// This gets instantiated when the CodeCompletionFile gets \#include'ed
335 /// for preprocessing.
336 SourceLocation CodeCompletionFileLoc;
337
338 /// The source location of the \c import contextual keyword we just
339 /// lexed, if any.
340 SourceLocation ModuleImportLoc;
341
342 /// The import path for named module that we're currently processing.
343 SmallVector<IdentifierLoc, 2> NamedModuleImportPath;
344
345 llvm::DenseMap<FileID, SmallVector<const char *>> CheckPoints;
346 unsigned CheckPointCounter = 0;
347
348 /// Whether the import is an `@import` or a standard c++ modules import.
349 bool IsAtImport = false;
350
351 /// Whether the last token we lexed was an '@'.
352 bool LastTokenWasAt = false;
353
354 /// First pp-token source location in current translation unit.
355 SourceLocation FirstPPTokenLoc;
356
357 /// A preprocessor directive tracer to trace whether the preprocessing
358 /// state changed. These changes would mean most semantically observable
359 /// preprocessor state, particularly anything that is order dependent.
360 NoTrivialPPDirectiveTracer *DirTracer = nullptr;
361
362 /// A position within a C++20 import-seq.
363 class StdCXXImportSeq {
364 public:
365 enum State : int {
366 // Positive values represent a number of unclosed brackets.
367 AtTopLevel = 0,
368 AfterTopLevelTokenSeq = -1,
369 AfterExport = -2,
370 AfterImportSeq = -3,
371 };
372
373 StdCXXImportSeq(State S) : S(S) {}
374
375 /// Saw any kind of open bracket.
376 void handleOpenBracket() {
377 S = static_cast<State>(std::max<int>(S, 0) + 1);
378 }
379 /// Saw any kind of close bracket other than '}'.
380 void handleCloseBracket() {
381 S = static_cast<State>(std::max<int>(S, 1) - 1);
382 }
383 /// Saw a close brace.
384 void handleCloseBrace() {
385 handleCloseBracket();
386 if (S == AtTopLevel && !AfterHeaderName)
387 S = AfterTopLevelTokenSeq;
388 }
389 /// Saw a semicolon.
390 void handleSemi() {
391 if (atTopLevel()) {
392 S = AfterTopLevelTokenSeq;
393 AfterHeaderName = false;
394 }
395 }
396
397 /// Saw an 'export' identifier.
398 void handleExport() {
399 if (S == AfterTopLevelTokenSeq)
400 S = AfterExport;
401 else if (S <= 0)
402 S = AtTopLevel;
403 }
404 /// Saw an 'import' identifier.
405 void handleImport() {
406 if (S == AfterTopLevelTokenSeq || S == AfterExport)
407 S = AfterImportSeq;
408 else if (S <= 0)
409 S = AtTopLevel;
410 }
411
412 /// Saw a 'header-name' token; do not recognize any more 'import' tokens
413 /// until we reach a top-level semicolon.
414 void handleHeaderName() {
415 if (S == AfterImportSeq)
416 AfterHeaderName = true;
417 handleMisc();
418 }
419
420 /// Saw any other token.
421 void handleMisc() {
422 if (S <= 0)
423 S = AtTopLevel;
424 }
425
426 bool atTopLevel() { return S <= 0; }
427 bool afterImportSeq() { return S == AfterImportSeq; }
428 bool afterTopLevelSeq() { return S == AfterTopLevelTokenSeq; }
429
430 private:
431 State S;
432 /// Whether we're in the pp-import-suffix following the header-name in a
433 /// pp-import. If so, a close-brace is not sufficient to end the
434 /// top-level-token-seq of an import-seq.
435 bool AfterHeaderName = false;
436 };
437
438 /// Our current position within a C++20 import-seq.
439 StdCXXImportSeq StdCXXImportSeqState = StdCXXImportSeq::AfterTopLevelTokenSeq;
440
441 /// Track whether we are in a Global Module Fragment
442 class TrackGMF {
443 public:
444 enum GMFState : int {
445 GMFActive = 1,
446 MaybeGMF = 0,
447 BeforeGMFIntroducer = -1,
448 GMFAbsentOrEnded = -2,
449 };
450
451 TrackGMF(GMFState S) : S(S) {}
452
453 /// Saw a semicolon.
454 void handleSemi() {
455 // If it is immediately after the first instance of the module keyword,
456 // then that introduces the GMF.
457 if (S == MaybeGMF)
458 S = GMFActive;
459 }
460
461 /// Saw an 'export' identifier.
462 void handleExport() {
463 // The presence of an 'export' keyword always ends or excludes a GMF.
464 S = GMFAbsentOrEnded;
465 }
466
467 /// Saw an 'import' identifier.
468 void handleImport(bool AfterTopLevelTokenSeq) {
469 // If we see this before any 'module' kw, then we have no GMF.
470 if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
471 S = GMFAbsentOrEnded;
472 }
473
474 /// Saw a 'module' identifier.
475 void handleModule(bool AfterTopLevelTokenSeq) {
476 // This was the first module identifier and not preceded by any token
477 // that would exclude a GMF. It could begin a GMF, but only if directly
478 // followed by a semicolon.
479 if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
480 S = MaybeGMF;
481 else
482 S = GMFAbsentOrEnded;
483 }
484
485 /// Saw any other token.
486 void handleMisc() {
487 // We saw something other than ; after the 'module' kw, so not a GMF.
488 if (S == MaybeGMF)
489 S = GMFAbsentOrEnded;
490 }
491
492 bool inGMF() { return S == GMFActive; }
493
494 private:
495 /// Track the transitions into and out of a Global Module Fragment,
496 /// if one is present.
497 GMFState S;
498 };
499
500 TrackGMF TrackGMFState = TrackGMF::BeforeGMFIntroducer;
501
502 /// Track the status of the c++20 module decl.
503 ///
504 /// module-declaration:
505 /// 'export'[opt] 'module' module-name module-partition[opt]
506 /// attribute-specifier-seq[opt] ';'
507 ///
508 /// module-name:
509 /// module-name-qualifier[opt] identifier
510 ///
511 /// module-partition:
512 /// ':' module-name-qualifier[opt] identifier
513 ///
514 /// module-name-qualifier:
515 /// identifier '.'
516 /// module-name-qualifier identifier '.'
517 ///
518 /// Transition state:
519 ///
520 /// NotAModuleDecl --- export ---> FoundExport
521 /// NotAModuleDecl --- module ---> ImplementationCandidate
522 /// FoundExport --- module ---> InterfaceCandidate
523 /// ImplementationCandidate --- Identifier ---> ImplementationCandidate
524 /// ImplementationCandidate --- period ---> ImplementationCandidate
525 /// ImplementationCandidate --- colon ---> ImplementationCandidate
526 /// InterfaceCandidate --- Identifier ---> InterfaceCandidate
527 /// InterfaceCandidate --- period ---> InterfaceCandidate
528 /// InterfaceCandidate --- colon ---> InterfaceCandidate
529 /// ImplementationCandidate --- Semi ---> NamedModuleImplementation
530 /// NamedModuleInterface --- Semi ---> NamedModuleInterface
531 /// NamedModuleImplementation --- Anything ---> NamedModuleImplementation
532 /// NamedModuleInterface --- Anything ---> NamedModuleInterface
533 ///
534 /// FIXME: We haven't handle attribute-specifier-seq here. It may not be bad
535 /// soon since we don't support any module attributes yet.
536 class ModuleDeclSeq {
537 enum ModuleDeclState : int {
538 NotAModuleDecl,
539 FoundExport,
540 InterfaceCandidate,
541 ImplementationCandidate,
542 NamedModuleInterface,
543 NamedModuleImplementation,
544 };
545
546 public:
547 ModuleDeclSeq() = default;
548
549 void handleExport() {
550 if (State == NotAModuleDecl)
551 State = FoundExport;
552 else if (!isNamedModule())
553 reset();
554 }
555
556 void handleModule() {
557 if (State == FoundExport)
558 State = InterfaceCandidate;
559 else if (State == NotAModuleDecl)
560 State = ImplementationCandidate;
561 else if (!isNamedModule())
562 reset();
563 }
564
565 void handleIdentifier(IdentifierInfo *Identifier) {
566 if (isModuleCandidate() && Identifier)
567 Name += Identifier->getName().str();
568 else if (!isNamedModule())
569 reset();
570 }
571
572 void handleColon() {
573 if (isModuleCandidate())
574 Name += ":";
575 else if (!isNamedModule())
576 reset();
577 }
578
579 void handlePeriod() {
580 if (isModuleCandidate())
581 Name += ".";
582 else if (!isNamedModule())
583 reset();
584 }
585
586 void handleSemi() {
587 if (!Name.empty() && isModuleCandidate()) {
588 if (State == InterfaceCandidate)
589 State = NamedModuleInterface;
590 else if (State == ImplementationCandidate)
591 State = NamedModuleImplementation;
592 else
593 llvm_unreachable("Unimaged ModuleDeclState.");
594 } else if (!isNamedModule())
595 reset();
596 }
597
598 void handleMisc() {
599 if (!isNamedModule())
600 reset();
601 }
602
603 bool isModuleCandidate() const {
604 return State == InterfaceCandidate || State == ImplementationCandidate;
605 }
606
607 bool isNamedModule() const {
608 return State == NamedModuleInterface ||
609 State == NamedModuleImplementation;
610 }
611
612 bool isNamedInterface() const { return State == NamedModuleInterface; }
613
614 bool isImplementationUnit() const {
615 return State == NamedModuleImplementation && !getName().contains(':');
616 }
617
618 bool isNotAModuleDecl() const { return State == NotAModuleDecl; }
619
620 StringRef getName() const {
621 assert(isNamedModule() && "Can't get name from a non named module");
622 return Name;
623 }
624
625 StringRef getPrimaryName() const {
626 assert(isNamedModule() && "Can't get name from a non named module");
627 return getName().split(':').first;
628 }
629
630 void reset() {
631 Name.clear();
632 State = NotAModuleDecl;
633 }
634
635 private:
636 ModuleDeclState State = NotAModuleDecl;
637 std::string Name;
638 };
639
640 ModuleDeclSeq ModuleDeclState;
641
642 /// Whether the module import expects an identifier next. Otherwise,
643 /// it expects a '.' or ';'.
644 bool ModuleImportExpectsIdentifier = false;
645
646 /// The identifier and source location of the currently-active
647 /// \#pragma clang arc_cf_code_audited begin.
648 IdentifierLoc PragmaARCCFCodeAuditedInfo;
649
650 /// The source location of the currently-active
651 /// \#pragma clang assume_nonnull begin.
652 SourceLocation PragmaAssumeNonNullLoc;
653
654 /// Set only for preambles which end with an active
655 /// \#pragma clang assume_nonnull begin.
656 ///
657 /// When the preamble is loaded into the main file,
658 /// `PragmaAssumeNonNullLoc` will be set to this to
659 /// replay the unterminated assume_nonnull.
660 SourceLocation PreambleRecordedPragmaAssumeNonNullLoc;
661
662 /// True if we hit the code-completion point.
663 bool CodeCompletionReached = false;
664
665 /// The code completion token containing the information
666 /// on the stem that is to be code completed.
667 IdentifierInfo *CodeCompletionII = nullptr;
668
669 /// Range for the code completion token.
670 SourceRange CodeCompletionTokenRange;
671
672 /// The directory that the main file should be considered to occupy,
673 /// if it does not correspond to a real file (as happens when building a
674 /// module).
675 OptionalDirectoryEntryRef MainFileDir;
676
677 /// The number of bytes that we will initially skip when entering the
678 /// main file, along with a flag that indicates whether skipping this number
679 /// of bytes will place the lexer at the start of a line.
680 ///
681 /// This is used when loading a precompiled preamble.
682 std::pair<int, bool> SkipMainFilePreamble;
683
684 /// Whether we hit an error due to reaching max allowed include depth. Allows
685 /// to avoid hitting the same error over and over again.
686 bool HasReachedMaxIncludeDepth = false;
687
688 /// The number of currently-active calls to Lex.
689 ///
690 /// Lex is reentrant, and asking for an (end-of-phase-4) token can often
691 /// require asking for multiple additional tokens. This counter makes it
692 /// possible for Lex to detect whether it's producing a token for the end
693 /// of phase 4 of translation or for some other situation.
694 unsigned LexLevel = 0;
695
696 /// The number of (LexLevel 0) preprocessor tokens.
697 unsigned TokenCount = 0;
698
699 /// Preprocess every token regardless of LexLevel.
700 bool PreprocessToken = false;
701
702 /// The maximum number of (LexLevel 0) tokens before issuing a -Wmax-tokens
703 /// warning, or zero for unlimited.
704 unsigned MaxTokens = 0;
705 SourceLocation MaxTokensOverrideLoc;
706
707public:
714
720 ElseLoc(ElseLoc) {}
721 };
722
723 using IncludedFilesSet = llvm::DenseSet<const FileEntry *>;
724
725private:
726 friend class ASTReader;
727 friend class MacroArgs;
728
729 class PreambleConditionalStackStore {
730 enum State {
731 Off = 0,
732 Recording = 1,
733 Replaying = 2,
734 };
735
736 public:
737 PreambleConditionalStackStore() = default;
738
739 void startRecording() { ConditionalStackState = Recording; }
740 void startReplaying() { ConditionalStackState = Replaying; }
741 bool isRecording() const { return ConditionalStackState == Recording; }
742 bool isReplaying() const { return ConditionalStackState == Replaying; }
743
744 ArrayRef<PPConditionalInfo> getStack() const {
745 return ConditionalStack;
746 }
747
748 void doneReplaying() {
749 ConditionalStack.clear();
750 ConditionalStackState = Off;
751 }
752
753 void setStack(ArrayRef<PPConditionalInfo> s) {
754 if (!isRecording() && !isReplaying())
755 return;
756 ConditionalStack.clear();
757 ConditionalStack.append(s.begin(), s.end());
758 }
759
760 bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
761
762 bool reachedEOFWhileSkipping() const { return SkipInfo.has_value(); }
763
764 void clearSkipInfo() { SkipInfo.reset(); }
765
766 std::optional<PreambleSkipInfo> SkipInfo;
767
768 private:
769 SmallVector<PPConditionalInfo, 4> ConditionalStack;
770 State ConditionalStackState = Off;
771 } PreambleConditionalStack;
772
773 /// The current top of the stack that we're lexing from if
774 /// not expanding a macro and we are lexing directly from source code.
775 ///
776 /// Only one of CurLexer, or CurTokenLexer will be non-null.
777 std::unique_ptr<Lexer> CurLexer;
778
779 /// The current top of the stack that we're lexing from
780 /// if not expanding a macro.
781 ///
782 /// This is an alias for CurLexer.
783 PreprocessorLexer *CurPPLexer = nullptr;
784
785 /// Used to find the current FileEntry, if CurLexer is non-null
786 /// and if applicable.
787 ///
788 /// This allows us to implement \#include_next and find directory-specific
789 /// properties.
790 ConstSearchDirIterator CurDirLookup = nullptr;
791
792 /// The current macro we are expanding, if we are expanding a macro.
793 ///
794 /// One of CurLexer and CurTokenLexer must be null.
795 std::unique_ptr<TokenLexer> CurTokenLexer;
796
797 /// The kind of lexer we're currently working with.
798 typedef bool (*LexerCallback)(Preprocessor &, Token &);
799 LexerCallback CurLexerCallback = &CLK_Lexer;
800
801 /// If the current lexer is for a submodule that is being built, this
802 /// is that submodule.
803 Module *CurLexerSubmodule = nullptr;
804
805 /// Keeps track of the stack of files currently
806 /// \#included, and macros currently being expanded from, not counting
807 /// CurLexer/CurTokenLexer.
808 struct IncludeStackInfo {
809 LexerCallback CurLexerCallback;
810 Module *TheSubmodule;
811 std::unique_ptr<Lexer> TheLexer;
812 PreprocessorLexer *ThePPLexer;
813 std::unique_ptr<TokenLexer> TheTokenLexer;
814 ConstSearchDirIterator TheDirLookup;
815
816 // The following constructors are completely useless copies of the default
817 // versions, only needed to pacify MSVC.
818 IncludeStackInfo(LexerCallback CurLexerCallback, Module *TheSubmodule,
819 std::unique_ptr<Lexer> &&TheLexer,
820 PreprocessorLexer *ThePPLexer,
821 std::unique_ptr<TokenLexer> &&TheTokenLexer,
822 ConstSearchDirIterator TheDirLookup)
823 : CurLexerCallback(std::move(CurLexerCallback)),
824 TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
825 ThePPLexer(std::move(ThePPLexer)),
826 TheTokenLexer(std::move(TheTokenLexer)),
827 TheDirLookup(std::move(TheDirLookup)) {}
828 };
829 std::vector<IncludeStackInfo> IncludeMacroStack;
830
831 /// Actions invoked when some preprocessor activity is
832 /// encountered (e.g. a file is \#included, etc).
833 std::unique_ptr<PPCallbacks> Callbacks;
834
835 struct MacroExpandsInfo {
836 Token Tok;
837 MacroDefinition MD;
838 SourceRange Range;
839
840 MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
841 : Tok(Tok), MD(MD), Range(Range) {}
842 };
843 SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
844
845 /// Information about a name that has been used to define a module macro.
846 struct ModuleMacroInfo {
847 /// The most recent macro directive for this identifier.
848 MacroDirective *MD;
849
850 /// The active module macros for this identifier.
851 llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
852
853 /// The generation number at which we last updated ActiveModuleMacros.
854 /// \see Preprocessor::VisibleModules.
855 unsigned ActiveModuleMacrosGeneration = 0;
856
857 /// Whether this macro name is ambiguous.
858 bool IsAmbiguous = false;
859
860 /// The module macros that are overridden by this macro.
861 llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
862
863 ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
864 };
865
866 /// The state of a macro for an identifier.
867 class MacroState {
868 mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
869
870 ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
871 const IdentifierInfo *II) const {
872 if (II->isOutOfDate())
873 PP.updateOutOfDateIdentifier(*II);
874 // FIXME: Find a spare bit on IdentifierInfo and store a
875 // HasModuleMacros flag.
876 if (!II->hasMacroDefinition() ||
877 (!PP.getLangOpts().Modules &&
878 !PP.getLangOpts().ModulesLocalVisibility) ||
879 !PP.CurSubmoduleState->VisibleModules.getGeneration())
880 return nullptr;
881
882 auto *Info = dyn_cast_if_present<ModuleMacroInfo *>(State);
883 if (!Info) {
884 Info = new (PP.getPreprocessorAllocator())
885 ModuleMacroInfo(cast<MacroDirective *>(State));
886 State = Info;
887 }
888
889 if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
890 Info->ActiveModuleMacrosGeneration)
891 PP.updateModuleMacroInfo(II, *Info);
892 return Info;
893 }
894
895 public:
896 MacroState() : MacroState(nullptr) {}
897 MacroState(MacroDirective *MD) : State(MD) {}
898
899 MacroState(MacroState &&O) noexcept : State(O.State) {
900 O.State = (MacroDirective *)nullptr;
901 }
902
903 MacroState &operator=(MacroState &&O) noexcept {
904 auto S = O.State;
905 O.State = (MacroDirective *)nullptr;
906 State = S;
907 return *this;
908 }
909
910 ~MacroState() {
911 if (auto *Info = dyn_cast_if_present<ModuleMacroInfo *>(State))
912 Info->~ModuleMacroInfo();
913 }
914
915 MacroDirective *getLatest() const {
916 if (auto *Info = dyn_cast_if_present<ModuleMacroInfo *>(State))
917 return Info->MD;
918 return cast<MacroDirective *>(State);
919 }
920
921 void setLatest(MacroDirective *MD) {
922 if (auto *Info = dyn_cast_if_present<ModuleMacroInfo *>(State))
923 Info->MD = MD;
924 else
925 State = MD;
926 }
927
928 bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
929 auto *Info = getModuleInfo(PP, II);
930 return Info ? Info->IsAmbiguous : false;
931 }
932
933 ArrayRef<ModuleMacro *>
934 getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
935 if (auto *Info = getModuleInfo(PP, II))
936 return Info->ActiveModuleMacros;
937 return {};
938 }
939
940 MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
941 SourceManager &SourceMgr) const {
942 // FIXME: Incorporate module macros into the result of this.
943 if (auto *Latest = getLatest())
944 return Latest->findDirectiveAtLoc(Loc, SourceMgr);
945 return {};
946 }
947
948 void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
949 if (auto *Info = getModuleInfo(PP, II)) {
950 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
951 Info->ActiveModuleMacros.begin(),
952 Info->ActiveModuleMacros.end());
953 Info->ActiveModuleMacros.clear();
954 Info->IsAmbiguous = false;
955 }
956 }
957
958 ArrayRef<ModuleMacro*> getOverriddenMacros() const {
959 if (auto *Info = dyn_cast_if_present<ModuleMacroInfo *>(State))
960 return Info->OverriddenMacros;
961 return {};
962 }
963
964 void setOverriddenMacros(Preprocessor &PP,
965 ArrayRef<ModuleMacro *> Overrides) {
966 auto *Info = dyn_cast_if_present<ModuleMacroInfo *>(State);
967 if (!Info) {
968 if (Overrides.empty())
969 return;
970 Info = new (PP.getPreprocessorAllocator())
971 ModuleMacroInfo(cast<MacroDirective *>(State));
972 State = Info;
973 }
974 Info->OverriddenMacros.clear();
975 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
976 Overrides.begin(), Overrides.end());
977 Info->ActiveModuleMacrosGeneration = 0;
978 }
979 };
980
981 /// For each IdentifierInfo that was associated with a macro, we
982 /// keep a mapping to the history of all macro definitions and #undefs in
983 /// the reverse order (the latest one is in the head of the list).
984 ///
985 /// This mapping lives within the \p CurSubmoduleState.
986 using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
987
988 struct SubmoduleState;
989
990 /// Information about a submodule that we're currently building.
991 struct BuildingSubmoduleInfo {
992 /// The module that we are building.
993 Module *M;
994
995 /// The location at which the module was included.
996 SourceLocation ImportLoc;
997
998 /// Whether we entered this submodule via a pragma.
999 bool IsPragma;
1000
1001 /// The previous SubmoduleState.
1002 SubmoduleState *OuterSubmoduleState;
1003
1004 /// The number of pending module macro names when we started building this.
1005 unsigned OuterPendingModuleMacroNames;
1006
1007 BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
1008 SubmoduleState *OuterSubmoduleState,
1009 unsigned OuterPendingModuleMacroNames)
1010 : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
1011 OuterSubmoduleState(OuterSubmoduleState),
1012 OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
1013 };
1014 SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
1015
1016 /// Information about a submodule's preprocessor state.
1017 struct SubmoduleState {
1018 /// The macros for the submodule.
1019 MacroMap Macros;
1020
1021 /// The set of modules that are visible within the submodule.
1022 VisibleModuleSet VisibleModules;
1023
1024 // FIXME: CounterValue?
1025 // FIXME: PragmaPushMacroInfo?
1026 };
1027 std::map<Module *, SubmoduleState> Submodules;
1028
1029 /// The preprocessor state for preprocessing outside of any submodule.
1030 SubmoduleState NullSubmoduleState;
1031
1032 /// The current submodule state. Will be \p NullSubmoduleState if we're not
1033 /// in a submodule.
1034 SubmoduleState *CurSubmoduleState;
1035
1036 /// The files that have been included.
1037 IncludedFilesSet IncludedFiles;
1038
1039 /// The set of top-level modules that affected preprocessing, but were not
1040 /// imported.
1041 llvm::SmallSetVector<Module *, 2> AffectingClangModules;
1042
1043 /// The set of known macros exported from modules.
1044 llvm::FoldingSet<ModuleMacro> ModuleMacros;
1045
1046 /// The names of potential module macros that we've not yet processed.
1047 llvm::SmallVector<IdentifierInfo *, 32> PendingModuleMacroNames;
1048
1049 /// The list of module macros, for each identifier, that are not overridden by
1050 /// any other module macro.
1051 llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
1052 LeafModuleMacros;
1053
1054 /// Macros that we want to warn because they are not used at the end
1055 /// of the translation unit.
1056 ///
1057 /// We store just their SourceLocations instead of
1058 /// something like MacroInfo*. The benefit of this is that when we are
1059 /// deserializing from PCH, we don't need to deserialize identifier & macros
1060 /// just so that we can report that they are unused, we just warn using
1061 /// the SourceLocations of this set (that will be filled by the ASTReader).
1062 using WarnUnusedMacroLocsTy = llvm::SmallDenseSet<SourceLocation, 32>;
1063 WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
1064
1065 /// This is a pair of an optional message and source location used for pragmas
1066 /// that annotate macros like pragma clang restrict_expansion and pragma clang
1067 /// deprecated. This pair stores the optional message and the location of the
1068 /// annotation pragma for use producing diagnostics and notes.
1069 using MsgLocationPair = std::pair<std::string, SourceLocation>;
1070
1071 struct MacroAnnotationInfo {
1072 SourceLocation Location;
1073 std::string Message;
1074 };
1075
1076 struct MacroAnnotations {
1077 std::optional<MacroAnnotationInfo> DeprecationInfo;
1078 std::optional<MacroAnnotationInfo> RestrictExpansionInfo;
1079 std::optional<SourceLocation> FinalAnnotationLoc;
1080 };
1081
1082 /// Warning information for macro annotations.
1083 llvm::DenseMap<const IdentifierInfo *, MacroAnnotations> AnnotationInfos;
1084
1085 /// A "freelist" of MacroArg objects that can be
1086 /// reused for quick allocation.
1087 MacroArgs *MacroArgCache = nullptr;
1088
1089 /// For each IdentifierInfo used in a \#pragma push_macro directive,
1090 /// we keep a MacroInfo stack used to restore the previous macro value.
1091 llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
1092 PragmaPushMacroInfo;
1093
1094 // Various statistics we track for performance analysis.
1095 unsigned NumDirectives = 0;
1096 unsigned NumDefined = 0;
1097 unsigned NumUndefined = 0;
1098 unsigned NumPragma = 0;
1099 unsigned NumIf = 0;
1100 unsigned NumElse = 0;
1101 unsigned NumEndif = 0;
1102 unsigned NumEnteredSourceFiles = 0;
1103 unsigned MaxIncludeStackDepth = 0;
1104 unsigned NumMacroExpanded = 0;
1105 unsigned NumFnMacroExpanded = 0;
1106 unsigned NumBuiltinMacroExpanded = 0;
1107 unsigned NumFastMacroExpanded = 0;
1108 unsigned NumTokenPaste = 0;
1109 unsigned NumFastTokenPaste = 0;
1110 unsigned NumSkipped = 0;
1111
1112 /// The predefined macros that preprocessor should use from the
1113 /// command line etc.
1114 std::string Predefines;
1115
1116 /// The file ID for the preprocessor predefines.
1117 FileID PredefinesFileID;
1118
1119 /// The file ID for the PCH through header.
1120 FileID PCHThroughHeaderFileID;
1121
1122 /// Whether tokens are being skipped until a #pragma hdrstop is seen.
1123 bool SkippingUntilPragmaHdrStop = false;
1124
1125 /// Whether tokens are being skipped until the through header is seen.
1126 bool SkippingUntilPCHThroughHeader = false;
1127
1128 /// \{
1129 /// Cache of macro expanders to reduce malloc traffic.
1130 enum { TokenLexerCacheSize = 8 };
1131 unsigned NumCachedTokenLexers;
1132 std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
1133 /// \}
1134
1135 /// Keeps macro expanded tokens for TokenLexers.
1136 //
1137 /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
1138 /// going to lex in the cache and when it finishes the tokens are removed
1139 /// from the end of the cache.
1140 SmallVector<Token, 16> MacroExpandedTokens;
1141 std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
1142
1143 /// A record of the macro definitions and expansions that
1144 /// occurred during preprocessing.
1145 ///
1146 /// This is an optional side structure that can be enabled with
1147 /// \c createPreprocessingRecord() prior to preprocessing.
1148 PreprocessingRecord *Record = nullptr;
1149
1150 /// Cached tokens state.
1151 using CachedTokensTy = SmallVector<Token, 1>;
1152
1153 /// Cached tokens are stored here when we do backtracking or
1154 /// lookahead. They are "lexed" by the CachingLex() method.
1155 CachedTokensTy CachedTokens;
1156
1157 /// The position of the cached token that CachingLex() should
1158 /// "lex" next.
1159 ///
1160 /// If it points beyond the CachedTokens vector, it means that a normal
1161 /// Lex() should be invoked.
1162 CachedTokensTy::size_type CachedLexPos = 0;
1163
1164 /// Stack of backtrack positions, allowing nested backtracks.
1165 ///
1166 /// The EnableBacktrackAtThisPos() method pushes a position to
1167 /// indicate where CachedLexPos should be set when the BackTrack() method is
1168 /// invoked (at which point the last position is popped).
1169 std::vector<CachedTokensTy::size_type> BacktrackPositions;
1170
1171 /// Stack of cached tokens/initial number of cached tokens pairs, allowing
1172 /// nested unannotated backtracks.
1173 std::vector<std::pair<CachedTokensTy, CachedTokensTy::size_type>>
1174 UnannotatedBacktrackTokens;
1175
1176 /// True if \p Preprocessor::SkipExcludedConditionalBlock() is running.
1177 /// This is used to guard against calling this function recursively.
1178 ///
1179 /// See comments at the use-site for more context about why it is needed.
1180 bool SkippingExcludedConditionalBlock = false;
1181
1182 /// Keeps track of skipped range mappings that were recorded while skipping
1183 /// excluded conditional directives. It maps the source buffer pointer at
1184 /// the beginning of a skipped block, to the number of bytes that should be
1185 /// skipped.
1186 llvm::DenseMap<const char *, unsigned> RecordedSkippedRanges;
1187
1188 void updateOutOfDateIdentifier(const IdentifierInfo &II) const;
1189
1190public:
1191 Preprocessor(const PreprocessorOptions &PPOpts, DiagnosticsEngine &diags,
1192 const LangOptions &LangOpts, SourceManager &SM,
1193 HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
1194 IdentifierInfoLookup *IILookup = nullptr,
1195 bool OwnsHeaderSearch = false,
1197
1198 ~Preprocessor();
1199
1200 /// Initialize the preprocessor using information about the target.
1201 ///
1202 /// \param Target is owned by the caller and must remain valid for the
1203 /// lifetime of the preprocessor.
1204 /// \param AuxTarget is owned by the caller and must remain valid for
1205 /// the lifetime of the preprocessor.
1206 void Initialize(const TargetInfo &Target,
1207 const TargetInfo *AuxTarget = nullptr);
1208
1209 /// Initialize the preprocessor to parse a model file
1210 ///
1211 /// To parse model files the preprocessor of the original source is reused to
1212 /// preserver the identifier table. However to avoid some duplicate
1213 /// information in the preprocessor some cleanup is needed before it is used
1214 /// to parse model files. This method does that cleanup.
1216
1217 /// Cleanup after model file parsing
1218 void FinalizeForModelFile();
1219
1220 /// Retrieve the preprocessor options used to initialize this preprocessor.
1221 const PreprocessorOptions &getPreprocessorOpts() const { return PPOpts; }
1222
1223 DiagnosticsEngine &getDiagnostics() const { return *Diags; }
1225
1226 const LangOptions &getLangOpts() const { return LangOpts; }
1227 const TargetInfo &getTargetInfo() const { return *Target; }
1228 const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
1229 FileManager &getFileManager() const { return FileMgr; }
1230 SourceManager &getSourceManager() const { return SourceMgr; }
1231 HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
1232
1233 IdentifierTable &getIdentifierTable() { return Identifiers; }
1234 const IdentifierTable &getIdentifierTable() const { return Identifiers; }
1235 SelectorTable &getSelectorTable() { return Selectors; }
1236 Builtin::Context &getBuiltinInfo() { return *BuiltinInfo; }
1237 llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
1238
1240 ExternalSource = Source;
1241 }
1242
1244 return ExternalSource;
1245 }
1246
1247 /// Retrieve the module loader associated with this preprocessor.
1248 ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
1249
1251 return TheModuleLoader.HadFatalFailure;
1252 }
1253
1254 /// Retrieve the number of Directives that have been processed by the
1255 /// Preprocessor.
1256 unsigned getNumDirectives() const {
1257 return NumDirectives;
1258 }
1259
1260 /// True if we are currently preprocessing a #if or #elif directive
1262 return ParsingIfOrElifDirective;
1263 }
1264
1265 /// Control whether the preprocessor retains comments in output.
1266 void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
1267 this->KeepComments = KeepComments | KeepMacroComments;
1268 this->KeepMacroComments = KeepMacroComments;
1269 }
1270
1271 bool getCommentRetentionState() const { return KeepComments; }
1272
1273 void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
1274 bool getPragmasEnabled() const { return PragmasEnabled; }
1275
1277 SuppressIncludeNotFoundError = Suppress;
1278 }
1279
1281 return SuppressIncludeNotFoundError;
1282 }
1283
1284 /// Sets whether the preprocessor is responsible for producing output or if
1285 /// it is producing tokens to be consumed by Parse and Sema.
1286 void setPreprocessedOutput(bool IsPreprocessedOutput) {
1287 PreprocessedOutput = IsPreprocessedOutput;
1288 }
1289
1290 /// Returns true if the preprocessor is responsible for generating output,
1291 /// false if it is producing tokens to be consumed by Parse and Sema.
1292 bool isPreprocessedOutput() const { return PreprocessedOutput; }
1293
1294 /// Return true if we are lexing directly from the specified lexer.
1295 bool isCurrentLexer(const PreprocessorLexer *L) const {
1296 return CurPPLexer == L;
1297 }
1298
1299 /// Return the current lexer being lexed from.
1300 ///
1301 /// Note that this ignores any potentially active macro expansions and _Pragma
1302 /// expansions going on at the time.
1303 PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
1304
1305 /// Return the current file lexer being lexed from.
1306 ///
1307 /// Note that this ignores any potentially active macro expansions and _Pragma
1308 /// expansions going on at the time.
1310
1311 /// Return the submodule owning the file being lexed. This may not be
1312 /// the current module if we have changed modules since entering the file.
1313 Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
1314
1315 /// Returns the FileID for the preprocessor predefines.
1316 FileID getPredefinesFileID() const { return PredefinesFileID; }
1317
1318 /// \{
1319 /// Accessors for preprocessor callbacks.
1320 ///
1321 /// Note that this class takes ownership of any PPCallbacks object given to
1322 /// it.
1323 PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
1324 void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
1325 if (Callbacks)
1326 C = std::make_unique<PPChainedCallbacks>(std::move(C),
1327 std::move(Callbacks));
1328 Callbacks = std::move(C);
1329 }
1330 /// \}
1331
1332 /// Get the number of tokens processed so far.
1333 unsigned getTokenCount() const { return TokenCount; }
1334
1335 /// Get the max number of tokens before issuing a -Wmax-tokens warning.
1336 unsigned getMaxTokens() const { return MaxTokens; }
1337
1339 MaxTokens = Value;
1340 MaxTokensOverrideLoc = Loc;
1341 };
1342
1343 SourceLocation getMaxTokensOverrideLoc() const { return MaxTokensOverrideLoc; }
1344
1345 /// Register a function that would be called on each token in the final
1346 /// expanded token stream.
1347 /// This also reports annotation tokens produced by the parser.
1348 void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
1349 OnToken = std::move(F);
1350 }
1351
1353 GetDependencyDirectives = &Get;
1354 }
1355
1356 void setPreprocessToken(bool Preprocess) { PreprocessToken = Preprocess; }
1357
1358 bool isMacroDefined(StringRef Id) {
1359 return isMacroDefined(&Identifiers.get(Id));
1360 }
1362 return II->hasMacroDefinition() &&
1363 (!getLangOpts().Modules || (bool)getMacroDefinition(II));
1364 }
1365
1366 /// Determine whether II is defined as a macro within the module M,
1367 /// if that is a module that we've already preprocessed. Does not check for
1368 /// macros imported into M.
1370 if (!II->hasMacroDefinition())
1371 return false;
1372 auto I = Submodules.find(M);
1373 if (I == Submodules.end())
1374 return false;
1375 auto J = I->second.Macros.find(II);
1376 if (J == I->second.Macros.end())
1377 return false;
1378 auto *MD = J->second.getLatest();
1379 return MD && MD->isDefined();
1380 }
1381
1383 if (!II->hasMacroDefinition())
1384 return {};
1385
1386 MacroState &S = CurSubmoduleState->Macros[II];
1387 auto *MD = S.getLatest();
1388 while (isa_and_nonnull<VisibilityMacroDirective>(MD))
1389 MD = MD->getPrevious();
1390 return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
1391 S.getActiveModuleMacros(*this, II),
1392 S.isAmbiguous(*this, II));
1393 }
1394
1397 if (!II->hadMacroDefinition())
1398 return {};
1399
1400 MacroState &S = CurSubmoduleState->Macros[II];
1402 if (auto *MD = S.getLatest())
1403 DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
1404 // FIXME: Compute the set of active module macros at the specified location.
1405 return MacroDefinition(DI.getDirective(),
1406 S.getActiveModuleMacros(*this, II),
1407 S.isAmbiguous(*this, II));
1408 }
1409
1410 /// Given an identifier, return its latest non-imported MacroDirective
1411 /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
1413 if (!II->hasMacroDefinition())
1414 return nullptr;
1415
1416 auto *MD = getLocalMacroDirectiveHistory(II);
1417 if (!MD || MD->getDefinition().isUndefined())
1418 return nullptr;
1419
1420 return MD;
1421 }
1422
1423 const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
1424 return const_cast<Preprocessor*>(this)->getMacroInfo(II);
1425 }
1426
1428 if (!II->hasMacroDefinition())
1429 return nullptr;
1430 if (auto MD = getMacroDefinition(II))
1431 return MD.getMacroInfo();
1432 return nullptr;
1433 }
1434
1435 /// Given an identifier, return the latest non-imported macro
1436 /// directive for that identifier.
1437 ///
1438 /// One can iterate over all previous macro directives from the most recent
1439 /// one.
1441
1442 /// Add a directive to the macro directive history for this identifier.
1446 DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
1447 appendMacroDirective(II, MD);
1448 return MD;
1449 }
1451 MacroInfo *MI) {
1452 return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
1453 }
1454
1455 /// Set a MacroDirective that was loaded from a PCH file.
1457 MacroDirective *MD);
1458
1459 /// Register an exported macro for a module and identifier.
1462 ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
1464
1465 /// Get the list of leaf (non-overridden) module macros for a name.
1467 if (II->isOutOfDate())
1468 updateOutOfDateIdentifier(*II);
1469 auto I = LeafModuleMacros.find(II);
1470 if (I != LeafModuleMacros.end())
1471 return I->second;
1472 return {};
1473 }
1474
1475 /// Get the list of submodules that we're currently building.
1477 return BuildingSubmoduleStack;
1478 }
1479
1480 /// \{
1481 /// Iterators for the macro history table. Currently defined macros have
1482 /// IdentifierInfo::hasMacroDefinition() set and an empty
1483 /// MacroInfo::getUndefLoc() at the head of the list.
1484 using macro_iterator = MacroMap::const_iterator;
1485
1486 macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
1487 macro_iterator macro_end(bool IncludeExternalMacros = true) const;
1488
1489 llvm::iterator_range<macro_iterator>
1490 macros(bool IncludeExternalMacros = true) const {
1491 macro_iterator begin = macro_begin(IncludeExternalMacros);
1492 macro_iterator end = macro_end(IncludeExternalMacros);
1493 return llvm::make_range(begin, end);
1494 }
1495
1496 /// \}
1497
1498 /// Mark the given clang module as affecting the current clang module or translation unit.
1500 assert(M->isModuleMapModule());
1501 if (!BuildingSubmoduleStack.empty()) {
1502 if (M != BuildingSubmoduleStack.back().M)
1503 BuildingSubmoduleStack.back().M->AffectingClangModules.insert(M);
1504 } else {
1505 AffectingClangModules.insert(M);
1506 }
1507 }
1508
1509 /// Get the set of top-level clang modules that affected preprocessing, but were not
1510 /// imported.
1512 return AffectingClangModules;
1513 }
1514
1515 /// Mark the file as included.
1516 /// Returns true if this is the first time the file was included.
1518 HeaderInfo.getFileInfo(File).IsLocallyIncluded = true;
1519 return IncludedFiles.insert(File).second;
1520 }
1521
1522 /// Return true if this header has already been included.
1524 HeaderInfo.getFileInfo(File);
1525 return IncludedFiles.count(File);
1526 }
1527
1528 /// Get the set of included files.
1529 IncludedFilesSet &getIncludedFiles() { return IncludedFiles; }
1530 const IncludedFilesSet &getIncludedFiles() const { return IncludedFiles; }
1531
1532 /// Return the name of the macro defined before \p Loc that has
1533 /// spelling \p Tokens. If there are multiple macros with same spelling,
1534 /// return the last one defined.
1536 ArrayRef<TokenValue> Tokens) const;
1537
1538 /// Get the predefines for this processor.
1539 /// Used by some third-party tools to inspect and add predefines (see
1540 /// https://github.com/llvm/llvm-project/issues/57483).
1541 const std::string &getPredefines() const { return Predefines; }
1542
1543 /// Set the predefines for this Preprocessor.
1544 ///
1545 /// These predefines are automatically injected when parsing the main file.
1546 void setPredefines(std::string P) { Predefines = std::move(P); }
1547
1548 /// Return information about the specified preprocessor
1549 /// identifier token.
1550 IdentifierInfo *getIdentifierInfo(StringRef Name) const {
1551 return &Identifiers.get(Name);
1552 }
1553
1554 /// Add the specified pragma handler to this preprocessor.
1555 ///
1556 /// If \p Namespace is non-null, then it is a token required to exist on the
1557 /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1558 void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1560 AddPragmaHandler(StringRef(), Handler);
1561 }
1562
1563 /// Remove the specific pragma handler from this preprocessor.
1564 ///
1565 /// If \p Namespace is non-null, then it should be the namespace that
1566 /// \p Handler was added to. It is an error to remove a handler that
1567 /// has not been registered.
1568 void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1570 RemovePragmaHandler(StringRef(), Handler);
1571 }
1572
1573 /// Install empty handlers for all pragmas (making them ignored).
1574 void IgnorePragmas();
1575
1576 /// Set empty line handler.
1577 void setEmptylineHandler(EmptylineHandler *Handler) { Emptyline = Handler; }
1578
1579 EmptylineHandler *getEmptylineHandler() const { return Emptyline; }
1580
1581 /// Add the specified comment handler to the preprocessor.
1582 void addCommentHandler(CommentHandler *Handler);
1583
1584 /// Remove the specified comment handler.
1585 ///
1586 /// It is an error to remove a handler that has not been registered.
1587 void removeCommentHandler(CommentHandler *Handler);
1588
1589 /// Set the code completion handler to the given object.
1591 CodeComplete = &Handler;
1592 }
1593
1594 /// Retrieve the current code-completion handler.
1596 return CodeComplete;
1597 }
1598
1599 /// Clear out the code completion handler.
1601 CodeComplete = nullptr;
1602 }
1603
1604 /// Hook used by the lexer to invoke the "included file" code
1605 /// completion point.
1606 void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
1607
1608 /// Hook used by the lexer to invoke the "natural language" code
1609 /// completion point.
1611
1612 /// Set the code completion token for filtering purposes.
1614 CodeCompletionII = Filter;
1615 }
1616
1617 /// Set the code completion token range for detecting replacement range later
1618 /// on.
1620 const SourceLocation End) {
1621 CodeCompletionTokenRange = {Start, End};
1622 }
1624 return CodeCompletionTokenRange;
1625 }
1626
1627 /// Get the code completion token for filtering purposes.
1629 if (CodeCompletionII)
1630 return CodeCompletionII->getName();
1631 return {};
1632 }
1633
1634 /// Retrieve the preprocessing record, or NULL if there is no
1635 /// preprocessing record.
1637
1638 /// Create a new preprocessing record, which will keep track of
1639 /// all macro expansions, macro definitions, etc.
1641
1642 /// Returns true if the FileEntry is the PCH through header.
1643 bool isPCHThroughHeader(const FileEntry *FE);
1644
1645 /// True if creating a PCH with a through header.
1647
1648 /// True if using a PCH with a through header.
1650
1651 /// True if creating a PCH with a #pragma hdrstop.
1653
1654 /// True if using a PCH with a #pragma hdrstop.
1656
1657 /// Skip tokens until after the #include of the through header or
1658 /// until after a #pragma hdrstop.
1660
1661 /// Process directives while skipping until the through header or
1662 /// #pragma hdrstop is found.
1664 SourceLocation HashLoc);
1665
1666 /// Enter the specified FileID as the main source file,
1667 /// which implicitly adds the builtin defines etc.
1668 void EnterMainSourceFile();
1669
1670 /// Inform the preprocessor callbacks that processing is complete.
1671 void EndSourceFile();
1672
1673 /// Add a source file to the top of the include stack and
1674 /// start lexing tokens from it instead of the current buffer.
1675 ///
1676 /// Emits a diagnostic, doesn't enter the file, and returns true on error.
1678 SourceLocation Loc, bool IsFirstIncludeOfFile = true);
1679
1680 /// Add a Macro to the top of the include stack and start lexing
1681 /// tokens from it instead of the current buffer.
1682 ///
1683 /// \param Args specifies the tokens input to a function-like macro.
1684 /// \param ILEnd specifies the location of the ')' for a function-like macro
1685 /// or the identifier for an object-like macro.
1686 void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro,
1687 MacroArgs *Args);
1688
1689private:
1690 /// Add a "macro" context to the top of the include stack,
1691 /// which will cause the lexer to start returning the specified tokens.
1692 ///
1693 /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1694 /// will not be subject to further macro expansion. Otherwise, these tokens
1695 /// will be re-macro-expanded when/if expansion is enabled.
1696 ///
1697 /// If \p OwnsTokens is false, this method assumes that the specified stream
1698 /// of tokens has a permanent owner somewhere, so they do not need to be
1699 /// copied. If it is true, it assumes the array of tokens is allocated with
1700 /// \c new[] and the Preprocessor will delete[] it.
1701 ///
1702 /// If \p IsReinject the resulting tokens will have Token::IsReinjected flag
1703 /// set, see the flag documentation for details.
1704 void EnterTokenStream(const Token *Toks, unsigned NumToks,
1705 bool DisableMacroExpansion, bool OwnsTokens,
1706 bool IsReinject);
1707
1708public:
1709 void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1710 bool DisableMacroExpansion, bool IsReinject) {
1711 EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true,
1712 IsReinject);
1713 }
1714
1715 void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion,
1716 bool IsReinject) {
1717 EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false,
1718 IsReinject);
1719 }
1720
1721 /// Pop the current lexer/macro exp off the top of the lexer stack.
1722 ///
1723 /// This should only be used in situations where the current state of the
1724 /// top-of-stack lexer is known.
1725 void RemoveTopOfLexerStack();
1726
1727 /// From the point that this method is called, and until
1728 /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1729 /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1730 /// make the Preprocessor re-lex the same tokens.
1731 ///
1732 /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1733 /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1734 /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1735 ///
1736 /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1737 /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1738 /// tokens will continue indefinitely.
1739 ///
1740 /// \param Unannotated Whether token annotations are reverted upon calling
1741 /// Backtrack().
1742 void EnableBacktrackAtThisPos(bool Unannotated = false);
1743
1744private:
1745 std::pair<CachedTokensTy::size_type, bool> LastBacktrackPos();
1746
1747 CachedTokensTy PopUnannotatedBacktrackTokens();
1748
1749public:
1750 /// Disable the last EnableBacktrackAtThisPos call.
1752
1753 /// Make Preprocessor re-lex the tokens that were lexed since
1754 /// EnableBacktrackAtThisPos() was previously called.
1755 void Backtrack();
1756
1757 /// True if EnableBacktrackAtThisPos() was called and
1758 /// caching of tokens is on.
1759 bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1760
1761 /// True if EnableBacktrackAtThisPos() was called and
1762 /// caching of unannotated tokens is on.
1764 return !UnannotatedBacktrackTokens.empty();
1765 }
1766
1767 /// Lex the next token for this preprocessor.
1768 void Lex(Token &Result);
1769
1770 /// Lex all tokens for this preprocessor until (and excluding) end of file.
1771 void LexTokensUntilEOF(std::vector<Token> *Tokens = nullptr);
1772
1773 /// Lex a token, forming a header-name token if possible.
1774 bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
1775
1776 /// Lex the parameters for an #embed directive, returns nullopt on error.
1777 std::optional<LexEmbedParametersResult> LexEmbedParameters(Token &Current,
1778 bool ForHasEmbed);
1779
1780 /// Get the start location of the first pp-token in main file.
1782 assert(FirstPPTokenLoc.isValid() &&
1783 "Did not see the first pp-token in the main file");
1784 return FirstPPTokenLoc;
1785 }
1786
1789
1791 bool IncludeExports = true);
1792
1794 return CurSubmoduleState->VisibleModules.getImportLoc(M);
1795 }
1796
1797 /// Lex a string literal, which may be the concatenation of multiple
1798 /// string literals and may even come from macro expansion.
1799 /// \returns true on success, false if a error diagnostic has been generated.
1800 bool LexStringLiteral(Token &Result, std::string &String,
1801 const char *DiagnosticTag, bool AllowMacroExpansion) {
1802 if (AllowMacroExpansion)
1803 Lex(Result);
1804 else
1806 return FinishLexStringLiteral(Result, String, DiagnosticTag,
1807 AllowMacroExpansion);
1808 }
1809
1810 /// Complete the lexing of a string literal where the first token has
1811 /// already been lexed (see LexStringLiteral).
1812 bool FinishLexStringLiteral(Token &Result, std::string &String,
1813 const char *DiagnosticTag,
1814 bool AllowMacroExpansion);
1815
1816 /// Lex a token. If it's a comment, keep lexing until we get
1817 /// something not a comment.
1818 ///
1819 /// This is useful in -E -C mode where comments would foul up preprocessor
1820 /// directive handling.
1822 do
1823 Lex(Result);
1824 while (Result.getKind() == tok::comment);
1825 }
1826
1827 /// Just like Lex, but disables macro expansion of identifier tokens.
1829 // Disable macro expansion.
1830 bool OldVal = DisableMacroExpansion;
1831 DisableMacroExpansion = true;
1832 // Lex the token.
1833 Lex(Result);
1834
1835 // Reenable it.
1836 DisableMacroExpansion = OldVal;
1837 }
1838
1839 /// Like LexNonComment, but this disables macro expansion of
1840 /// identifier tokens.
1842 do
1844 while (Result.getKind() == tok::comment);
1845 }
1846
1847 /// Parses a simple integer literal to get its numeric value. Floating
1848 /// point literals and user defined literals are rejected. Used primarily to
1849 /// handle pragmas that accept integer arguments.
1850 bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1851
1852 /// Disables macro expansion everywhere except for preprocessor directives.
1854 DisableMacroExpansion = true;
1855 MacroExpansionInDirectivesOverride = true;
1856 }
1857
1859 DisableMacroExpansion = MacroExpansionInDirectivesOverride = false;
1860 }
1861
1862 /// Peeks ahead N tokens and returns that token without consuming any
1863 /// tokens.
1864 ///
1865 /// LookAhead(0) returns the next token that would be returned by Lex(),
1866 /// LookAhead(1) returns the token after it, etc. This returns normal
1867 /// tokens after phase 5. As such, it is equivalent to using
1868 /// 'Lex', not 'LexUnexpandedToken'.
1869 const Token &LookAhead(unsigned N) {
1870 assert(LexLevel == 0 && "cannot use lookahead while lexing");
1871 if (CachedLexPos + N < CachedTokens.size())
1872 return CachedTokens[CachedLexPos+N];
1873 else
1874 return PeekAhead(N+1);
1875 }
1876
1877 /// When backtracking is enabled and tokens are cached,
1878 /// this allows to revert a specific number of tokens.
1879 ///
1880 /// Note that the number of tokens being reverted should be up to the last
1881 /// backtrack position, not more.
1882 void RevertCachedTokens(unsigned N) {
1883 assert(isBacktrackEnabled() &&
1884 "Should only be called when tokens are cached for backtracking");
1885 assert(signed(CachedLexPos) - signed(N) >=
1886 signed(LastBacktrackPos().first) &&
1887 "Should revert tokens up to the last backtrack position, not more");
1888 assert(signed(CachedLexPos) - signed(N) >= 0 &&
1889 "Corrupted backtrack positions ?");
1890 CachedLexPos -= N;
1891 }
1892
1893 /// Enters a token in the token stream to be lexed next.
1894 ///
1895 /// If BackTrack() is called afterwards, the token will remain at the
1896 /// insertion point.
1897 /// If \p IsReinject is true, resulting token will have Token::IsReinjected
1898 /// flag set. See the flag documentation for details.
1899 void EnterToken(const Token &Tok, bool IsReinject) {
1900 if (LexLevel) {
1901 // It's not correct in general to enter caching lex mode while in the
1902 // middle of a nested lexing action.
1903 auto TokCopy = std::make_unique<Token[]>(1);
1904 TokCopy[0] = Tok;
1905 EnterTokenStream(std::move(TokCopy), 1, true, IsReinject);
1906 } else {
1907 EnterCachingLexMode();
1908 assert(IsReinject && "new tokens in the middle of cached stream");
1909 CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1910 }
1911 }
1912
1913 /// We notify the Preprocessor that if it is caching tokens (because
1914 /// backtrack is enabled) it should replace the most recent cached tokens
1915 /// with the given annotation token. This function has no effect if
1916 /// backtracking is not enabled.
1917 ///
1918 /// Note that the use of this function is just for optimization, so that the
1919 /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1920 /// invoked.
1921 void AnnotateCachedTokens(const Token &Tok) {
1922 assert(Tok.isAnnotation() && "Expected annotation token");
1923 if (CachedLexPos != 0 && isBacktrackEnabled())
1924 AnnotatePreviousCachedTokens(Tok);
1925 }
1926
1927 /// Get the location of the last cached token, suitable for setting the end
1928 /// location of an annotation token.
1930 assert(CachedLexPos != 0);
1931 return CachedTokens[CachedLexPos-1].getLastLoc();
1932 }
1933
1934 /// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1935 /// CachedTokens.
1936 bool IsPreviousCachedToken(const Token &Tok) const;
1937
1938 /// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1939 /// in \p NewToks.
1940 ///
1941 /// Useful when a token needs to be split in smaller ones and CachedTokens
1942 /// most recent token must to be updated to reflect that.
1944
1945 /// Replace the last token with an annotation token.
1946 ///
1947 /// Like AnnotateCachedTokens(), this routine replaces an
1948 /// already-parsed (and resolved) token with an annotation
1949 /// token. However, this routine only replaces the last token with
1950 /// the annotation token; it does not affect any other cached
1951 /// tokens. This function has no effect if backtracking is not
1952 /// enabled.
1954 assert(Tok.isAnnotation() && "Expected annotation token");
1955 if (CachedLexPos != 0 && isBacktrackEnabled())
1956 CachedTokens[CachedLexPos-1] = Tok;
1957 }
1958
1959 /// Enter an annotation token into the token stream.
1961 void *AnnotationVal);
1962
1963 /// Determine whether it's possible for a future call to Lex to produce an
1964 /// annotation token created by a previous call to EnterAnnotationToken.
1966 return CurLexerCallback != CLK_Lexer;
1967 }
1968
1969 /// Update the current token to represent the provided
1970 /// identifier, in order to cache an action performed by typo correction.
1971 void TypoCorrectToken(const Token &Tok) {
1972 assert(Tok.getIdentifierInfo() && "Expected identifier token");
1973 if (CachedLexPos != 0 && isBacktrackEnabled())
1974 CachedTokens[CachedLexPos-1] = Tok;
1975 }
1976
1977 /// Recompute the current lexer kind based on the CurLexer/
1978 /// CurTokenLexer pointers.
1979 void recomputeCurLexerKind();
1980
1981 /// Returns true if incremental processing is enabled
1982 bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1983
1984 /// Enables the incremental processing
1985 void enableIncrementalProcessing(bool value = true) {
1986 IncrementalProcessing = value;
1987 }
1988
1989 /// Specify the point at which code-completion will be performed.
1990 ///
1991 /// \param File the file in which code completion should occur. If
1992 /// this file is included multiple times, code-completion will
1993 /// perform completion the first time it is included. If NULL, this
1994 /// function clears out the code-completion point.
1995 ///
1996 /// \param Line the line at which code completion should occur
1997 /// (1-based).
1998 ///
1999 /// \param Column the column at which code completion should occur
2000 /// (1-based).
2001 ///
2002 /// \returns true if an error occurred, false otherwise.
2004 unsigned Column);
2005
2006 /// Determine if we are performing code completion.
2007 bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
2008
2009 /// Returns the location of the code-completion point.
2010 ///
2011 /// Returns an invalid location if code-completion is not enabled or the file
2012 /// containing the code-completion point has not been lexed yet.
2013 SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
2014
2015 /// Returns the start location of the file of code-completion point.
2016 ///
2017 /// Returns an invalid location if code-completion is not enabled or the file
2018 /// containing the code-completion point has not been lexed yet.
2020 return CodeCompletionFileLoc;
2021 }
2022
2023 /// Returns true if code-completion is enabled and we have hit the
2024 /// code-completion point.
2025 bool isCodeCompletionReached() const { return CodeCompletionReached; }
2026
2027 /// Note that we hit the code-completion point.
2029 assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
2030 CodeCompletionReached = true;
2031 // Silence any diagnostics that occur after we hit the code-completion.
2033 }
2034
2035 /// The location of the currently-active \#pragma clang
2036 /// arc_cf_code_audited begin.
2037 ///
2038 /// Returns an invalid location if there is no such pragma active.
2040 return PragmaARCCFCodeAuditedInfo;
2041 }
2042
2043 /// Set the location of the currently-active \#pragma clang
2044 /// arc_cf_code_audited begin. An invalid location ends the pragma.
2047 PragmaARCCFCodeAuditedInfo = IdentifierLoc(Loc, Ident);
2048 }
2049
2050 /// The location of the currently-active \#pragma clang
2051 /// assume_nonnull begin.
2052 ///
2053 /// Returns an invalid location if there is no such pragma active.
2055 return PragmaAssumeNonNullLoc;
2056 }
2057
2058 /// Set the location of the currently-active \#pragma clang
2059 /// assume_nonnull begin. An invalid location ends the pragma.
2061 PragmaAssumeNonNullLoc = Loc;
2062 }
2063
2064 /// Get the location of the recorded unterminated \#pragma clang
2065 /// assume_nonnull begin in the preamble, if one exists.
2066 ///
2067 /// Returns an invalid location if the premable did not end with
2068 /// such a pragma active or if there is no recorded preamble.
2070 return PreambleRecordedPragmaAssumeNonNullLoc;
2071 }
2072
2073 /// Record the location of the unterminated \#pragma clang
2074 /// assume_nonnull begin in the preamble.
2076 PreambleRecordedPragmaAssumeNonNullLoc = Loc;
2077 }
2078
2079 /// Set the directory in which the main file should be considered
2080 /// to have been found, if it is not a real file.
2081 void setMainFileDir(DirectoryEntryRef Dir) { MainFileDir = Dir; }
2082
2083 /// Instruct the preprocessor to skip part of the main source file.
2084 ///
2085 /// \param Bytes The number of bytes in the preamble to skip.
2086 ///
2087 /// \param StartOfLine Whether skipping these bytes puts the lexer at the
2088 /// start of a line.
2089 void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
2090 SkipMainFilePreamble.first = Bytes;
2091 SkipMainFilePreamble.second = StartOfLine;
2092 }
2093
2094 /// Forwarding function for diagnostics. This emits a diagnostic at
2095 /// the specified Token's location, translating the token's start
2096 /// position in the current buffer into a SourcePosition object for rendering.
2097 DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
2098 return Diags->Report(Loc, DiagID);
2099 }
2100
2101 DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
2102 return Diags->Report(Tok.getLocation(), DiagID);
2103 }
2104
2105 /// Return the 'spelling' of the token at the given
2106 /// location; does not go up to the spelling location or down to the
2107 /// expansion location.
2108 ///
2109 /// \param buffer A buffer which will be used only if the token requires
2110 /// "cleaning", e.g. if it contains trigraphs or escaped newlines
2111 /// \param invalid If non-null, will be set \c true if an error occurs.
2113 SmallVectorImpl<char> &buffer,
2114 bool *invalid = nullptr) const {
2115 return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
2116 }
2117
2118 /// Return the 'spelling' of the Tok token.
2119 ///
2120 /// The spelling of a token is the characters used to represent the token in
2121 /// the source file after trigraph expansion and escaped-newline folding. In
2122 /// particular, this wants to get the true, uncanonicalized, spelling of
2123 /// things like digraphs, UCNs, etc.
2124 ///
2125 /// \param Invalid If non-null, will be set \c true if an error occurs.
2126 std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
2127 return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
2128 }
2129
2130 /// Get the spelling of a token into a preallocated buffer, instead
2131 /// of as an std::string.
2132 ///
2133 /// The caller is required to allocate enough space for the token, which is
2134 /// guaranteed to be at least Tok.getLength() bytes long. The length of the
2135 /// actual result is returned.
2136 ///
2137 /// Note that this method may do two possible things: it may either fill in
2138 /// the buffer specified with characters, or it may *change the input pointer*
2139 /// to point to a constant buffer with the data already in it (avoiding a
2140 /// copy). The caller is not allowed to modify the returned buffer pointer
2141 /// if an internal buffer is returned.
2142 unsigned getSpelling(const Token &Tok, const char *&Buffer,
2143 bool *Invalid = nullptr) const {
2144 return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
2145 }
2146
2147 /// Get the spelling of a token into a SmallVector.
2148 ///
2149 /// Note that the returned StringRef may not point to the
2150 /// supplied buffer if a copy can be avoided.
2151 StringRef getSpelling(const Token &Tok,
2152 SmallVectorImpl<char> &Buffer,
2153 bool *Invalid = nullptr) const;
2154
2155 /// Relex the token at the specified location.
2156 /// \returns true if there was a failure, false on success.
2158 bool IgnoreWhiteSpace = false) {
2159 return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
2160 }
2161
2162 /// Given a Token \p Tok that is a numeric constant with length 1,
2163 /// return the value of constant as an unsigned 8-bit integer.
2164 uint8_t
2166 bool *Invalid = nullptr) const {
2167 assert((Tok.is(tok::numeric_constant) || Tok.is(tok::binary_data)) &&
2168 Tok.getLength() == 1 && "Called on unsupported token");
2169 assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
2170
2171 // If the token is carrying a literal data pointer, just use it.
2172 if (const char *D = Tok.getLiteralData())
2173 return (Tok.getKind() == tok::binary_data) ? *D : *D - '0';
2174
2175 assert(Tok.is(tok::numeric_constant) && "binary data with no data");
2176 // Otherwise, fall back on getCharacterData, which is slower, but always
2177 // works.
2178 return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid) - '0';
2179 }
2180
2181 /// Retrieve the name of the immediate macro expansion.
2182 ///
2183 /// This routine starts from a source location, and finds the name of the
2184 /// macro responsible for its immediate expansion. It looks through any
2185 /// intervening macro argument expansions to compute this. It returns a
2186 /// StringRef that refers to the SourceManager-owned buffer of the source
2187 /// where that macro name is spelled. Thus, the result shouldn't out-live
2188 /// the SourceManager.
2190 return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
2191 }
2192
2193 /// Plop the specified string into a scratch buffer and set the
2194 /// specified token's location and length to it.
2195 ///
2196 /// If specified, the source location provides a location of the expansion
2197 /// point of the token.
2198 void CreateString(StringRef Str, Token &Tok,
2199 SourceLocation ExpansionLocStart = SourceLocation(),
2200 SourceLocation ExpansionLocEnd = SourceLocation());
2201
2202 /// Split the first Length characters out of the token starting at TokLoc
2203 /// and return a location pointing to the split token. Re-lexing from the
2204 /// split token will return the split token rather than the original.
2205 SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
2206
2207 /// Computes the source location just past the end of the
2208 /// token at this source location.
2209 ///
2210 /// This routine can be used to produce a source location that
2211 /// points just past the end of the token referenced by \p Loc, and
2212 /// is generally used when a diagnostic needs to point just after a
2213 /// token where it expected something different that it received. If
2214 /// the returned source location would not be meaningful (e.g., if
2215 /// it points into a macro), this routine returns an invalid
2216 /// source location.
2217 ///
2218 /// \param Offset an offset from the end of the token, where the source
2219 /// location should refer to. The default offset (0) produces a source
2220 /// location pointing just past the end of the token; an offset of 1 produces
2221 /// a source location pointing to the last character in the token, etc.
2223 return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
2224 }
2225
2226 /// Returns true if the given MacroID location points at the first
2227 /// token of the macro expansion.
2228 ///
2229 /// \param MacroBegin If non-null and function returns true, it is set to
2230 /// begin location of the macro.
2232 SourceLocation *MacroBegin = nullptr) const {
2233 return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
2234 MacroBegin);
2235 }
2236
2237 /// Returns true if the given MacroID location points at the last
2238 /// token of the macro expansion.
2239 ///
2240 /// \param MacroEnd If non-null and function returns true, it is set to
2241 /// end location of the macro.
2243 SourceLocation *MacroEnd = nullptr) const {
2244 return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
2245 }
2246
2247 /// Print the token to stderr, used for debugging.
2248 void DumpToken(const Token &Tok, bool DumpFlags = false) const;
2249 void DumpLocation(SourceLocation Loc) const;
2250 void DumpMacro(const MacroInfo &MI) const;
2251 void dumpMacroInfo(const IdentifierInfo *II);
2252
2253 /// Given a location that specifies the start of a
2254 /// token, return a new location that specifies a character within the token.
2256 unsigned Char) const {
2257 return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
2258 }
2259
2260 /// Increment the counters for the number of token paste operations
2261 /// performed.
2262 ///
2263 /// If fast was specified, this is a 'fast paste' case we handled.
2264 void IncrementPasteCounter(bool isFast) {
2265 if (isFast)
2266 ++NumFastTokenPaste;
2267 else
2268 ++NumTokenPaste;
2269 }
2270
2271 void PrintStats();
2272
2273 size_t getTotalMemory() const;
2274
2275 /// When the macro expander pastes together a comment (/##/) in Microsoft
2276 /// mode, this method handles updating the current state, returning the
2277 /// token on the next source line.
2279
2280 //===--------------------------------------------------------------------===//
2281 // Preprocessor callback methods. These are invoked by a lexer as various
2282 // directives and events are found.
2283
2284 /// Given a tok::raw_identifier token, look up the
2285 /// identifier information for the token and install it into the token,
2286 /// updating the token kind accordingly.
2288
2289private:
2290 llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
2291
2292public:
2293 /// Specifies the reason for poisoning an identifier.
2294 ///
2295 /// If that identifier is accessed while poisoned, then this reason will be
2296 /// used instead of the default "poisoned" diagnostic.
2297 void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
2298
2299 /// Display reason for poisoned identifier.
2301
2303 if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
2304 if(II->isPoisoned()) {
2306 }
2307 }
2308 }
2309
2310 /// Check whether the next pp-token is one of the specificed token kind. this
2311 /// method should have no observable side-effect on the lexed tokens.
2312 template <typename... Ts> bool isNextPPTokenOneOf(Ts... Ks) {
2313 static_assert(sizeof...(Ts) > 0,
2314 "requires at least one tok::TokenKind specified");
2315 // Do some quick tests for rejection cases.
2316 std::optional<Token> Val;
2317 if (CurLexer)
2318 Val = CurLexer->peekNextPPToken();
2319 else
2320 Val = CurTokenLexer->peekNextPPToken();
2321
2322 if (!Val) {
2323 // We have run off the end. If it's a source file we don't
2324 // examine enclosing ones (C99 5.1.1.2p4). Otherwise walk up the
2325 // macro stack.
2326 if (CurPPLexer)
2327 return false;
2328 for (const IncludeStackInfo &Entry : llvm::reverse(IncludeMacroStack)) {
2329 if (Entry.TheLexer)
2330 Val = Entry.TheLexer->peekNextPPToken();
2331 else
2332 Val = Entry.TheTokenLexer->peekNextPPToken();
2333
2334 if (Val)
2335 break;
2336
2337 // Ran off the end of a source file?
2338 if (Entry.ThePPLexer)
2339 return false;
2340 }
2341 }
2342
2343 // Okay, we found the token and return. Otherwise we found the end of the
2344 // translation unit.
2345 return Val->isOneOf(Ks...);
2346 }
2347
2348private:
2349 /// Identifiers used for SEH handling in Borland. These are only
2350 /// allowed in particular circumstances
2351 // __except block
2352 IdentifierInfo *Ident__exception_code,
2353 *Ident___exception_code,
2354 *Ident_GetExceptionCode;
2355 // __except filter expression
2356 IdentifierInfo *Ident__exception_info,
2357 *Ident___exception_info,
2358 *Ident_GetExceptionInfo;
2359 // __finally
2360 IdentifierInfo *Ident__abnormal_termination,
2361 *Ident___abnormal_termination,
2362 *Ident_AbnormalTermination;
2363
2364 const char *getCurLexerEndPos();
2365 void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
2366
2367public:
2368 void PoisonSEHIdentifiers(bool Poison = true); // Borland
2369
2370 /// Callback invoked when the lexer reads an identifier and has
2371 /// filled in the tokens IdentifierInfo member.
2372 ///
2373 /// This callback potentially macro expands it or turns it into a named
2374 /// token (like 'for').
2375 ///
2376 /// \returns true if we actually computed a token, false if we need to
2377 /// lex again.
2379
2380 /// Callback invoked when the lexer hits the end of the current file.
2381 ///
2382 /// This either returns the EOF token and returns true, or
2383 /// pops a level off the include stack and returns false, at which point the
2384 /// client should call lex again.
2385 bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
2386
2387 /// Callback invoked when the current TokenLexer hits the end of its
2388 /// token stream.
2390
2391 /// Callback invoked when the lexer sees a # token at the start of a
2392 /// line.
2393 ///
2394 /// This consumes the directive, modifies the lexer/preprocessor state, and
2395 /// advances the lexer(s) so that the next token read is the correct one.
2397
2398 /// Ensure that the next token is a tok::eod token.
2399 ///
2400 /// If not, emit a diagnostic and consume up until the eod.
2401 /// If \p EnableMacros is true, then we consider macros that expand to zero
2402 /// tokens as being ok.
2403 ///
2404 /// \return The location of the end of the directive (the terminating
2405 /// newline).
2406 SourceLocation CheckEndOfDirective(const char *DirType,
2407 bool EnableMacros = false);
2408
2409 /// Read and discard all tokens remaining on the current line until
2410 /// the tok::eod token is found. Returns the range of the skipped tokens.
2412 Token Tmp;
2413 return DiscardUntilEndOfDirective(Tmp);
2414 }
2415
2416 /// Same as above except retains the token that was found.
2418
2419 /// Returns true if the preprocessor has seen a use of
2420 /// __DATE__ or __TIME__ in the file so far.
2421 bool SawDateOrTime() const {
2422 return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
2423 }
2424 unsigned getCounterValue() const { return CounterValue; }
2425 void setCounterValue(unsigned V) { CounterValue = V; }
2426
2428 assert(CurrentFPEvalMethod != LangOptions::FEM_UnsetOnCommandLine &&
2429 "FPEvalMethod should be set either from command line or from the "
2430 "target info");
2431 return CurrentFPEvalMethod;
2432 }
2433
2435 return TUFPEvalMethod;
2436 }
2437
2439 return LastFPEvalPragmaLocation;
2440 }
2441
2445 "FPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2446 // This is the location of the '#pragma float_control" where the
2447 // execution state is modifed.
2448 LastFPEvalPragmaLocation = PragmaLoc;
2449 CurrentFPEvalMethod = Val;
2450 TUFPEvalMethod = Val;
2451 }
2452
2455 "TUPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2456 TUFPEvalMethod = Val;
2457 }
2458
2459 /// Retrieves the module that we're currently building, if any.
2461
2462 /// Retrieves the module whose implementation we're current compiling, if any.
2464
2465 /// If we are preprocessing a named module.
2466 bool isInNamedModule() const { return ModuleDeclState.isNamedModule(); }
2467
2468 /// If we are proprocessing a named interface unit.
2469 /// Note that a module implementation partition is not considered as an
2470 /// named interface unit here although it is importable
2471 /// to ease the parsing.
2473 return ModuleDeclState.isNamedInterface();
2474 }
2475
2476 /// Get the named module name we're preprocessing.
2477 /// Requires we're preprocessing a named module.
2478 StringRef getNamedModuleName() const { return ModuleDeclState.getName(); }
2479
2480 /// If we are implementing an implementation module unit.
2481 /// Note that the module implementation partition is not considered as an
2482 /// implementation unit.
2484 return ModuleDeclState.isImplementationUnit();
2485 }
2486
2487 /// If we're importing a standard C++20 Named Modules.
2489 // NamedModuleImportPath will be non-empty only if we're importing
2490 // Standard C++ named modules.
2491 return !NamedModuleImportPath.empty() && getLangOpts().CPlusPlusModules &&
2492 !IsAtImport;
2493 }
2494
2495 /// Allocate a new MacroInfo object with the provided SourceLocation.
2497
2498 /// Turn the specified lexer token into a fully checked and spelled
2499 /// filename, e.g. as an operand of \#include.
2500 ///
2501 /// The caller is expected to provide a buffer that is large enough to hold
2502 /// the spelling of the filename, but is also expected to handle the case
2503 /// when this method decides to use a different buffer.
2504 ///
2505 /// \returns true if the input filename was in <>'s or false if it was
2506 /// in ""'s.
2507 bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Buffer);
2508
2509 /// Given a "foo" or <foo> reference, look up the indicated file.
2510 ///
2511 /// Returns std::nullopt on failure. \p isAngled indicates whether the file
2512 /// reference is for system \#include's or not (i.e. using <> instead of "").
2514 LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled,
2515 ConstSearchDirIterator FromDir, const FileEntry *FromFile,
2516 ConstSearchDirIterator *CurDir, SmallVectorImpl<char> *SearchPath,
2517 SmallVectorImpl<char> *RelativePath,
2518 ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped,
2519 bool *IsFrameworkFound, bool SkipCache = false,
2520 bool OpenFile = true, bool CacheFailures = true);
2521
2522 /// Given a "Filename" or <Filename> reference, look up the indicated embed
2523 /// resource. \p isAngled indicates whether the file reference is for
2524 /// system \#include's or not (i.e. using <> instead of ""). If \p OpenFile
2525 /// is true, the file looked up is opened for reading, otherwise it only
2526 /// validates that the file exists. Quoted filenames are looked up relative
2527 /// to \p LookupFromFile if it is nonnull.
2528 ///
2529 /// Returns std::nullopt on failure.
2531 LookupEmbedFile(StringRef Filename, bool isAngled, bool OpenFile,
2532 const FileEntry *LookupFromFile = nullptr);
2533
2534 /// Return true if we're in the top-level file, not in a \#include.
2535 bool isInPrimaryFile() const;
2536
2537 /// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
2538 /// followed by EOD. Return true if the token is not a valid on-off-switch.
2540
2541 bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
2542 bool *ShadowFlag = nullptr);
2543
2544 void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
2545 Module *LeaveSubmodule(bool ForPragma);
2546
2547private:
2549
2550 void PushIncludeMacroStack() {
2551 assert(CurLexerCallback != CLK_CachingLexer &&
2552 "cannot push a caching lexer");
2553 IncludeMacroStack.emplace_back(CurLexerCallback, CurLexerSubmodule,
2554 std::move(CurLexer), CurPPLexer,
2555 std::move(CurTokenLexer), CurDirLookup);
2556 CurPPLexer = nullptr;
2557 }
2558
2559 void PopIncludeMacroStack() {
2560 CurLexer = std::move(IncludeMacroStack.back().TheLexer);
2561 CurPPLexer = IncludeMacroStack.back().ThePPLexer;
2562 CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
2563 CurDirLookup = IncludeMacroStack.back().TheDirLookup;
2564 CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
2565 CurLexerCallback = IncludeMacroStack.back().CurLexerCallback;
2566 IncludeMacroStack.pop_back();
2567 }
2568
2569 void PropagateLineStartLeadingSpaceInfo(Token &Result);
2570
2571 /// Determine whether we need to create module macros for #defines in the
2572 /// current context.
2573 bool needModuleMacros() const;
2574
2575 /// Update the set of active module macros and ambiguity flag for a module
2576 /// macro name.
2577 void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
2578
2579 DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
2580 SourceLocation Loc);
2581 UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
2582 VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
2583 bool isPublic);
2584
2585 /// Lex and validate a macro name, which occurs after a
2586 /// \#define or \#undef.
2587 ///
2588 /// \param MacroNameTok Token that represents the name defined or undefined.
2589 /// \param IsDefineUndef Kind if preprocessor directive.
2590 /// \param ShadowFlag Points to flag that is set if macro name shadows
2591 /// a keyword.
2592 ///
2593 /// This emits a diagnostic, sets the token kind to eod,
2594 /// and discards the rest of the macro line if the macro name is invalid.
2595 void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
2596 bool *ShadowFlag = nullptr);
2597
2598 /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
2599 /// entire line) of the macro's tokens and adds them to MacroInfo, and while
2600 /// doing so performs certain validity checks including (but not limited to):
2601 /// - # (stringization) is followed by a macro parameter
2602 /// \param MacroNameTok - Token that represents the macro name
2603 /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
2604 ///
2605 /// Either returns a pointer to a MacroInfo object OR emits a diagnostic and
2606 /// returns a nullptr if an invalid sequence of tokens is encountered.
2607 MacroInfo *ReadOptionalMacroParameterListAndBody(
2608 const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
2609
2610 /// The ( starting an argument list of a macro definition has just been read.
2611 /// Lex the rest of the parameters and the closing ), updating \p MI with
2612 /// what we learn and saving in \p LastTok the last token read.
2613 /// Return true if an error occurs parsing the arg list.
2614 bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
2615
2616 /// Provide a suggestion for a typoed directive. If there is no typo, then
2617 /// just skip suggesting.
2618 ///
2619 /// \param Tok - Token that represents the directive
2620 /// \param Directive - String reference for the directive name
2621 void SuggestTypoedDirective(const Token &Tok, StringRef Directive) const;
2622
2623 /// We just read a \#if or related directive and decided that the
2624 /// subsequent tokens are in the \#if'd out portion of the
2625 /// file. Lex the rest of the file, until we see an \#endif. If \p
2626 /// FoundNonSkipPortion is true, then we have already emitted code for part of
2627 /// this \#if directive, so \#else/\#elif blocks should never be entered. If
2628 /// \p FoundElse is false, then \#else directives are ok, if not, then we have
2629 /// already seen one so a \#else directive is a duplicate. When this returns,
2630 /// the caller can lex the first valid token.
2631 void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
2632 SourceLocation IfTokenLoc,
2633 bool FoundNonSkipPortion, bool FoundElse,
2634 SourceLocation ElseLoc = SourceLocation());
2635
2636 /// Information about the result for evaluating an expression for a
2637 /// preprocessor directive.
2638 struct DirectiveEvalResult {
2639 /// The integral value of the expression.
2640 std::optional<llvm::APSInt> Value;
2641
2642 /// Whether the expression was evaluated as true or not.
2643 bool Conditional;
2644
2645 /// True if the expression contained identifiers that were undefined.
2646 bool IncludedUndefinedIds;
2647
2648 /// The source range for the expression.
2649 SourceRange ExprRange;
2650 };
2651
2652 /// Evaluate an integer constant expression that may occur after a
2653 /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
2654 ///
2655 /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
2656 DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro,
2657 bool CheckForEoD = true);
2658
2659 /// Evaluate an integer constant expression that may occur after a
2660 /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
2661 ///
2662 /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
2663 /// \p EvaluatedDefined will contain the result of whether "defined" appeared
2664 /// in the evaluated expression or not.
2665 DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro,
2666 Token &Tok,
2667 bool &EvaluatedDefined,
2668 bool CheckForEoD = true);
2669
2670 /// Process a '__has_embed("path" [, ...])' expression.
2671 ///
2672 /// Returns predefined `__STDC_EMBED_*` macro values if
2673 /// successful.
2674 EmbedResult EvaluateHasEmbed(Token &Tok, IdentifierInfo *II);
2675
2676 /// Process a '__has_include("path")' expression.
2677 ///
2678 /// Returns true if successful.
2679 bool EvaluateHasInclude(Token &Tok, IdentifierInfo *II);
2680
2681 /// Process '__has_include_next("path")' expression.
2682 ///
2683 /// Returns true if successful.
2684 bool EvaluateHasIncludeNext(Token &Tok, IdentifierInfo *II);
2685
2686 /// Get the directory and file from which to start \#include_next lookup.
2687 std::pair<ConstSearchDirIterator, const FileEntry *>
2688 getIncludeNextStart(const Token &IncludeNextTok) const;
2689
2690 /// Install the standard preprocessor pragmas:
2691 /// \#pragma GCC poison/system_header/dependency and \#pragma once.
2692 void RegisterBuiltinPragmas();
2693
2694 /// RegisterBuiltinMacro - Register the specified identifier in the identifier
2695 /// table and mark it as a builtin macro to be expanded.
2696 IdentifierInfo *RegisterBuiltinMacro(const char *Name) {
2697 // Get the identifier.
2698 IdentifierInfo *Id = getIdentifierInfo(Name);
2699
2700 // Mark it as being a macro that is builtin.
2701 MacroInfo *MI = AllocateMacroInfo(SourceLocation());
2702 MI->setIsBuiltinMacro();
2704 return Id;
2705 }
2706
2707 /// Register builtin macros such as __LINE__ with the identifier table.
2708 void RegisterBuiltinMacros();
2709
2710 /// If an identifier token is read that is to be expanded as a macro, handle
2711 /// it and return the next token as 'Tok'. If we lexed a token, return true;
2712 /// otherwise the caller should lex again.
2713 bool HandleMacroExpandedIdentifier(Token &Identifier, const MacroDefinition &MD);
2714
2715 /// Cache macro expanded tokens for TokenLexers.
2716 //
2717 /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
2718 /// going to lex in the cache and when it finishes the tokens are removed
2719 /// from the end of the cache.
2720 Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2721 ArrayRef<Token> tokens);
2722
2723 void removeCachedMacroExpandedTokensOfLastLexer();
2724
2725 /// After reading "MACRO(", this method is invoked to read all of the formal
2726 /// arguments specified for the macro invocation. Returns null on error.
2727 MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2728 SourceLocation &MacroEnd);
2729
2730 /// If an identifier token is read that is to be expanded
2731 /// as a builtin macro, handle it and return the next token as 'Tok'.
2732 void ExpandBuiltinMacro(Token &Tok);
2733
2734 /// Read a \c _Pragma directive, slice it up, process it, then
2735 /// return the first token after the directive.
2736 /// This assumes that the \c _Pragma token has just been read into \p Tok.
2737 void Handle_Pragma(Token &Tok);
2738
2739 /// Like Handle_Pragma except the pragma text is not enclosed within
2740 /// a string literal.
2741 void HandleMicrosoft__pragma(Token &Tok);
2742
2743 /// Add a lexer to the top of the include stack and
2744 /// start lexing tokens from it instead of the current buffer.
2745 void EnterSourceFileWithLexer(Lexer *TheLexer, ConstSearchDirIterator Dir);
2746
2747 /// Set the FileID for the preprocessor predefines.
2748 void setPredefinesFileID(FileID FID) {
2749 assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
2750 PredefinesFileID = FID;
2751 }
2752
2753 /// Set the FileID for the PCH through header.
2754 void setPCHThroughHeaderFileID(FileID FID);
2755
2756 /// Returns true if we are lexing from a file and not a
2757 /// pragma or a macro.
2758 static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
2759 return L ? !L->isPragmaLexer() : P != nullptr;
2760 }
2761
2762 static bool IsFileLexer(const IncludeStackInfo& I) {
2763 return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2764 }
2765
2766 bool IsFileLexer() const {
2767 return IsFileLexer(CurLexer.get(), CurPPLexer);
2768 }
2769
2770 //===--------------------------------------------------------------------===//
2771 // Standard Library Identification
2772 std::optional<CXXStandardLibraryVersionInfo> CXXStandardLibraryVersion;
2773
2774public:
2775 std::optional<std::uint64_t> getStdLibCxxVersion();
2776 bool NeedsStdLibCxxWorkaroundBefore(std::uint64_t FixedVersion);
2777
2778private:
2779 //===--------------------------------------------------------------------===//
2780 // Caching stuff.
2781 void CachingLex(Token &Result);
2782
2783 bool InCachingLexMode() const {
2784 // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
2785 // that we are past EOF, not that we are in CachingLex mode.
2786 return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2787 }
2788
2789 void EnterCachingLexMode();
2790 void EnterCachingLexModeUnchecked();
2791
2792 void ExitCachingLexMode() {
2793 if (InCachingLexMode())
2795 }
2796
2797 const Token &PeekAhead(unsigned N);
2798 void AnnotatePreviousCachedTokens(const Token &Tok);
2799
2800 //===--------------------------------------------------------------------===//
2801 /// Handle*Directive - implement the various preprocessor directives. These
2802 /// should side-effect the current preprocessor object so that the next call
2803 /// to Lex() will return the appropriate token next.
2804 void HandleLineDirective();
2805 void HandleDigitDirective(Token &Tok);
2806 void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
2807 void HandleIdentSCCSDirective(Token &Tok);
2808 void HandleMacroPublicDirective(Token &Tok);
2809 void HandleMacroPrivateDirective();
2810
2811 /// An additional notification that can be produced by a header inclusion or
2812 /// import to tell the parser what happened.
2813 struct ImportAction {
2814 enum ActionKind {
2815 None,
2816 ModuleBegin,
2817 ModuleImport,
2818 HeaderUnitImport,
2819 SkippedModuleImport,
2820 Failure,
2821 } Kind;
2822 Module *ModuleForHeader = nullptr;
2823
2824 ImportAction(ActionKind AK, Module *Mod = nullptr)
2825 : Kind(AK), ModuleForHeader(Mod) {
2826 assert((AK == None || Mod || AK == Failure) &&
2827 "no module for module action");
2828 }
2829 };
2830
2831 OptionalFileEntryRef LookupHeaderIncludeOrImport(
2832 ConstSearchDirIterator *CurDir, StringRef &Filename,
2833 SourceLocation FilenameLoc, CharSourceRange FilenameRange,
2834 const Token &FilenameTok, bool &IsFrameworkFound, bool IsImportDecl,
2835 bool &IsMapped, ConstSearchDirIterator LookupFrom,
2836 const FileEntry *LookupFromFile, StringRef &LookupFilename,
2837 SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
2838 ModuleMap::KnownHeader &SuggestedModule, bool isAngled);
2839 // Binary data inclusion
2840 void HandleEmbedDirective(SourceLocation HashLoc, Token &Tok,
2841 const FileEntry *LookupFromFile = nullptr);
2842 void HandleEmbedDirectiveImpl(SourceLocation HashLoc,
2843 const LexEmbedParametersResult &Params,
2844 StringRef BinaryContents, StringRef FileName);
2845
2846 // File inclusion.
2847 void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
2848 ConstSearchDirIterator LookupFrom = nullptr,
2849 const FileEntry *LookupFromFile = nullptr);
2850 ImportAction
2851 HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok,
2852 Token &FilenameTok, SourceLocation EndLoc,
2853 ConstSearchDirIterator LookupFrom = nullptr,
2854 const FileEntry *LookupFromFile = nullptr);
2855 void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2856 void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2857 void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2858 void HandleMicrosoftImportDirective(Token &Tok);
2859
2860public:
2861 /// Check that the given module is available, producing a diagnostic if not.
2862 /// \return \c true if the check failed (because the module is not available).
2863 /// \c false if the module appears to be usable.
2864 static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2865 const TargetInfo &TargetInfo,
2866 const Module &M, DiagnosticsEngine &Diags);
2867
2868 // Module inclusion testing.
2869 /// Find the module that owns the source or header file that
2870 /// \p Loc points to. If the location is in a file that was included
2871 /// into a module, or is outside any module, returns nullptr.
2872 Module *getModuleForLocation(SourceLocation Loc, bool AllowTextual);
2873
2874 /// We want to produce a diagnostic at location IncLoc concerning an
2875 /// unreachable effect at location MLoc (eg, where a desired entity was
2876 /// declared or defined). Determine whether the right way to make MLoc
2877 /// reachable is by #include, and if so, what header should be included.
2878 ///
2879 /// This is not necessarily fast, and might load unexpected module maps, so
2880 /// should only be called by code that intends to produce an error.
2881 ///
2882 /// \param IncLoc The location at which the missing effect was detected.
2883 /// \param MLoc A location within an unimported module at which the desired
2884 /// effect occurred.
2885 /// \return A file that can be #included to provide the desired effect. Null
2886 /// if no such file could be determined or if a #include is not
2887 /// appropriate (eg, if a module should be imported instead).
2889 SourceLocation MLoc);
2890
2891 bool isRecordingPreamble() const {
2892 return PreambleConditionalStack.isRecording();
2893 }
2894
2895 bool hasRecordedPreamble() const {
2896 return PreambleConditionalStack.hasRecordedPreamble();
2897 }
2898
2900 return PreambleConditionalStack.getStack();
2901 }
2902
2904 PreambleConditionalStack.setStack(s);
2905 }
2906
2908 ArrayRef<PPConditionalInfo> s, std::optional<PreambleSkipInfo> SkipInfo) {
2909 PreambleConditionalStack.startReplaying();
2910 PreambleConditionalStack.setStack(s);
2911 PreambleConditionalStack.SkipInfo = SkipInfo;
2912 }
2913
2914 std::optional<PreambleSkipInfo> getPreambleSkipInfo() const {
2915 return PreambleConditionalStack.SkipInfo;
2916 }
2917
2918private:
2919 /// After processing predefined file, initialize the conditional stack from
2920 /// the preamble.
2921 void replayPreambleConditionalStack();
2922
2923 // Macro handling.
2924 void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard);
2925 void HandleUndefDirective();
2926
2927 // Conditional Inclusion.
2928 void HandleIfdefDirective(Token &Result, const Token &HashToken,
2929 bool isIfndef, bool ReadAnyTokensBeforeDirective);
2930 void HandleIfDirective(Token &IfToken, const Token &HashToken,
2931 bool ReadAnyTokensBeforeDirective);
2932 void HandleEndifDirective(Token &EndifToken);
2933 void HandleElseDirective(Token &Result, const Token &HashToken);
2934 void HandleElifFamilyDirective(Token &ElifToken, const Token &HashToken,
2936
2937 // Pragmas.
2938 void HandlePragmaDirective(PragmaIntroducer Introducer);
2939
2940public:
2941 void HandlePragmaOnce(Token &OnceTok);
2942 void HandlePragmaMark(Token &MarkTok);
2943 void HandlePragmaPoison();
2944 void HandlePragmaSystemHeader(Token &SysHeaderTok);
2945 void HandlePragmaDependency(Token &DependencyTok);
2946 void HandlePragmaPushMacro(Token &Tok);
2947 void HandlePragmaPopMacro(Token &Tok);
2949 void HandlePragmaModuleBuild(Token &Tok);
2950 void HandlePragmaHdrstop(Token &Tok);
2952
2953 // Return true and store the first token only if any CommentHandler
2954 // has inserted some tokens and getCommentRetentionState() is false.
2955 bool HandleComment(Token &result, SourceRange Comment);
2956
2957 /// A macro is used, update information about macros that need unused
2958 /// warnings.
2959 void markMacroAsUsed(MacroInfo *MI);
2960
2961 void addMacroDeprecationMsg(const IdentifierInfo *II, std::string Msg,
2962 SourceLocation AnnotationLoc) {
2963 AnnotationInfos[II].DeprecationInfo =
2964 MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2965 }
2966
2967 void addRestrictExpansionMsg(const IdentifierInfo *II, std::string Msg,
2968 SourceLocation AnnotationLoc) {
2969 AnnotationInfos[II].RestrictExpansionInfo =
2970 MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2971 }
2972
2973 void addFinalLoc(const IdentifierInfo *II, SourceLocation AnnotationLoc) {
2974 AnnotationInfos[II].FinalAnnotationLoc = AnnotationLoc;
2975 }
2976
2977 const MacroAnnotations &getMacroAnnotations(const IdentifierInfo *II) const {
2978 return AnnotationInfos.find(II)->second;
2979 }
2980
2982 bool IsIfnDef = false) const {
2983 IdentifierInfo *Info = Identifier.getIdentifierInfo();
2984 if (Info->isDeprecatedMacro())
2985 emitMacroDeprecationWarning(Identifier);
2986
2987 if (Info->isRestrictExpansion() &&
2988 !SourceMgr.isInMainFile(Identifier.getLocation()))
2989 emitRestrictExpansionWarning(Identifier);
2990
2991 if (!IsIfnDef) {
2992 if (Info->getName() == "INFINITY" && getLangOpts().NoHonorInfs)
2993 emitRestrictInfNaNWarning(Identifier, 0);
2994 if (Info->getName() == "NAN" && getLangOpts().NoHonorNaNs)
2995 emitRestrictInfNaNWarning(Identifier, 1);
2996 }
2997 }
2998
3000 const LangOptions &LangOpts,
3001 const TargetInfo &TI);
3002
3004 const PresumedLoc &PLoc,
3005 const LangOptions &LangOpts,
3006 const TargetInfo &TI);
3007
3008private:
3009 void emitMacroDeprecationWarning(const Token &Identifier) const;
3010 void emitRestrictExpansionWarning(const Token &Identifier) const;
3011 void emitFinalMacroWarning(const Token &Identifier, bool IsUndef) const;
3012 void emitRestrictInfNaNWarning(const Token &Identifier,
3013 unsigned DiagSelection) const;
3014
3015 /// This boolean state keeps track if the current scanned token (by this PP)
3016 /// is in an "-Wunsafe-buffer-usage" opt-out region. Assuming PP scans a
3017 /// translation unit in a linear order.
3018 bool InSafeBufferOptOutRegion = false;
3019
3020 /// Hold the start location of the current "-Wunsafe-buffer-usage" opt-out
3021 /// region if PP is currently in such a region. Hold undefined value
3022 /// otherwise.
3023 SourceLocation CurrentSafeBufferOptOutStart; // It is used to report the start location of an never-closed region.
3024
3025 using SafeBufferOptOutRegionsTy =
3027 // An ordered sequence of "-Wunsafe-buffer-usage" opt-out regions in this
3028 // translation unit. Each region is represented by a pair of start and
3029 // end locations.
3030 SafeBufferOptOutRegionsTy SafeBufferOptOutMap;
3031
3032 // The "-Wunsafe-buffer-usage" opt-out regions in loaded ASTs. We use the
3033 // following structure to manage them by their ASTs.
3034 struct {
3035 // A map from unique IDs to region maps of loaded ASTs. The ID identifies a
3036 // loaded AST. See `SourceManager::getUniqueLoadedASTID`.
3037 llvm::DenseMap<FileID, SafeBufferOptOutRegionsTy> LoadedRegions;
3038
3039 // Returns a reference to the safe buffer opt-out regions of the loaded
3040 // AST where `Loc` belongs to. (Construct if absent)
3042 findAndConsLoadedOptOutMap(SourceLocation Loc, SourceManager &SrcMgr) {
3044 }
3045
3046 // Returns a reference to the safe buffer opt-out regions of the loaded
3047 // AST where `Loc` belongs to. (This const function returns nullptr if
3048 // absent.)
3049 const SafeBufferOptOutRegionsTy *
3050 lookupLoadedOptOutMap(SourceLocation Loc,
3051 const SourceManager &SrcMgr) const {
3052 FileID FID = SrcMgr.getUniqueLoadedASTFileID(Loc);
3053 auto Iter = LoadedRegions.find(FID);
3054
3055 if (Iter == LoadedRegions.end())
3056 return nullptr;
3057 return &Iter->getSecond();
3058 }
3059 } LoadedSafeBufferOptOutMap;
3060
3061public:
3062 /// \return true iff the given `Loc` is in a "-Wunsafe-buffer-usage" opt-out
3063 /// region. This `Loc` must be a source location that has been pre-processed.
3064 bool isSafeBufferOptOut(const SourceManager&SourceMgr, const SourceLocation &Loc) const;
3065
3066 /// Alter the state of whether this PP currently is in a
3067 /// "-Wunsafe-buffer-usage" opt-out region.
3068 ///
3069 /// \param isEnter true if this PP is entering a region; otherwise, this PP
3070 /// is exiting a region
3071 /// \param Loc the location of the entry or exit of a
3072 /// region
3073 /// \return true iff it is INVALID to enter or exit a region, i.e.,
3074 /// attempt to enter a region before exiting a previous region, or exiting a
3075 /// region that PP is not currently in.
3076 bool enterOrExitSafeBufferOptOutRegion(bool isEnter,
3077 const SourceLocation &Loc);
3078
3079 /// \return true iff this PP is currently in a "-Wunsafe-buffer-usage"
3080 /// opt-out region
3082
3083 /// \param StartLoc output argument. It will be set to the start location of
3084 /// the current "-Wunsafe-buffer-usage" opt-out region iff this function
3085 /// returns true.
3086 /// \return true iff this PP is currently in a "-Wunsafe-buffer-usage"
3087 /// opt-out region
3088 bool isPPInSafeBufferOptOutRegion(SourceLocation &StartLoc);
3089
3090 /// \return a sequence of SourceLocations representing ordered opt-out regions
3091 /// specified by
3092 /// `\#pragma clang unsafe_buffer_usage begin/end`s of this translation unit.
3093 SmallVector<SourceLocation, 64> serializeSafeBufferOptOutMap() const;
3094
3095 /// \param SrcLocSeqs a sequence of SourceLocations deserialized from a
3096 /// record of code `PP_UNSAFE_BUFFER_USAGE`.
3097 /// \return true iff the `Preprocessor` has been updated; false `Preprocessor`
3098 /// is same as itself before the call.
3100 const SmallVectorImpl<SourceLocation> &SrcLocSeqs);
3101
3102 /// Whether we've seen pp-directives which may have changed the preprocessing
3103 /// state.
3104 bool hasSeenNoTrivialPPDirective() const;
3105
3106private:
3107 /// Helper functions to forward lexing to the actual lexer. They all share the
3108 /// same signature.
3109 static bool CLK_Lexer(Preprocessor &P, Token &Result) {
3110 return P.CurLexer->Lex(Result);
3111 }
3112 static bool CLK_TokenLexer(Preprocessor &P, Token &Result) {
3113 return P.CurTokenLexer->Lex(Result);
3114 }
3115 static bool CLK_CachingLexer(Preprocessor &P, Token &Result) {
3116 P.CachingLex(Result);
3117 return true;
3118 }
3119 static bool CLK_DependencyDirectivesLexer(Preprocessor &P, Token &Result) {
3120 return P.CurLexer->LexDependencyDirectiveToken(Result);
3121 }
3122 static bool CLK_LexAfterModuleImport(Preprocessor &P, Token &Result) {
3123 return P.LexAfterModuleImport(Result);
3124 }
3125};
3126
3127/// Abstract base class that describes a handler that will receive
3128/// source ranges for each of the comments encountered in the source file.
3130public:
3132
3133 // The handler shall return true if it has pushed any tokens
3134 // to be read using e.g. EnterToken or EnterTokenStream.
3135 virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
3136};
3137
3138/// Abstract base class that describes a handler that will receive
3139/// source ranges for empty lines encountered in the source file.
3141public:
3143
3144 // The handler handles empty lines.
3146};
3147
3148/// Helper class to shuttle information about #embed directives from the
3149/// preprocessor to the parser through an annotation token.
3151 StringRef BinaryData;
3152 StringRef FileName;
3153};
3154
3155/// Registry of pragma handlers added by plugins
3156using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
3157
3158} // namespace clang
3159
3160namespace llvm {
3161extern template class CLANG_TEMPLATE_ABI Registry<clang::PragmaHandler>;
3162} // namespace llvm
3163
3164#endif // LLVM_CLANG_LEX_PREPROCESSOR_H
#define V(N, I)
Definition: ASTContext.h:3597
StringRef P
Defines the Diagnostic-related interfaces.
const Decl * D
IndirectLocalPath & Path
enum clang::sema::@1840::IndirectLocalPathEntry::EntryKind Kind
Defines the Diagnostic IDs-related interfaces.
StringRef Filename
Definition: Format.cpp:3177
StringRef Identifier
Definition: Format.cpp:3185
unsigned Iter
Definition: HTMLLogger.cpp:153
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified.
Defines the clang::LangOptions interface.
llvm::MachO::Target Target
Definition: MachO.h:51
llvm::MachO::Record Record
Definition: MachO.h:31
Defines the clang::MacroInfo and clang::MacroDirective classes.
Defines the clang::Module class, which describes a module in the source code.
#define SM(sm)
Definition: OffloadArch.cpp:16
Defines the PPCallbacks interface.
uint32_t Id
Definition: SemaARM.cpp:1179
SourceRange Range
Definition: SemaObjC.cpp:753
SourceLocation Loc
Definition: SemaObjC.cpp:754
Defines the clang::SourceLocation class and associated facilities.
Defines the SourceManager interface.
Defines the clang::TokenKind enum and support functions.
__device__ __2f16 float __ockl_bool s
Reads an AST files chain containing the contents of a translation unit.
Definition: ASTReader.h:429
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Definition: Builtins.h:228
Callback handler that receives notifications when performing code completion within the preprocessor.
Abstract base class that describes a handler that will receive source ranges for each of the comments...
virtual bool HandleComment(Preprocessor &PP, SourceRange Comment)=0
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:432
Functor that returns the dependency directives for a given file.
A little helper class used to produce diagnostics.
Definition: Diagnostic.h:1233
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:231
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1529
void setSuppressAllDiagnostics(bool Val)
Suppress all diagnostics, to silence the front end when we know that we don't want any more diagnosti...
Definition: Diagnostic.h:729
A reference to a DirectoryEntry that includes the name of the directory as it was accessed by the Fil...
Abstract base class that describes a handler that will receive source ranges for empty lines encounte...
virtual void HandleEmptyline(SourceRange Range)=0
Abstract interface for external sources of preprocessor information.
A reference to a FileEntry that includes the name of the file as it was accessed by the FileManager's...
Definition: FileEntry.h:57
Cached information about one file (either on disk or in the virtual file system).
Definition: FileEntry.h:306
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Implements support for file system lookup, file system caching, and directory search management.
Definition: FileManager.h:53
Encapsulates the information needed to find the file referenced by a #include or #include_next,...
Definition: HeaderSearch.h:237
HeaderFileInfo & getFileInfo(FileEntryRef FE)
Return the HeaderFileInfo structure for the specified FileEntry, in preparation for updating it in so...
One of these records is kept for each identifier that is lexed.
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
bool isDeprecatedMacro() const
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
StringRef getName() const
Return the actual identifier string.
bool isRestrictExpansion() const
A simple pair of identifier info and location.
Implements an efficient mapping from strings to IdentifierInfo nodes.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
FPEvalMethodKind
Possible float expression evaluation method choices.
Definition: LangOptions.h:239
@ FEM_UnsetOnCommandLine
Used only for FE option processing; this is only used to indicate that the user did not specify an ex...
Definition: LangOptions.h:250
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:434
static StringRef getImmediateMacroName(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts)
Retrieve the name of the immediate macro expansion.
Definition: Lexer.cpp:1056
static bool isAtStartOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroBegin=nullptr)
Returns true if the given MacroID location points at the first token of the macro expansion.
Definition: Lexer.cpp:870
static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Characters, const SourceManager &SM, const LangOptions &LangOpts)
AdvanceToTokenCharacter - If the current SourceLocation specifies a location at the start of a token,...
Definition: Lexer.h:399
static bool isAtEndOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroEnd=nullptr)
Returns true if the given MacroID location points at the last token of the macro expansion.
Definition: Lexer.cpp:892
static unsigned getSpelling(const Token &Tok, const char *&Buffer, const SourceManager &SourceMgr, const LangOptions &LangOpts, bool *Invalid=nullptr)
getSpelling - This method is used to get the spelling of a token into a preallocated buffer,...
Definition: Lexer.cpp:451
static bool getRawToken(SourceLocation Loc, Token &Result, const SourceManager &SM, const LangOptions &LangOpts, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
Definition: Lexer.cpp:509
static SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset, const SourceManager &SM, const LangOptions &LangOpts)
Computes the source location just past the end of the token at this source location.
Definition: Lexer.cpp:848
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:30
A description of the current definition of a macro.
Definition: MacroInfo.h:590
const DefMacroDirective * getDirective() const
Definition: MacroInfo.h:375
Encapsulates changes to the "macros namespace" (the location where the macro name became active,...
Definition: MacroInfo.h:313
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Definition: MacroInfo.h:125
Abstract interface for a module loader.
Definition: ModuleLoader.h:83
Represents a macro directive exported by a module.
Definition: MacroInfo.h:514
A header that is known to reside within a given module, whether it was included or excluded.
Definition: ModuleMap.h:158
Describes a module or submodule.
Definition: Module.h:144
bool isModuleMapModule() const
Definition: Module.h:251
This interface provides a way to observe the actions of the preprocessor as it does its thing.
Definition: PPCallbacks.h:37
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
Definition: Pragma.h:65
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:145
SourceLocation getLastFPEvalPragmaLocation() const
bool isMacroDefined(const IdentifierInfo *II)
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define'd and not #undef'...
bool markIncluded(FileEntryRef File)
Mark the file as included.
void HandlePragmaPushMacro(Token &Tok)
Handle #pragma push_macro.
Definition: Pragma.cpp:633
void FinalizeForModelFile()
Cleanup after model file parsing.
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
void HandlePragmaPoison()
HandlePragmaPoison - Handle #pragma GCC poison. PoisonTok is the 'poison'.
Definition: Pragma.cpp:438
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
void dumpMacroInfo(const IdentifierInfo *II)
void HandlePragmaSystemHeader(Token &SysHeaderTok)
HandlePragmaSystemHeader - Implement #pragma GCC system_header.
Definition: Pragma.cpp:480
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
void MaybeHandlePoisonedIdentifier(Token &Identifier)
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED, MacroDirective *MD)
Set a MacroDirective that was loaded from a PCH file.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
void markClangModuleAsAffecting(Module *M)
Mark the given clang module as affecting the current clang module or translation unit.
void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident, SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin.
void HandlePragmaModuleBuild(Token &Tok)
Definition: Pragma.cpp:810
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
OptionalFileEntryRef LookupEmbedFile(StringRef Filename, bool isAngled, bool OpenFile, const FileEntry *LookupFromFile=nullptr)
Given a "Filename" or <Filename> reference, look up the indicated embed resource.
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
void EnterToken(const Token &Tok, bool IsReinject)
Enters a token in the token stream to be lexed next.
void IgnorePragmas()
Install empty handlers for all pragmas (making them ignored).
Definition: Pragma.cpp:2216
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
PPCallbacks * getPPCallbacks() const
bool isInNamedInterfaceUnit() const
If we are proprocessing a named interface unit.
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
void setPreambleRecordedPragmaAssumeNonNullLoc(SourceLocation Loc)
Record the location of the unterminated #pragma clang assume_nonnull begin in the preamble.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
ArrayRef< BuildingSubmoduleInfo > getBuildingSubmodules() const
Get the list of submodules that we're currently building.
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
SourceRange getCodeCompletionTokenRange() const
SourceLocation getModuleImportLoc(Module *M) const
void overrideMaxTokens(unsigned Value, SourceLocation Loc)
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
bool isRecordingPreamble() const
void HandleSkippedDirectiveWhileUsingPCH(Token &Result, SourceLocation HashLoc)
Process directives while skipping until the through header or #pragma hdrstop is found.
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
bool GetSuppressIncludeNotFoundError()
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we've alread...
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
void markMacroAsUsed(MacroInfo *MI)
A macro is used, update information about macros that need unused warnings.
LangOptions::FPEvalMethodKind getCurrentFPEvalMethod() const
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
bool isSafeBufferOptOut(const SourceManager &SourceMgr, const SourceLocation &Loc) const
void addMacroDeprecationMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
const char * getCheckPoint(FileID FID, const char *Start) const
Returns a pointer into the given file's buffer that's guaranteed to be between tokens.
void addRestrictExpansionMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
MacroDirective * getLocalMacroDirectiveHistory(const IdentifierInfo *II) const
Given an identifier, return the latest non-imported macro directive for that identifier.
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
void addFinalLoc(const IdentifierInfo *II, SourceLocation AnnotationLoc)
bool IsPreviousCachedToken(const Token &Tok) const
Whether Tok is the most recent token (CachedLexPos - 1) in CachedTokens.
Definition: PPCaching.cpp:171
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
const TargetInfo * getAuxTargetInfo() const
void CommitBacktrackedTokens()
Disable the last EnableBacktrackAtThisPos call.
Definition: PPCaching.cpp:56
friend class MacroArgs
Definition: Preprocessor.h:727
void DumpMacro(const MacroInfo &MI) const
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
void setDiagnostics(DiagnosticsEngine &D)
IncludedFilesSet & getIncludedFiles()
Get the set of included files.
friend void TokenLexer::ExpandFunctionArguments()
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool SetCodeCompletionPoint(FileEntryRef File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
StringRef getNamedModuleName() const
Get the named module name we're preprocessing.
unsigned getCounterValue() const
bool mightHavePendingAnnotationTokens()
Determine whether it's possible for a future call to Lex to produce an annotation token created by a ...
bool isInImportingCXXNamedModules() const
If we're importing a standard C++20 Named Modules.
void Lex(Token &Result)
Lex the next token for this preprocessor.
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion, bool IsReinject)
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
Definition: Preprocessor.h:309
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
unsigned getNumDirectives() const
Retrieve the number of Directives that have been processed by the Preprocessor.
bool isInImplementationUnit() const
If we are implementing an implementation module unit.
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
void LexNonComment(Token &Result)
Lex a token.
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
SourceRange DiscardUntilEndOfDirective()
Read and discard all tokens remaining on the current line until the tok::eod token is found.
bool LexOnOffSwitch(tok::OnOffSwitch &Result)
Lex an on-off-switch (C99 6.10.6p2) and verify that it is followed by EOD.
Definition: Pragma.cpp:971
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
void setMainFileDir(DirectoryEntryRef Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
const IdentifierTable & getIdentifierTable() const
void HandlePragmaDependency(Token &DependencyTok)
HandlePragmaDependency - Handle #pragma GCC dependency "foo" blah.
Definition: Pragma.cpp:512
SourceLocation CheckEndOfDirective(const char *DirType, bool EnableMacros=false)
Ensure that the next token is a tok::eod token.
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
void Backtrack()
Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...
Definition: PPCaching.cpp:66
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
bool enterOrExitSafeBufferOptOutRegion(bool isEnter, const SourceLocation &Loc)
Alter the state of whether this PP currently is in a "-Wunsafe-buffer-usage" opt-out region.
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
IdentifierLoc getPragmaARCCFCodeAuditedInfo() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, std::optional< PreambleSkipInfo > SkipInfo)
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
const MacroAnnotations & getMacroAnnotations(const IdentifierInfo *II) const
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
uint8_t getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the value of constant as an unsign...
macro_iterator macro_end(bool IncludeExternalMacros=true) const
SourceManager & getSourceManager() const
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef, bool *ShadowFlag=nullptr)
std::optional< PreambleSkipInfo > getPreambleSkipInfo() const
void setPreprocessToken(bool Preprocess)
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
void HandlePragmaOnce(Token &OnceTok)
HandlePragmaOnce - Handle #pragma once. OnceTok is the 'once'.
Definition: Pragma.cpp:413
EmptylineHandler * getEmptylineHandler() const
bool getCommentRetentionState() const
bool isMacroDefined(StringRef Id)
static bool checkModuleIsAvailable(const LangOptions &LangOpts, const TargetInfo &TargetInfo, const Module &M, DiagnosticsEngine &Diags)
Check that the given module is available, producing a diagnostic if not.
Module * getCurrentModuleImplementation()
Retrieves the module whose implementation we're current compiling, if any.
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
bool hasRecordedPreamble() const
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
bool isNextPPTokenOneOf(Ts... Ks)
Check whether the next pp-token is one of the specificed token kind.
MacroMap::const_iterator macro_iterator
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions,...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
bool isUnannotatedBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of unannotated tokens is on.
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion, bool IsReinject)
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
std::optional< std::uint64_t > getStdLibCxxVersion()
void RemovePragmaHandler(PragmaHandler *Handler)
bool isPPInSafeBufferOptOutRegion()
unsigned getTokenCount() const
Get the number of tokens processed so far.
unsigned getMaxTokens() const
Get the max number of tokens before issuing a -Wmax-tokens warning.
SourceLocation getMaxTokensOverrideLoc() const
void makeModuleVisible(Module *M, SourceLocation Loc, bool IncludeExports=true)
bool hadModuleLoaderFatalFailure() const
static void processPathToFileName(SmallVectorImpl< char > &FileName, const PresumedLoc &PLoc, const LangOptions &LangOpts, const TargetInfo &TI)
void setCurrentFPEvalMethod(SourceLocation PragmaLoc, LangOptions::FPEvalMethodKind Val)
const TargetInfo & getTargetInfo() const
FileManager & getFileManager() const
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the 'spelling' of the Tok token.
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void DumpLocation(SourceLocation Loc) const
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
MacroInfo * AllocateMacroInfo(SourceLocation L)
Allocate a new MacroInfo object with the provided SourceLocation.
void setDependencyDirectivesGetter(DependencyDirectivesGetter &Get)
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
bool alreadyIncluded(FileEntryRef File) const
Return true if this header has already been included.
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Add the specified pragma handler to this preprocessor.
Definition: Pragma.cpp:918
llvm::BumpPtrAllocator & getPreprocessorAllocator()
ModuleMacro * getModuleMacro(Module *Mod, const IdentifierInfo *II)
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool HandleComment(Token &result, SourceRange Comment)
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
bool GetIncludeFilenameSpelling(SourceLocation Loc, StringRef &Buffer)
Turn the specified lexer token into a fully checked and spelled filename, e.g.
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
HeaderSearch & getHeaderSearchInfo() const
void emitMacroExpansionWarnings(const Token &Identifier, bool IsIfnDef=false) const
bool setDeserializedSafeBufferOptOutMap(const SmallVectorImpl< SourceLocation > &SrcLocSeqs)
void HandlePragmaPopMacro(Token &Tok)
Handle #pragma pop_macro.
Definition: Pragma.cpp:656
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
ExternalPreprocessorSource * getExternalSource() const
bool NeedsStdLibCxxWorkaroundBefore(std::uint64_t FixedVersion)
Module * LeaveSubmodule(bool ForPragma)
const std::string & getPredefines() const
Get the predefines for this processor.
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
SmallVector< SourceLocation, 64 > serializeSafeBufferOptOutMap() const
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind, void *AnnotationVal)
Enter an annotation token into the token stream.
void setTokenWatcher(llvm::unique_function< void(const clang::Token &)> F)
Register a function that would be called on each token in the final expanded token stream.
MacroInfo * getMacroInfo(const IdentifierInfo *II)
void setPredefines(std::string P)
Set the predefines for this Preprocessor.
OptionalFileEntryRef LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, ConstSearchDirIterator FromDir, const FileEntry *FromFile, ConstSearchDirIterator *CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false, bool OpenFile=true, bool CacheFailures=true)
Given a "foo" or <foo> reference, look up the indicated file.
IdentifierTable & getIdentifierTable()
Builtin::Context & getBuiltinInfo()
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
const PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
void ReplacePreviousCachedToken(ArrayRef< Token > NewToks)
Replace token in CachedLexPos - 1 in CachedTokens by the tokens in NewToks.
Definition: PPCaching.cpp:189
LangOptions::FPEvalMethodKind getTUFPEvalMethod() const
const LangOptions & getLangOpts() const
void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val)
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void SetSuppressIncludeNotFoundError(bool Suppress)
static void processPathForFileMacro(SmallVectorImpl< char > &Path, const LangOptions &LangOpts, const TargetInfo &TI)
llvm::DenseMap< FileID, SafeBufferOptOutRegionsTy > LoadedRegions
bool isInNamedModule() const
If we are preprocessing a named module.
void EnableBacktrackAtThisPos(bool Unannotated=false)
From the point that this method is called, and until CommitBacktrackedTokens() or Backtrack() is call...
Definition: PPCaching.cpp:34
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
void PoisonSEHIdentifiers(bool Poison=true)
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion.
size_t getTotalMemory() const
void setExternalSource(ExternalPreprocessorSource *Source)
void clearCodeCompletionHandler()
Clear out the code completion handler.
void AddPragmaHandler(PragmaHandler *Handler)
OptionalFileEntryRef getHeaderToIncludeForDiagnostics(SourceLocation IncLoc, SourceLocation MLoc)
We want to produce a diagnostic at location IncLoc concerning an unreachable effect at location MLoc ...
void setCounterValue(unsigned V)
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
IdentifierInfo * ParsePragmaPushOrPopMacro(Token &Tok)
ParsePragmaPushOrPopMacro - Handle parsing of pragma push_macro/pop_macro.
Definition: Pragma.cpp:568
void LexTokensUntilEOF(std::vector< Token > *Tokens=nullptr)
Lex all tokens for this preprocessor until (and excluding) end of file.
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
void setPragmasEnabled(bool Enabled)
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
SourceLocation getMainFileFirstPPTokenLoc() const
Get the start location of the first pp-token in main file.
void HandlePragmaMark(Token &MarkTok)
Definition: Pragma.cpp:428
bool getPragmasEnabled() const
void HandlePragmaHdrstop(Token &Tok)
Definition: Pragma.cpp:884
void SetEnableMacroExpansion()
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
void setEmptylineHandler(EmptylineHandler *Handler)
Set empty line handler.
DiagnosticsEngine & getDiagnostics() const
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
bool hasSeenNoTrivialPPDirective() const
Whether we've seen pp-directives which may have changed the preprocessing state.
llvm::DenseSet< const FileEntry * > IncludedFilesSet
Definition: Preprocessor.h:723
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
SelectorTable & getSelectorTable()
void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Remove the specific pragma handler from this preprocessor.
Definition: Pragma.cpp:949
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
const llvm::SmallSetVector< Module *, 2 > & getAffectingClangModules() const
Get the set of top-level clang modules that affected preprocessing, but were not imported.
std::optional< LexEmbedParametersResult > LexEmbedParameters(Token &Current, bool ForHasEmbed)
Lex the parameters for an #embed directive, returns nullopt on error.
const IncludedFilesSet & getIncludedFiles() const
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void HandlePragmaIncludeAlias(Token &Tok)
Definition: Pragma.cpp:691
Module * getModuleForLocation(SourceLocation Loc, bool AllowTextual)
Find the module that owns the source or header file that Loc points to.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
SourceLocation getPreambleRecordedPragmaAssumeNonNullLoc() const
Get the location of the recorded unterminated #pragma clang assume_nonnull begin in the preamble,...
void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop.
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
Preprocessor(const PreprocessorOptions &PPOpts, DiagnosticsEngine &diags, const LangOptions &LangOpts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Add a directive to the macro directive history for this identifier.
Represents an unpacked "presumed" location which can be presented to the user.
This table allows us to fully hide how we implement multi-keyword caching.
Encodes a location in the source.
bool isValid() const
Return true if this is a valid SourceLocation object.
This class handles loading and caching of source files into memory.
FileID getUniqueLoadedASTFileID(SourceLocation Loc) const
A trivial tuple used to represent a source range.
Exposes information about the current target.
Definition: TargetInfo.h:226
Stores token information for comparing actual tokens with predefined values.
Definition: Preprocessor.h:93
TokenValue(IdentifierInfo *II)
Definition: Preprocessor.h:106
TokenValue(tok::TokenKind Kind)
Definition: Preprocessor.h:98
bool operator==(const Token &Tok) const
Definition: Preprocessor.h:108
Token - This structure provides full information about a lexed token.
Definition: Token.h:36
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:189
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
Definition: Token.h:134
unsigned getLength() const
Definition: Token.h:137
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:102
tok::TokenKind getKind() const
Definition: Token.h:97
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:123
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:297
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:227
A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a variadic mac...
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
Directive - Abstract class representing a parsed verify directive.
#define bool
Definition: gpuintrin.h:32
constexpr XRayInstrMask None
Definition: XRayInstr.h:38
StringRef getName(const HeaderType T)
Definition: HeaderFile.h:38
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:25
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
Definition: TokenKinds.h:56
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
Definition: TokenKinds.h:97
PPKeywordKind
Provides a namespace for preprocessor keywords which start with a '#' at the beginning of the line.
Definition: TokenKinds.h:33
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
Definition: TokenKinds.cpp:58
@ HeaderSearch
Remove unused header search paths including header maps.
The JSON file list parser is used to communicate input to InstallAPI.
CustomizableOptional< DirectoryEntryRef > OptionalDirectoryEntryRef
detail::SearchDirIteratorImpl< true > ConstSearchDirIterator
Definition: HeaderSearch.h:229
MacroUse
Context in which macro name is used.
Definition: Preprocessor.h:115
@ MU_Undef
Definition: Preprocessor.h:123
@ MU_Other
Definition: Preprocessor.h:117
@ MU_Define
Definition: Preprocessor.h:120
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
@ Module
Module linkage, which indicates that the entity can be referred to from other translation units withi...
@ Result
The result type of a method or function.
CustomizableOptional< FileEntryRef > OptionalFileEntryRef
Definition: FileEntry.h:208
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:1097
@ TU_Complete
The translation unit is a complete translation unit.
Definition: LangOptions.h:1099
Diagnostic wrappers for TextAPI types for error reporting.
Definition: Dominators.h:30
Helper class to shuttle information about #embed directives from the preprocessor to the parser throu...
unsigned IsLocallyIncluded
True if this file has been included (or imported) locally.
Definition: HeaderSearch.h:64
Describes how and where the pragma was introduced.
Definition: Pragma.h:51
PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)
Definition: Preprocessor.h:715