54#include "llvm/ADT/APInt.h"
55#include "llvm/ADT/ArrayRef.h"
56#include "llvm/ADT/DenseMap.h"
57#include "llvm/ADT/STLExtras.h"
58#include "llvm/ADT/SmallVector.h"
59#include "llvm/ADT/StringRef.h"
60#include "llvm/Support/Capacity.h"
61#include "llvm/Support/ErrorHandling.h"
62#include "llvm/Support/MemoryBuffer.h"
63#include "llvm/Support/raw_ostream.h"
87 : PPOpts(PPOpts), Diags(&diags), LangOpts(opts),
88 FileMgr(Headers.getFileMgr()), SourceMgr(
SM),
89 ScratchBuf(new
ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
94 Identifiers(IILookup), PragmaHandlers(new
PragmaNamespace(StringRef())),
95 TUKind(TUKind), SkipMainFilePreamble(0,
true),
96 CurSubmoduleState(&NullSubmoduleState) {
97 OwnsHeaderSearch = OwnsHeaders;
100 KeepComments =
false;
101 KeepMacroComments =
false;
102 SuppressIncludeNotFoundError =
false;
105 DisableMacroExpansion =
false;
106 MacroExpansionInDirectivesOverride =
false;
109 InMacroArgPreExpansion =
false;
110 NumCachedTokenLexers = 0;
111 PragmasEnabled =
true;
112 ParsingIfOrElifDirective =
false;
113 PreprocessedOutput =
false;
116 ReadMacrosFromExternalSource =
false;
118 BuiltinInfo = std::make_unique<Builtin::Context>();
128 RegisterBuiltinPragmas();
131 RegisterBuiltinMacros();
133 if(LangOpts.Borland) {
144 Ident__exception_info = Ident__exception_code =
nullptr;
145 Ident__abnormal_termination = Ident___exception_info =
nullptr;
146 Ident___exception_code = Ident___abnormal_termination =
nullptr;
147 Ident_GetExceptionInfo = Ident_GetExceptionCode =
nullptr;
148 Ident_AbnormalTermination =
nullptr;
153 IncrementalProcessing = LangOpts.IncrementalExtensions;
157 SkippingUntilPragmaHdrStop =
true;
161 !this->PPOpts.ImplicitPCHInclude.empty())
162 SkippingUntilPCHThroughHeader =
true;
165 PreambleConditionalStack.startRecording();
167 MaxTokens = LangOpts.MaxTokens;
173 IncludeMacroStack.clear();
178 std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers,
nullptr);
179 CurTokenLexer.reset();
182 for (
MacroArgs *ArgList = MacroArgCache; ArgList;)
183 ArgList = ArgList->deallocate();
186 if (OwnsHeaderSearch)
192 assert((!this->Target || this->Target == &
Target) &&
193 "Invalid override of target information");
196 assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
197 "Invalid override of aux target information.");
198 this->AuxTarget = AuxTarget;
201 BuiltinInfo->InitializeTarget(
Target, AuxTarget);
219 NumEnteredSourceFiles = 0;
222 PragmaHandlersBackup = std::move(PragmaHandlers);
223 PragmaHandlers = std::make_unique<PragmaNamespace>(StringRef());
224 RegisterBuiltinPragmas();
227 PredefinesFileID =
FileID();
231 NumEnteredSourceFiles = 1;
233 PragmaHandlers = std::move(PragmaHandlersBackup);
242 if (!DumpFlags)
return;
244 llvm::errs() <<
"\t";
246 llvm::errs() <<
" [StartOfLine]";
248 llvm::errs() <<
" [LeadingSpace]";
250 llvm::errs() <<
" [ExpandDisabled]";
253 llvm::errs() <<
" [UnClean='" << StringRef(Start, Tok.
getLength())
257 llvm::errs() <<
"\tLoc=<";
267 llvm::errs() <<
"MACRO: ";
268 for (
unsigned i = 0, e = MI.
getNumTokens(); i != e; ++i) {
272 llvm::errs() <<
"\n";
276 llvm::errs() <<
"\n*** Preprocessor Stats:\n";
277 llvm::errs() << NumDirectives <<
" directives found:\n";
278 llvm::errs() <<
" " << NumDefined <<
" #define.\n";
279 llvm::errs() <<
" " << NumUndefined <<
" #undef.\n";
280 llvm::errs() <<
" #include/#include_next/#import:\n";
281 llvm::errs() <<
" " << NumEnteredSourceFiles <<
" source files entered.\n";
282 llvm::errs() <<
" " << MaxIncludeStackDepth <<
" max include stack depth\n";
283 llvm::errs() <<
" " << NumIf <<
" #if/#ifndef/#ifdef.\n";
284 llvm::errs() <<
" " << NumElse <<
" #else/#elif/#elifdef/#elifndef.\n";
285 llvm::errs() <<
" " << NumEndif <<
" #endif.\n";
286 llvm::errs() <<
" " << NumPragma <<
" #pragma.\n";
287 llvm::errs() << NumSkipped <<
" #if/#ifndef#ifdef regions skipped\n";
289 llvm::errs() << NumMacroExpanded <<
"/" << NumFnMacroExpanded <<
"/"
290 << NumBuiltinMacroExpanded <<
" obj/fn/builtin macros expanded, "
291 << NumFastMacroExpanded <<
" on the fast path.\n";
292 llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
293 <<
" token paste (##) operations performed, "
294 << NumFastTokenPaste <<
" on the fast path.\n";
296 llvm::errs() <<
"\nPreprocessor Memory: " <<
getTotalMemory() <<
"B total";
298 llvm::errs() <<
"\n BumpPtr: " << BP.getTotalMemory();
299 llvm::errs() <<
"\n Macro Expanded Tokens: "
300 << llvm::capacity_in_bytes(MacroExpandedTokens);
301 llvm::errs() <<
"\n Predefines Buffer: " << Predefines.capacity();
303 llvm::errs() <<
"\n Macros: "
304 << llvm::capacity_in_bytes(CurSubmoduleState->Macros);
305 llvm::errs() <<
"\n #pragma push_macro Info: "
306 << llvm::capacity_in_bytes(PragmaPushMacroInfo);
307 llvm::errs() <<
"\n Poison Reasons: "
308 << llvm::capacity_in_bytes(PoisonReasons);
309 llvm::errs() <<
"\n Comment Handlers: "
310 << llvm::capacity_in_bytes(CommentHandlers) <<
"\n";
316 !ReadMacrosFromExternalSource) {
317 ReadMacrosFromExternalSource =
true;
323 CurSubmoduleState->Macros.try_emplace(
Macro.II);
325 return CurSubmoduleState->Macros.begin();
329 return BP.getTotalMemory()
330 + llvm::capacity_in_bytes(MacroExpandedTokens)
331 + Predefines.capacity()
334 + llvm::capacity_in_bytes(CurSubmoduleState->Macros)
335 + llvm::capacity_in_bytes(PragmaPushMacroInfo)
336 + llvm::capacity_in_bytes(PoisonReasons)
337 + llvm::capacity_in_bytes(CommentHandlers);
343 !ReadMacrosFromExternalSource) {
344 ReadMacrosFromExternalSource =
true;
348 return CurSubmoduleState->Macros.end();
355 std::equal(Tokens.begin(), Tokens.end(), MI->
tokens_begin());
362 StringRef BestSpelling;
366 Def = I->second.findDirectiveAtLoc(
Loc, SourceMgr);
378 BestLocation = Location;
379 BestSpelling = I->first->getName();
387 CurLexerCallback = CurLexer->isDependencyDirectivesLexer()
388 ? CLK_DependencyDirectivesLexer
390 else if (CurTokenLexer)
391 CurLexerCallback = CLK_TokenLexer;
393 CurLexerCallback = CLK_CachingLexer;
397 unsigned CompleteLine,
398 unsigned CompleteColumn) {
399 assert(CompleteLine && CompleteColumn &&
"Starts from 1:1");
400 assert(!CodeCompletionFile &&
"Already set");
403 std::optional<llvm::MemoryBufferRef> Buffer =
409 const char *Position = Buffer->getBufferStart();
411 for (; *Position; ++Position) {
412 if (*Position !=
'\r' && *Position !=
'\n')
416 if ((Position[1] ==
'\r' || Position[1] ==
'\n') &&
417 Position[0] != Position[1])
424 Position += CompleteColumn - 1;
428 if (SkipMainFilePreamble.first &&
430 if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
431 Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
434 if (Position > Buffer->getBufferEnd())
435 Position = Buffer->getBufferEnd();
437 CodeCompletionFile =
File;
438 CodeCompletionOffset = Position - Buffer->getBufferStart();
440 auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
441 Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier());
442 char *NewBuf = NewBuffer->getBufferStart();
443 char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
445 std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
471 if (Tok.
isNot(tok::raw_identifier) && !Tok.
hasUCN()) {
474 return II->getName();
481 const char *Ptr = Buffer.data();
483 return StringRef(Ptr, Len);
497 if (ExpansionLocStart.
isValid())
499 ExpansionLocEnd, Str.size());
503 if (Tok.
is(tok::raw_identifier))
514 StringRef Buffer =
SM.getBufferData(LocInfo.first, &
Invalid);
521 ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr);
533 if (!
getLangOpts().isCompilingModuleImplementation())
549 assert(NumEnteredSourceFiles == 0 &&
"Cannot reenter the main file!");
560 if (SkipMainFilePreamble.first > 0)
561 CurLexer->SetByteOffset(SkipMainFilePreamble.first,
562 SkipMainFilePreamble.second);
579 auto Tracer = std::make_unique<NoTrivialPPDirectiveTracer>(*
this);
580 DirTracer = Tracer.get();
582 std::optional<Token> FirstPPTok = CurLexer->peekNextPPToken();
584 FirstPPTokenLoc = FirstPPTok->getLocation();
589 std::unique_ptr<llvm::MemoryBuffer> SB =
590 llvm::MemoryBuffer::getMemBufferCopy(Predefines,
"<built-in>");
591 assert(SB &&
"Cannot create predefined source buffer");
593 assert(FID.
isValid() &&
"Could not create FileID for predefines?");
594 setPredefinesFileID(FID);
604 false,
nullptr,
nullptr,
605 nullptr,
nullptr,
nullptr,
613 setPCHThroughHeaderFileID(
623void Preprocessor::setPCHThroughHeaderFileID(
FileID FID) {
624 assert(PCHThroughHeaderFileID.
isInvalid() &&
625 "PCHThroughHeaderFileID already set!");
626 PCHThroughHeaderFileID = FID;
630 assert(PCHThroughHeaderFileID.
isValid() &&
631 "Invalid PCH through header FileID");
637 PCHThroughHeaderFileID.
isValid();
642 PCHThroughHeaderFileID.
isValid();
659 bool ReachedMainFileEOF =
false;
660 bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
661 bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
666 CurLexerCallback(*
this, Tok);
667 if (Tok.
is(tok::eof) && !InPredefines) {
668 ReachedMainFileEOF =
true;
671 if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
673 if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
676 if (ReachedMainFileEOF) {
677 if (UsingPCHThroughHeader)
685void Preprocessor::replayPreambleConditionalStack() {
687 if (PreambleConditionalStack.isReplaying()) {
689 "CurPPLexer is null when calling replayPreambleConditionalStack.");
691 PreambleConditionalStack.doneReplaying();
692 if (PreambleConditionalStack.reachedEOFWhileSkipping())
693 SkipExcludedConditionalBlock(
694 PreambleConditionalStack.SkipInfo->HashTokenLoc,
695 PreambleConditionalStack.SkipInfo->IfTokenLoc,
696 PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
697 PreambleConditionalStack.SkipInfo->FoundElse,
698 PreambleConditionalStack.SkipInfo->ElseLoc);
705 Callbacks->EndOfMainFile();
716 assert(!
Identifier.getRawIdentifier().empty() &&
"No raw identifier data!");
751 PoisonReasons[II] = DiagID;
755 assert(Ident__exception_code && Ident__exception_info);
756 assert(Ident___exception_code && Ident___exception_info);
770 "Can't handle identifiers without identifier info!");
771 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
772 PoisonReasons.find(
Identifier.getIdentifierInfo());
773 if(it == PoisonReasons.end())
779void Preprocessor::updateOutOfDateIdentifier(
const IdentifierInfo &II)
const {
782 "getExternalSource() should not return nullptr");
796 "Can't handle identifiers without identifier info!");
806 bool CurrentIsPoisoned =
false;
807 const bool IsSpecialVariadicMacro =
808 &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
809 if (IsSpecialVariadicMacro)
812 updateOutOfDateIdentifier(II);
815 if (IsSpecialVariadicMacro)
827 const auto *MI = MD.getMacroInfo();
828 assert(MI &&
"macro definition with no macro info?");
829 if (!DisableMacroExpansion) {
830 if (!
Identifier.isExpandDisabled() && MI->isEnabled()) {
834 return HandleMacroExpandedIdentifier(
Identifier, MD);
880 !InMacroArgs && !DisableMacroExpansion &&
882 CurLexerCallback != CLK_CachingLexer) {
884 NamedModuleImportPath.clear();
886 ModuleImportExpectsIdentifier =
true;
887 CurLexerCallback = CLK_LexAfterModuleImport;
896 while (!CurLexerCallback(*
this,
Result))
902 if (
Result.is(tok::code_completion) &&
Result.getIdentifierInfo()) {
908 Result.setIdentifierInfo(
nullptr);
916 if (
getLangOpts().CPlusPlusModules && LexLevel == 1 &&
918 switch (
Result.getKind()) {
919 case tok::l_paren:
case tok::l_square:
case tok::l_brace:
920 StdCXXImportSeqState.handleOpenBracket();
922 case tok::r_paren:
case tok::r_square:
923 StdCXXImportSeqState.handleCloseBracket();
926 StdCXXImportSeqState.handleCloseBrace();
928#define PRAGMA_ANNOTATION(X) case tok::annot_##X:
930#include "clang/Basic/TokenKinds.def"
931#undef PRAGMA_ANNOTATION
934 case tok::annot_module_include:
936 TrackGMFState.handleSemi();
937 StdCXXImportSeqState.handleSemi();
938 ModuleDeclState.handleSemi();
940 case tok::header_name:
941 case tok::annot_header_unit:
942 StdCXXImportSeqState.handleHeaderName();
947 TrackGMFState.handleExport();
948 StdCXXImportSeqState.handleExport();
949 ModuleDeclState.handleExport();
952 ModuleDeclState.handleColon();
955 ModuleDeclState.handlePeriod();
959 case tok::identifier:
962 if (StdCXXImportSeqState.atTopLevel()) {
963 if (
Result.getIdentifierInfo()->isModulesImport()) {
964 TrackGMFState.handleImport(StdCXXImportSeqState.afterTopLevelSeq());
965 StdCXXImportSeqState.handleImport();
966 if (StdCXXImportSeqState.afterImportSeq()) {
967 ModuleImportLoc =
Result.getLocation();
968 NamedModuleImportPath.clear();
970 ModuleImportExpectsIdentifier =
true;
971 CurLexerCallback = CLK_LexAfterModuleImport;
977 TrackGMFState.handleModule(StdCXXImportSeqState.afterTopLevelSeq());
978 ModuleDeclState.handleModule();
982 ModuleDeclState.handleIdentifier(
Result.getIdentifierInfo());
983 if (ModuleDeclState.isModuleCandidate())
987 TrackGMFState.handleMisc();
988 StdCXXImportSeqState.handleMisc();
989 ModuleDeclState.handleMisc();
995 CheckPoints[CurLexer->getFileID()].push_back(CurLexer->BufferPtr);
996 CheckPointCounter = 0;
999 LastTokenWasAt =
Result.is(tok::at);
1002 if ((LexLevel == 0 || PreprocessToken) &&
1015 if (Tok.
isOneOf(tok::unknown, tok::eof, tok::eod,
1016 tok::annot_repl_input_end))
1018 if (Tokens !=
nullptr)
1019 Tokens->push_back(Tok);
1045 if (FilenameTok.
is(tok::less) && AllowMacroExpansion) {
1052 FilenameBuffer.push_back(
'<');
1058 while (FilenameTok.
isNot(tok::greater)) {
1060 if (FilenameTok.
isOneOf(tok::eod, tok::eof)) {
1062 Diag(Start, diag::note_matching) << tok::less;
1069 if (FilenameTok.
is(tok::code_completion)) {
1078 FilenameBuffer.push_back(
' ');
1082 size_t PreAppendSize = FilenameBuffer.size();
1083 FilenameBuffer.resize(PreAppendSize + FilenameTok.
getLength());
1085 const char *BufPtr = &FilenameBuffer[PreAppendSize];
1086 unsigned ActualLen =
getSpelling(FilenameTok, BufPtr);
1089 if (BufPtr != &FilenameBuffer[PreAppendSize])
1090 memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen);
1093 if (FilenameTok.
getLength() != ActualLen)
1094 FilenameBuffer.resize(PreAppendSize + ActualLen);
1098 FilenameTok.
setKind(tok::header_name);
1103 }
else if (FilenameTok.
is(tok::string_literal) && AllowMacroExpansion) {
1114 StringRef Str =
getSpelling(FilenameTok, FilenameBuffer);
1115 if (Str.size() >= 2 && Str.front() ==
'"' && Str.back() ==
'"')
1116 FilenameTok.
setKind(tok::header_name);
1127 unsigned BracketDepth = 0;
1129 Toks.emplace_back();
1132 switch (Toks.back().getKind()) {
1133 case tok::l_paren:
case tok::l_square:
case tok::l_brace:
1137 case tok::r_paren:
case tok::r_square:
case tok::r_brace:
1138 if (BracketDepth == 0)
1144 if (BracketDepth == 0)
1187 if (NamedModuleImportPath.empty() &&
getLangOpts().CPlusPlusModules) {
1191 if (
Result.is(tok::colon) && ModuleDeclState.isNamedModule()) {
1192 std::string Name = ModuleDeclState.getPrimaryName().str();
1194 NamedModuleImportPath.emplace_back(
Result.getLocation(),
1196 CurLexerCallback = CLK_LexAfterModuleImport;
1206 auto ToksCopy = std::make_unique<Token[]>(Toks.size());
1207 std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
1208 EnterTokenStream(std::move(ToksCopy), Toks.size(),
1212 bool ImportingHeader =
Result.is(tok::header_name);
1215 if (ImportingHeader) {
1219 Suffix.push_back(
Result);
1224 if (Suffix.back().isNot(tok::semi)) {
1226 EnterTokens(Suffix);
1235 Diag(SemiLoc, diag::err_header_import_semi_in_macro);
1240 ImportTok.
setKind(tok::kw_import);
1245 auto Action = HandleHeaderIncludeOrImport(
1247 switch (Action.Kind) {
1248 case ImportAction::None:
1251 case ImportAction::ModuleBegin:
1253 Suffix.emplace_back();
1254 Suffix.back().startToken();
1255 Suffix.back().setKind(tok::annot_module_begin);
1256 Suffix.back().setLocation(SemiLoc);
1257 Suffix.back().setAnnotationEndLoc(SemiLoc);
1258 Suffix.back().setAnnotationValue(Action.ModuleForHeader);
1261 case ImportAction::ModuleImport:
1262 case ImportAction::HeaderUnitImport:
1263 case ImportAction::SkippedModuleImport:
1266 Suffix[0].setKind(tok::annot_header_unit);
1267 Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
1268 Suffix[0].setAnnotationValue(Action.ModuleForHeader);
1271 case ImportAction::Failure:
1273 "This should be an early exit only to a fatal error");
1274 Result.setKind(tok::eof);
1275 CurLexer->cutOffLexing();
1276 EnterTokens(Suffix);
1280 EnterTokens(Suffix);
1290 if (ModuleImportExpectsIdentifier &&
Result.getKind() == tok::identifier) {
1293 NamedModuleImportPath.emplace_back(
Result.getLocation(),
1294 Result.getIdentifierInfo());
1295 ModuleImportExpectsIdentifier =
false;
1296 CurLexerCallback = CLK_LexAfterModuleImport;
1303 if (!ModuleImportExpectsIdentifier &&
Result.getKind() == tok::period) {
1304 ModuleImportExpectsIdentifier =
true;
1305 CurLexerCallback = CLK_LexAfterModuleImport;
1310 if (NamedModuleImportPath.empty() ||
Result.is(tok::eof))
1316 if (
Result.isNot(tok::semi)) {
1317 Suffix.push_back(
Result);
1319 if (Suffix.back().isNot(tok::semi)) {
1321 EnterTokens(Suffix);
1324 SemiLoc = Suffix.back().getLocation();
1331 std::string FlatModuleName;
1333 for (
auto &Piece : NamedModuleImportPath) {
1335 if (!FlatModuleName.empty() && FlatModuleName.back() !=
':')
1336 FlatModuleName +=
".";
1337 FlatModuleName += Piece.getIdentifierInfo()->getName();
1340 NamedModuleImportPath.clear();
1341 NamedModuleImportPath.emplace_back(FirstPathLoc,
1345 Module *Imported =
nullptr;
1348 Imported = TheModuleLoader.
loadModule(ModuleImportLoc,
1349 NamedModuleImportPath,
1357 Callbacks->moduleImport(ModuleImportLoc, NamedModuleImportPath, Imported);
1359 if (!Suffix.empty()) {
1360 EnterTokens(Suffix);
1367 bool IncludeExports) {
1368 CurSubmoduleState->VisibleModules.setVisible(
1369 M,
Loc, IncludeExports, [](
Module *) {},
1373 Diag(ModuleImportLoc, diag::warn_module_conflict)
1374 <<
Path[0]->getFullModuleName()
1375 << Conflict->getFullModuleName()
1380 if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1381 BuildingSubmoduleStack.back().M->Imports.insert(M);
1385 const char *DiagnosticTag,
1386 bool AllowMacroExpansion) {
1388 if (
Result.isNot(tok::string_literal)) {
1389 Diag(
Result, diag::err_expected_string_literal)
1390 << 0 << DiagnosticTag;
1397 StrToks.push_back(
Result);
1399 if (
Result.hasUDSuffix())
1402 if (AllowMacroExpansion)
1406 }
while (
Result.is(tok::string_literal));
1410 assert(Literal.isOrdinary() &&
"Didn't allow wide strings in");
1412 if (Literal.hadError)
1415 if (Literal.Pascal) {
1416 Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
1417 << 0 << DiagnosticTag;
1421 String = std::string(Literal.GetString());
1426 assert(Tok.
is(tok::numeric_constant));
1428 bool NumberInvalid =
false;
1429 StringRef Spelling =
getSpelling(Tok, IntegerBuffer, &NumberInvalid);
1435 if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1437 llvm::APInt APVal(64, 0);
1438 if (Literal.GetIntegerValue(APVal))
1441 Value = APVal.getLimitedValue();
1446 assert(Handler &&
"NULL comment handler");
1447 assert(!llvm::is_contained(CommentHandlers, Handler) &&
1448 "Comment handler already registered");
1449 CommentHandlers.push_back(Handler);
1453 std::vector<CommentHandler *>::iterator Pos =
1454 llvm::find(CommentHandlers, Handler);
1455 assert(Pos != CommentHandlers.end() &&
"Comment handler not registered");
1456 CommentHandlers.erase(Pos);
1460 bool AnyPendingTokens =
false;
1461 for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1462 HEnd = CommentHandlers.end();
1464 if ((*H)->HandleComment(*
this, Comment))
1465 AnyPendingTokens =
true;
1473void Preprocessor::emitMacroDeprecationWarning(
const Token &
Identifier)
const {
1474 const MacroAnnotations &A =
1476 assert(A.DeprecationInfo &&
1477 "Macro deprecation warning without recorded annotation!");
1478 const MacroAnnotationInfo &Info = *A.DeprecationInfo;
1479 if (Info.Message.empty())
1484 <<
Identifier.getIdentifierInfo() << 1 << Info.Message;
1485 Diag(Info.Location, diag::note_pp_macro_annotation) << 0;
1488void Preprocessor::emitRestrictExpansionWarning(
const Token &
Identifier)
const {
1489 const MacroAnnotations &A =
1491 assert(A.RestrictExpansionInfo &&
1492 "Macro restricted expansion warning without recorded annotation!");
1493 const MacroAnnotationInfo &Info = *A.RestrictExpansionInfo;
1494 if (Info.Message.empty())
1499 <<
Identifier.getIdentifierInfo() << 1 << Info.Message;
1500 Diag(Info.Location, diag::note_pp_macro_annotation) << 1;
1504 unsigned DiagSelection)
const {
1505 Diag(
Identifier, diag::warn_fp_nan_inf_when_disabled) << DiagSelection << 1;
1509 bool IsUndef)
const {
1510 const MacroAnnotations &A =
1512 assert(A.FinalAnnotationLoc &&
1513 "Final macro warning without recorded annotation!");
1516 <<
Identifier.getIdentifierInfo() << (IsUndef ? 0 : 1);
1517 Diag(*A.FinalAnnotationLoc, diag::note_pp_macro_annotation) << 2;
1527 auto FirstRegionEndingAfterLoc = llvm::partition_point(
1529 &
Loc](
const std::pair<SourceLocation, SourceLocation> &Region) {
1533 if (FirstRegionEndingAfterLoc != Map.end()) {
1536 FirstRegionEndingAfterLoc->first,
Loc);
1540 if (!Map.empty() && Map.back().first == Map.back().second)
1564 return TestInMap(SafeBufferOptOutMap,
Loc);
1567 LoadedSafeBufferOptOutMap.lookupLoadedOptOutMap(
Loc, SourceMgr);
1579 InSafeBufferOptOutRegion =
true;
1580 CurrentSafeBufferOptOutStart =
Loc;
1584 if (!SafeBufferOptOutMap.empty()) {
1585 [[maybe_unused]]
auto *PrevRegion = &SafeBufferOptOutMap.back();
1586 assert(PrevRegion->first != PrevRegion->second &&
1587 "Shall not begin a safe buffer opt-out region before closing the "
1593 SafeBufferOptOutMap.emplace_back(
Loc,
Loc);
1597 InSafeBufferOptOutRegion =
false;
1601 assert(!SafeBufferOptOutMap.empty() &&
1602 "Misordered safe buffer opt-out regions");
1603 auto *CurrRegion = &SafeBufferOptOutMap.back();
1604 assert(CurrRegion->first == CurrRegion->second &&
1605 "Set end location to a closed safe buffer opt-out region");
1606 CurrRegion->second =
Loc;
1612 return InSafeBufferOptOutRegion;
1615 StartLoc = CurrentSafeBufferOptOutStart;
1616 return InSafeBufferOptOutRegion;
1621 assert(!InSafeBufferOptOutRegion &&
1622 "Attempt to serialize safe buffer opt-out regions before file being "
1623 "completely preprocessed");
1627 for (
const auto &[begin, end] : SafeBufferOptOutMap) {
1628 SrcSeq.push_back(begin);
1629 SrcSeq.push_back(end);
1641 if (SourceLocations.size() == 0)
1644 assert(SourceLocations.size() % 2 == 0 &&
1645 "ill-formed SourceLocation sequence");
1647 auto It = SourceLocations.begin();
1649 LoadedSafeBufferOptOutMap.findAndConsLoadedOptOutMap(*It, SourceMgr);
1655 Regions.emplace_back(
Begin, End);
1656 }
while (It != SourceLocations.end());
1677 if (
auto It = CheckPoints.find(FID); It != CheckPoints.end()) {
1679 const char *
Last =
nullptr;
1681 for (
const char *
P : FileCheckPoints) {
1697 return SeenNoTrivialPPDirective;
1700void NoTrivialPPDirectiveTracer::setSeenNoTrivialPPDirective() {
1701 if (InMainFile && !SeenNoTrivialPPDirective)
1702 SeenNoTrivialPPDirective =
true;
1717 setSeenNoTrivialPPDirective();
Defines enum values for all the target-independent builtin functions.
Defines the clang::FileManager interface and associated types.
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified.
Defines the clang::LangOptions interface.
llvm::MachO::FileType FileType
llvm::MachO::Target Target
llvm::MachO::Record Record
Defines the clang::MacroInfo and clang::MacroDirective classes.
Defines the clang::Module class, which describes a module in the source code.
Defines the PreprocessorLexer interface.
static bool MacroDefinitionEquals(const MacroInfo *MI, ArrayRef< TokenValue > Tokens)
Compares macro tokens with a specified token value sequence.
static constexpr unsigned CheckPointStepSize
Minimum distance between two check points, in tokens.
Defines the clang::Preprocessor interface.
Defines the clang::SourceLocation class and associated facilities.
Defines the SourceManager interface.
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
virtual void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Callback invoked when performing code completion inside the filename part of an #include directive.
virtual ~CodeCompletionHandler()
virtual void CodeCompleteNaturalLanguage()
Callback invoked when performing code completion in a part of the file where we expect natural langua...
Concrete class used by the front-end to report problems and issues.
virtual ~EmptylineHandler()
virtual ~ExternalPreprocessorSource()
virtual void updateOutOfDateIdentifier(const IdentifierInfo &II)=0
Update an out-of-date identifier.
A reference to a FileEntry that includes the name of the file as it was accessed by the FileManager's...
Cached information about one file (either on disk or in the virtual file system).
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Provides lookups to, and iteration over, IdentiferInfo objects.
One of these records is kept for each identifier that is lexed.
bool isModulesImport() const
Determine whether this is the contextual keyword import.
bool IsKeywordInCPlusPlus() const
Return true if this identifier would be a keyword in C++ mode.
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
void setIsPoisoned(bool Value=true)
setIsPoisoned - Mark this identifier as poisoned.
bool isPoisoned() const
Return true if this token has been poisoned.
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
void setIsFutureCompatKeyword(bool Val)
StringRef getName() const
Return the actual identifier string.
bool isFutureCompatKeyword() const
is/setIsFutureCompatKeyword - Initialize information about whether or not this language token is a ke...
bool isExtensionToken() const
get/setExtension - Initialize information about whether or not this language token is an extension.
void AddKeywords(const LangOptions &LangOpts)
Populate the identifier table with info about the language keywords for the language specified by Lan...
@ FEM_UnsetOnCommandLine
Used only for FE option processing; this is only used to indicate that the user did not specify an ex...
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
A description of the current definition of a macro.
MacroInfo * getMacroInfo() const
Get the MacroInfo that should be used for this definition.
MacroInfo * getMacroInfo()
SourceLocation getLocation() const
Encapsulates the data about a macro definition (e.g.
const_tokens_iterator tokens_begin() const
unsigned getNumTokens() const
Return the number of tokens that this macro expands to.
const Token & getReplacementToken(unsigned Tok) const
bool isBuiltinMacro() const
Return true if this macro requires processing before expansion.
bool isObjectLike() const
Abstract interface for a module loader.
virtual ModuleLoadResult loadModule(SourceLocation ImportLoc, ModuleIdPath Path, Module::NameVisibilityKind Visibility, bool IsInclusionDirective)=0
Attempt to load the given module.
Represents a macro directive exported by a module.
Describes a module or submodule.
@ Hidden
All of the names in this module are hidden.
void MacroExpands(const Token &MacroNameTok, const MacroDefinition &MD, SourceRange Range, const MacroArgs *Args) override
Called by Preprocessor::HandleMacroExpandedIdentifier when a macro invocation is found.
bool hasSeenNoTrivialPPDirective() const
void LexedFileChanged(FileID FID, LexedFileChangeReason Reason, SrcMgr::CharacteristicKind FileType, FileID PrevFID, SourceLocation Loc) override
Callback invoked whenever the Lexer moves to a different file for lexing.
NumericLiteralParser - This performs strict semantic analysis of the content of a ppnumber,...
PragmaNamespace - This PragmaHandler subdivides the namespace of pragmas, allowing hierarchical pragm...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
void setConditionalLevels(ArrayRef< PPConditionalInfo > CL)
void LexIncludeFilename(Token &FilenameTok)
Lex a token, producing a header-name token if possible.
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
bool PCHWithHdrStopCreate
When true, we are creating a PCH or creating the PCH object while expecting a #pragma hdrstop to sepa...
std::string PCHThroughHeader
If non-empty, the filename used in an #include directive in the primary source file (or command-line ...
bool PCHWithHdrStop
When true, we are creating or using a PCH where a #pragma hdrstop is expected to indicate the beginni...
bool GeneratePreamble
True indicates that a preamble is being generated.
bool markIncluded(FileEntryRef File)
Mark the file as included.
void FinalizeForModelFile()
Cleanup after model file parsing.
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
bool isSafeBufferOptOut(const SourceManager &SourceMgr, const SourceLocation &Loc) const
const char * getCheckPoint(FileID FID, const char *Start) const
Returns a pointer into the given file's buffer that's guaranteed to be between tokens.
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
void DumpMacro(const MacroInfo &MI) const
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool SetCodeCompletionPoint(FileEntryRef File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
bool isInImportingCXXNamedModules() const
If we're importing a standard C++20 Named Modules.
void Lex(Token &Result)
Lex the next token for this preprocessor.
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
bool enterOrExitSafeBufferOptOutRegion(bool isEnter, const SourceLocation &Loc)
Alter the state of whether this PP currently is in a "-Wunsafe-buffer-usage" opt-out region.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
const MacroAnnotations & getMacroAnnotations(const IdentifierInfo *II) const
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
macro_iterator macro_end(bool IncludeExternalMacros=true) const
SourceManager & getSourceManager() const
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
bool getCommentRetentionState() const
Module * getCurrentModuleImplementation()
Retrieves the module whose implementation we're current compiling, if any.
bool isNextPPTokenOneOf(Ts... Ks)
Check whether the next pp-token is one of the specificed token kind.
MacroMap::const_iterator macro_iterator
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions,...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
bool isPPInSafeBufferOptOutRegion()
void makeModuleVisible(Module *M, SourceLocation Loc, bool IncludeExports=true)
void setCurrentFPEvalMethod(SourceLocation PragmaLoc, LangOptions::FPEvalMethodKind Val)
const TargetInfo & getTargetInfo() const
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void DumpLocation(SourceLocation Loc) const
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool HandleComment(Token &result, SourceRange Comment)
HeaderSearch & getHeaderSearchInfo() const
bool setDeserializedSafeBufferOptOutMap(const SmallVectorImpl< SourceLocation > &SrcLocSeqs)
ExternalPreprocessorSource * getExternalSource() const
SmallVector< SourceLocation, 64 > serializeSafeBufferOptOutMap() const
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
OptionalFileEntryRef LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, ConstSearchDirIterator FromDir, const FileEntry *FromFile, ConstSearchDirIterator *CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false, bool OpenFile=true, bool CacheFailures=true)
Given a "foo" or <foo> reference, look up the indicated file.
IdentifierTable & getIdentifierTable()
const LangOptions & getLangOpts() const
void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val)
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
llvm::DenseMap< FileID, SafeBufferOptOutRegionsTy > LoadedRegions
void PoisonSEHIdentifiers(bool Poison=true)
size_t getTotalMemory() const
void LexTokensUntilEOF(std::vector< Token > *Tokens=nullptr)
Lex all tokens for this preprocessor until (and excluding) end of file.
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
DiagnosticsEngine & getDiagnostics() const
bool hasSeenNoTrivialPPDirective() const
Whether we've seen pp-directives which may have changed the preprocessing state.
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop.
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
Preprocessor(const PreprocessorOptions &PPOpts, DiagnosticsEngine &diags, const LangOptions &LangOpts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens.
Encodes a location in the source.
bool isValid() const
Return true if this is a valid SourceLocation object.
void print(raw_ostream &OS, const SourceManager &SM) const
SourceLocation getLocWithOffset(IntTy Offset) const
Return a source location with the specified offset from this SourceLocation.
This class handles loading and caching of source files into memory.
bool isLocalSourceLocation(SourceLocation Loc) const
Returns true if Loc did not come from a PCH/Module.
OptionalFileEntryRef getFileEntryRefForID(FileID FID) const
Returns the FileEntryRef for the provided FileID.
FileID createFileID(FileEntryRef SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, SourceLocation::UIntTy LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
FileID getMainFileID() const
Returns the FileID of the main source file.
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer.
void overrideFileContents(FileEntryRef SourceFile, const llvm::MemoryBufferRef &Buffer)
Override the contents of the given source file by providing an already-allocated buffer.
bool isLoadedFileID(FileID FID) const
Returns true if FID came from a PCH/Module.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
SourceLocation createExpansionLoc(SourceLocation SpellingLoc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned Length, bool ExpansionIsTokenRange=true, int LoadedID=0, SourceLocation::UIntTy LoadedOffset=0)
Creates an expansion SLocEntry for a macro use.
bool isBeforeInTranslationUnit(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the translation unit.
std::optional< llvm::MemoryBufferRef > getMemoryBufferForFileOrNone(FileEntryRef File)
Retrieve the memory buffer associated with the given file.
A trivial tuple used to represent a source range.
StringLiteralParser - This decodes string escape characters and performs wide string analysis and Tra...
Exposes information about the current target.
Token - This structure provides full information about a lexed token.
IdentifierInfo * getIdentifierInfo() const
void setLiteralData(const char *Ptr)
bool hasUCN() const
Returns true if this token contains a universal character name.
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
unsigned getLength() const
void setLength(unsigned Len)
bool isExpandDisabled() const
Return true if this identifier token should never be expanded in the future, due to C99 6....
void setKind(tok::TokenKind K)
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
tok::TokenKind getKind() const
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
bool isOneOf(Ts... Ks) const
@ HasSeenNoTrivialPPDirective
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
void setLocation(SourceLocation L)
bool hasLeadingEmptyMacro() const
Return true if this token has an empty macro before it.
void setRawIdentifierData(const char *Ptr)
bool isNot(tok::TokenKind K) const
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
void startToken()
Reset all flags to cleared.
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
void setIdentifierInfo(IdentifierInfo *II)
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Defines the clang::TargetInfo interface.
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
const char * getTokenName(TokenKind Kind) LLVM_READNONE
Determines the name of a token as used within the front end.
The JSON file list parser is used to communicate input to InstallAPI.
std::pair< FileID, unsigned > FileIDAndOffset
void expandUCNs(SmallVectorImpl< char > &Buf, StringRef Input)
Copy characters from Input to Buf, expanding any UCNs.
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
@ Result
The result type of a method or function.
TranslationUnitKind
Describes the kind of translation unit being processed.
@ TU_Prefix
The translation unit is a prefix to a translation unit, and is not complete.