30#include "llvm/ADT/SmallSet.h"
31#include "llvm/BinaryFormat/Dwarf.h"
32#include "llvm/Frontend/OpenMP/OMPConstants.h"
33#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
34#include "llvm/IR/Constants.h"
35#include "llvm/IR/DebugInfoMetadata.h"
36#include "llvm/IR/Instructions.h"
37#include "llvm/IR/IntrinsicInst.h"
38#include "llvm/IR/Metadata.h"
39#include "llvm/Support/AtomicOrdering.h"
40#include "llvm/Support/Debug.h"
43using namespace CodeGen;
44using namespace llvm::omp;
46#define TTL_CODEGEN_TYPE "target-teams-loop-codegen"
57 for (
const auto *
C : S.clauses()) {
59 if (
const auto *PreInit =
60 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
61 for (
const auto *I : PreInit->decls()) {
62 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
80 cast<BlockDecl>(CGF.
CurCodeDecl)->capturesVariable(VD));
86 const std::optional<OpenMPDirectiveKind> CapturedRegion = std::nullopt,
87 const bool EmitPreInitStmt =
true)
91 emitPreInitStmt(CGF, S);
94 assert(S.hasAssociatedStmt() &&
95 "Expected associated statement for inlined directive.");
96 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
98 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
99 auto *VD =
C.getCapturedVar();
101 "Canonical decl must be captured.");
116class OMPParallelScope final :
public OMPLexicalScope {
126 : OMPLexicalScope(CGF, S,
std::nullopt,
127 EmitPreInitStmt(S)) {}
132class OMPTeamsScope final :
public OMPLexicalScope {
141 : OMPLexicalScope(CGF, S,
std::nullopt,
142 EmitPreInitStmt(S)) {}
149 const Stmt *PreInits;
151 if (
auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
152 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
153 for (
const auto *
E : LD->counters()) {
154 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
161 for (
const Expr *IRef :
C->varlist()) {
163 cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
164 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
165 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
175 (void)PreCondVars.
apply(CGF);
178 LD->getInnermostCapturedStmt()->getCapturedStmt(),
179 true, LD->getLoopsNumber(),
180 [&
CGF](
unsigned Cnt,
const Stmt *CurStmt) {
181 if (
const auto *CXXFor = dyn_cast<CXXForRangeStmt>(CurStmt)) {
182 if (
const Stmt *
Init = CXXFor->getInit())
184 CGF.
EmitStmt(CXXFor->getRangeStmt());
189 PreInits = LD->getPreInits();
190 }
else if (
const auto *
Tile = dyn_cast<OMPTileDirective>(&S)) {
191 PreInits =
Tile->getPreInits();
192 }
else if (
const auto *Stripe = dyn_cast<OMPStripeDirective>(&S)) {
193 PreInits = Stripe->getPreInits();
194 }
else if (
const auto *Unroll = dyn_cast<OMPUnrollDirective>(&S)) {
195 PreInits = Unroll->getPreInits();
196 }
else if (
const auto *Reverse = dyn_cast<OMPReverseDirective>(&S)) {
197 PreInits = Reverse->getPreInits();
198 }
else if (
const auto *Interchange =
199 dyn_cast<OMPInterchangeDirective>(&S)) {
200 PreInits = Interchange->getPreInits();
202 llvm_unreachable(
"Unknown loop-based directive kind.");
209 if (
auto *PreInitCompound = dyn_cast<CompoundStmt>(PreInits))
210 llvm::append_range(PreInitStmts, PreInitCompound->body());
212 PreInitStmts.push_back(PreInits);
214 for (
const Stmt *S : PreInitStmts) {
217 if (
auto *PreInitDecl = dyn_cast<DeclStmt>(S)) {
218 for (
Decl *I : PreInitDecl->decls())
231 emitPreInitStmt(CGF, S);
242 cast<BlockDecl>(CGF.
CurCodeDecl)->capturesVariable(VD));
248 InlinedShareds(CGF) {
249 for (
const auto *
C : S.clauses()) {
251 if (
const auto *PreInit =
252 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
253 for (
const auto *I : PreInit->decls()) {
254 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
263 }
else if (
const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(
C)) {
264 for (
const Expr *
E : UDP->varlist()) {
265 const Decl *
D = cast<DeclRefExpr>(
E)->getDecl();
266 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(
D))
269 }
else if (
const auto *UDP = dyn_cast<OMPUseDeviceAddrClause>(
C)) {
270 for (
const Expr *
E : UDP->varlist()) {
272 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(
D))
279 if (
const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
280 if (
const Expr *
E = TG->getReductionRef())
281 CGF.
EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl()));
285 llvm::DenseSet<CanonicalDeclPtr<const Decl>> CopyArrayTemps;
287 if (
C->getModifier() != OMPC_REDUCTION_inscan)
289 for (
const Expr *
E :
C->copy_array_temps())
290 CopyArrayTemps.insert(cast<DeclRefExpr>(
E)->getDecl());
292 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
295 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
296 auto *VD =
C.getCapturedVar();
297 if (CopyArrayTemps.contains(VD))
300 "Canonical decl must be captured.");
302 isCapturedVar(CGF, VD) ||
323 if (Kind != OMPD_loop)
328 BindKind =
C->getBindKind();
331 case OMPC_BIND_parallel:
333 case OMPC_BIND_teams:
334 return OMPD_distribute;
335 case OMPC_BIND_thread:
347 if (
const auto *OrigDRE = dyn_cast<DeclRefExpr>(
E)) {
348 if (
const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
349 OrigVD = OrigVD->getCanonicalDecl();
364 llvm::Value *Size =
nullptr;
365 auto SizeInChars =
C.getTypeSizeInChars(Ty);
366 if (SizeInChars.isZero()) {
374 SizeInChars =
C.getTypeSizeInChars(Ty);
375 if (SizeInChars.isZero())
376 return llvm::ConstantInt::get(
SizeTy, 0);
384 const RecordDecl *RD = S.getCapturedRecordDecl();
386 auto CurCap = S.captures().begin();
388 E = S.capture_init_end();
389 I !=
E; ++I, ++CurField, ++CurCap) {
390 if (CurField->hasCapturedVLAType()) {
393 CapturedVars.push_back(Val);
394 }
else if (CurCap->capturesThis()) {
395 CapturedVars.push_back(CXXThisValue);
396 }
else if (CurCap->capturesVariableByCopy()) {
401 if (!CurField->getType()->isAnyPointerType()) {
405 Twine(CurCap->getCapturedVar()->getName(),
".casted"));
421 CapturedVars.push_back(CV);
423 assert(CurCap->capturesVariable() &&
"Expected capture by reference.");
445 return C.getLValueReferenceType(
451 if (
const auto *VLA = dyn_cast<VariableArrayType>(A))
453 if (!A->isVariablyModifiedType())
454 return C.getCanonicalType(
T);
456 return C.getCanonicalParamType(
T);
461struct FunctionOptions {
466 const bool UIntPtrCastRequired =
true;
469 const bool RegisterCastedArgsOnly =
false;
471 const StringRef FunctionName;
474 explicit FunctionOptions(
const CapturedStmt *S,
bool UIntPtrCastRequired,
475 bool RegisterCastedArgsOnly, StringRef FunctionName,
477 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
478 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
479 FunctionName(FunctionName),
Loc(
Loc) {}
485 llvm::MapVector<
const Decl *, std::pair<const VarDecl *, Address>>
487 llvm::DenseMap<
const Decl *, std::pair<const Expr *, llvm::Value *>>
489 llvm::Value *&CXXThisValue,
const FunctionOptions &FO) {
491 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
492 assert(CD->
hasBody() &&
"missing CapturedDecl body");
494 CXXThisValue =
nullptr;
504 auto I = FO.S->captures().begin();
506 if (!FO.UIntPtrCastRequired) {
526 if (FO.UIntPtrCastRequired &&
528 I->capturesVariableArrayType()))
531 if (I->capturesVariable() || I->capturesVariableByCopy()) {
532 CapVar = I->getCapturedVar();
534 }
else if (I->capturesThis()) {
537 assert(I->capturesVariableArrayType());
547 }
else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
549 Ctx, DebugFunctionDecl,
550 CapVar ? CapVar->
getBeginLoc() : FD->getBeginLoc(),
551 CapVar ? CapVar->
getLocation() : FD->getLocation(), II, ArgType,
557 Args.emplace_back(Arg);
559 TargetArgs.emplace_back(
560 FO.UIntPtrCastRequired
577 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
581 F->setDoesNotThrow();
582 F->setDoesNotRecurse();
586 F->removeFnAttr(llvm::Attribute::NoInline);
587 F->addFnAttr(llvm::Attribute::AlwaysInline);
592 FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),
593 FO.UIntPtrCastRequired ? FO.Loc
596 I = FO.S->captures().begin();
600 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
608 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
609 const VarDecl *CurVD = I->getCapturedVar();
610 if (!FO.RegisterCastedArgsOnly)
611 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
619 if (FD->hasCapturedVLAType()) {
620 if (FO.UIntPtrCastRequired) {
623 Args[Cnt]->getName(), ArgLVal),
628 VLASizes.try_emplace(Args[Cnt], VAT->
getSizeExpr(), ExprArg);
629 }
else if (I->capturesVariable()) {
630 const VarDecl *Var = I->getCapturedVar();
640 if (!FO.RegisterCastedArgsOnly) {
644 }
else if (I->capturesVariableByCopy()) {
645 assert(!FD->getType()->isAnyPointerType() &&
646 "Not expecting a captured pointer.");
647 const VarDecl *Var = I->getCapturedVar();
648 LocalAddrs.insert({Args[Cnt],
649 {Var, FO.UIntPtrCastRequired
651 CGF, I->getLocation(), FD->getType(),
652 Args[Cnt]->getName(), ArgLVal)
656 assert(I->capturesThis());
658 LocalAddrs.insert({Args[Cnt], {
nullptr, ArgLVal.
getAddress()}});
672 "CapturedStmtInfo should be set when generating the captured function");
675 bool NeedWrapperFunction =
678 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs,
680 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes,
683 llvm::raw_svector_ostream Out(Buffer);
687 llvm::Function *WrapperF =
nullptr;
688 if (NeedWrapperFunction) {
691 FunctionOptions WrapperFO(&S,
true,
697 WrapperCGF.CXXThisValue, WrapperFO);
700 FunctionOptions FO(&S, !NeedWrapperFunction,
false,
703 *
this, WrapperArgs, WrapperLocalAddrs, WrapperVLASizes, CXXThisValue, FO);
705 for (
const auto &LocalAddrPair : WrapperLocalAddrs) {
706 if (LocalAddrPair.second.first) {
707 LocalScope.
addPrivate(LocalAddrPair.second.first,
708 LocalAddrPair.second.second);
712 for (
const auto &VLASizePair : WrapperVLASizes)
713 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
718 if (!NeedWrapperFunction)
722 WrapperF->removeFromParent();
723 F->getParent()->getFunctionList().insertAfter(F->getIterator(), WrapperF);
726 auto *PI = F->arg_begin();
727 for (
const auto *Arg : Args) {
729 auto I = LocalAddrs.find(Arg);
730 if (I != LocalAddrs.end()) {
733 I->second.first ? I->second.first->getType() : Arg->getType(),
739 auto EI = VLASizes.find(Arg);
740 if (EI != VLASizes.end()) {
768 llvm::Value *NumElements =
emitArrayLength(ArrayTy, ElementTy, DestAddr);
775 DestBegin, NumElements);
780 llvm::Value *IsEmpty =
781 Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arraycpy.isempty");
782 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
785 llvm::BasicBlock *EntryBB =
Builder.GetInsertBlock();
790 llvm::PHINode *SrcElementPHI =
791 Builder.CreatePHI(SrcBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
792 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
797 llvm::PHINode *DestElementPHI =
Builder.CreatePHI(
798 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
799 DestElementPHI->addIncoming(DestBegin, EntryBB);
805 CopyGen(DestElementCurrent, SrcElementCurrent);
808 llvm::Value *DestElementNext =
810 1,
"omp.arraycpy.dest.element");
811 llvm::Value *SrcElementNext =
813 1,
"omp.arraycpy.src.element");
816 Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
817 Builder.CreateCondBr(Done, DoneBB, BodyBB);
818 DestElementPHI->addIncoming(DestElementNext,
Builder.GetInsertBlock());
819 SrcElementPHI->addIncoming(SrcElementNext,
Builder.GetInsertBlock());
829 const auto *BO = dyn_cast<BinaryOperator>(
Copy);
830 if (BO && BO->getOpcode() == BO_Assign) {
839 DestAddr, SrcAddr, OriginalType,
867 bool DeviceConstTarget =
getLangOpts().OpenMPIsTargetDevice &&
869 bool FirstprivateIsLastprivate =
false;
870 llvm::DenseMap<const VarDecl *, OpenMPLastprivateModifier> Lastprivates;
872 for (
const auto *
D :
C->varlist())
873 Lastprivates.try_emplace(
877 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
882 bool MustEmitFirstprivateCopy =
883 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
885 const auto *IRef =
C->varlist_begin();
886 const auto *InitsRef =
C->inits().begin();
887 for (
const Expr *IInit :
C->private_copies()) {
888 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
889 bool ThisFirstprivateIsLastprivate =
890 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
892 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
893 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
895 (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())) {
896 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
903 if (DeviceConstTarget && OrigVD->getType().isConstant(
getContext()) &&
905 (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())) {
906 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
911 FirstprivateIsLastprivate =
912 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
913 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
915 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
933 assert(!CE &&
"Expected non-constant firstprivate.");
956 RunCleanupsScope InitScope(*this);
958 setAddrOfLocalVar(VDInit, SrcElement);
959 EmitAnyExprToMem(Init, DestElement,
960 Init->getType().getQualifiers(),
962 LocalDeclMap.erase(VDInit);
973 setAddrOfLocalVar(VDInit, OriginalAddr);
975 LocalDeclMap.erase(VDInit);
977 if (ThisFirstprivateIsLastprivate &&
978 Lastprivates[OrigVD->getCanonicalDecl()] ==
979 OMPC_LASTPRIVATE_conditional) {
984 (*IRef)->getExprLoc());
989 LocalDeclMap.erase(VD);
990 setAddrOfLocalVar(VD, VDAddr);
992 IsRegistered = PrivateScope.
addPrivate(OrigVD, VDAddr);
994 assert(IsRegistered &&
995 "firstprivate var already registered as private");
1003 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
1011 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
1013 auto IRef =
C->varlist_begin();
1014 for (
const Expr *IInit :
C->private_copies()) {
1015 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1016 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
1017 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1022 assert(IsRegistered &&
"private var already registered as private");
1038 llvm::DenseSet<const VarDecl *> CopiedVars;
1039 llvm::BasicBlock *CopyBegin =
nullptr, *CopyEnd =
nullptr;
1041 auto IRef =
C->varlist_begin();
1042 auto ISrcRef =
C->source_exprs().begin();
1043 auto IDestRef =
C->destination_exprs().begin();
1044 for (
const Expr *AssignOp :
C->assignment_ops()) {
1045 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1053 getContext().getTargetInfo().isTLSSupported()) {
1055 "Copyin threadprivates should have been captured!");
1059 LocalDeclMap.erase(VD);
1069 if (CopiedVars.size() == 1) {
1075 auto *MasterAddrInt =
Builder.CreatePtrToInt(
1077 auto *PrivateAddrInt =
Builder.CreatePtrToInt(
1080 Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,
1085 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1086 const auto *DestVD =
1087 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1107 bool HasAtLeastOneLastprivate =
false;
1109 llvm::DenseSet<const VarDecl *> SIMDLCVs;
1111 const auto *LoopDirective = cast<OMPLoopDirective>(&
D);
1112 for (
const Expr *
C : LoopDirective->counters()) {
1117 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1119 HasAtLeastOneLastprivate =
true;
1122 const auto *IRef =
C->varlist_begin();
1123 const auto *IDestRef =
C->destination_exprs().begin();
1124 for (
const Expr *IInit :
C->private_copies()) {
1127 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1130 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
1131 const auto *DestVD =
1132 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1141 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
1142 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1144 if (
C->getKind() == OMPC_LASTPRIVATE_conditional) {
1147 setAddrOfLocalVar(VD, VDAddr);
1153 bool IsRegistered = PrivateScope.
addPrivate(OrigVD, VDAddr);
1154 assert(IsRegistered &&
1155 "lastprivate var already registered as private");
1163 return HasAtLeastOneLastprivate;
1168 llvm::Value *IsLastIterCond) {
1177 llvm::BasicBlock *ThenBB =
nullptr;
1178 llvm::BasicBlock *DoneBB =
nullptr;
1179 if (IsLastIterCond) {
1185 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1194 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1197 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1198 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1199 if (
const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&
D)) {
1200 auto IC = LoopDirective->counters().begin();
1201 for (
const Expr *F : LoopDirective->finals()) {
1205 AlreadyEmittedVars.insert(
D);
1207 LoopCountersAndUpdates[
D] = F;
1212 auto IRef =
C->varlist_begin();
1213 auto ISrcRef =
C->source_exprs().begin();
1214 auto IDestRef =
C->destination_exprs().begin();
1215 for (
const Expr *AssignOp :
C->assignment_ops()) {
1216 const auto *PrivateVD =
1217 cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1219 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1220 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1224 if (
const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1227 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1228 const auto *DestVD =
1229 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1232 if (
const auto *RefTy = PrivateVD->getType()->getAs<
ReferenceType>())
1238 if (
C->getKind() == OMPC_LASTPRIVATE_conditional)
1241 (*IRef)->getExprLoc());
1244 EmitOMPCopy(
Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1250 if (
const Expr *PostUpdate =
C->getPostUpdateExpr())
1271 if (ForInscan != (
C->getModifier() == OMPC_REDUCTION_inscan))
1273 Shareds.append(
C->varlist_begin(),
C->varlist_end());
1274 Privates.append(
C->privates().begin(),
C->privates().end());
1275 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
1276 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1277 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1278 if (
C->getModifier() == OMPC_REDUCTION_task) {
1279 Data.ReductionVars.append(
C->privates().begin(),
C->privates().end());
1280 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
1281 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
1282 Data.ReductionOps.append(
C->reduction_ops().begin(),
1283 C->reduction_ops().end());
1284 TaskLHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1285 TaskRHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1290 auto *ILHS = LHSs.begin();
1291 auto *IRHS = RHSs.begin();
1292 auto *IPriv = Privates.begin();
1293 for (
const Expr *IRef : Shareds) {
1294 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1302 CGF.EmitAutoVarInit(Emission);
1310 assert(IsRegistered &&
"private var already registered as private");
1314 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1315 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1317 bool isaOMPArraySectionExpr = isa<ArraySectionExpr>(IRef);
1324 isa<ArraySubscriptExpr>(IRef)) {
1341 PrivateScope.
addPrivate(LHSVD, OriginalAddr);
1352 if (!
Data.ReductionVars.empty()) {
1354 Data.IsReductionWithTaskMod =
true;
1358 const Expr *TaskRedRef =
nullptr;
1361 TaskRedRef = cast<OMPParallelDirective>(
D).getTaskReductionRefExpr();
1364 TaskRedRef = cast<OMPForDirective>(
D).getTaskReductionRefExpr();
1367 TaskRedRef = cast<OMPSectionsDirective>(
D).getTaskReductionRefExpr();
1369 case OMPD_parallel_for:
1370 TaskRedRef = cast<OMPParallelForDirective>(
D).getTaskReductionRefExpr();
1372 case OMPD_parallel_master:
1374 cast<OMPParallelMasterDirective>(
D).getTaskReductionRefExpr();
1376 case OMPD_parallel_sections:
1378 cast<OMPParallelSectionsDirective>(
D).getTaskReductionRefExpr();
1380 case OMPD_target_parallel:
1382 cast<OMPTargetParallelDirective>(
D).getTaskReductionRefExpr();
1384 case OMPD_target_parallel_for:
1386 cast<OMPTargetParallelForDirective>(
D).getTaskReductionRefExpr();
1388 case OMPD_distribute_parallel_for:
1390 cast<OMPDistributeParallelForDirective>(
D).getTaskReductionRefExpr();
1392 case OMPD_teams_distribute_parallel_for:
1393 TaskRedRef = cast<OMPTeamsDistributeParallelForDirective>(
D)
1394 .getTaskReductionRefExpr();
1396 case OMPD_target_teams_distribute_parallel_for:
1397 TaskRedRef = cast<OMPTargetTeamsDistributeParallelForDirective>(
D)
1398 .getTaskReductionRefExpr();
1406 case OMPD_parallel_for_simd:
1408 case OMPD_taskyield:
1412 case OMPD_taskgroup:
1420 case OMPD_cancellation_point:
1422 case OMPD_target_data:
1423 case OMPD_target_enter_data:
1424 case OMPD_target_exit_data:
1426 case OMPD_taskloop_simd:
1427 case OMPD_master_taskloop:
1428 case OMPD_master_taskloop_simd:
1429 case OMPD_parallel_master_taskloop:
1430 case OMPD_parallel_master_taskloop_simd:
1431 case OMPD_distribute:
1432 case OMPD_target_update:
1433 case OMPD_distribute_parallel_for_simd:
1434 case OMPD_distribute_simd:
1435 case OMPD_target_parallel_for_simd:
1436 case OMPD_target_simd:
1437 case OMPD_teams_distribute:
1438 case OMPD_teams_distribute_simd:
1439 case OMPD_teams_distribute_parallel_for_simd:
1440 case OMPD_target_teams:
1441 case OMPD_target_teams_distribute:
1442 case OMPD_target_teams_distribute_parallel_for_simd:
1443 case OMPD_target_teams_distribute_simd:
1444 case OMPD_declare_target:
1445 case OMPD_end_declare_target:
1446 case OMPD_threadprivate:
1448 case OMPD_declare_reduction:
1449 case OMPD_declare_mapper:
1450 case OMPD_declare_simd:
1452 case OMPD_declare_variant:
1453 case OMPD_begin_declare_variant:
1454 case OMPD_end_declare_variant:
1457 llvm_unreachable(
"Unexpected directive with task reductions.");
1460 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(TaskRedRef)->getDecl());
1463 false, TaskRedRef->
getType());
1476 bool HasAtLeastOneReduction =
false;
1477 bool IsReductionWithTaskMod =
false;
1480 if (
C->getModifier() == OMPC_REDUCTION_inscan)
1482 HasAtLeastOneReduction =
true;
1483 Privates.append(
C->privates().begin(),
C->privates().end());
1484 LHSExprs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1485 RHSExprs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1486 IsPrivateVarReduction.append(
C->private_var_reduction_flags().begin(),
1487 C->private_var_reduction_flags().end());
1488 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
1489 IsReductionWithTaskMod =
1490 IsReductionWithTaskMod ||
C->getModifier() == OMPC_REDUCTION_task;
1492 if (HasAtLeastOneReduction) {
1494 if (IsReductionWithTaskMod) {
1498 bool TeamsLoopCanBeParallel =
false;
1499 if (
auto *TTLD = dyn_cast<OMPTargetTeamsGenericLoopDirective>(&
D))
1500 TeamsLoopCanBeParallel = TTLD->canBeParallelFor();
1503 TeamsLoopCanBeParallel || ReductionKind == OMPD_simd;
1504 bool SimpleReduction = ReductionKind == OMPD_simd;
1508 *
this,
D.
getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1509 {WithNowait, SimpleReduction, IsPrivateVarReduction, ReductionKind});
1518 llvm::BasicBlock *DoneBB =
nullptr;
1520 if (
const Expr *PostUpdate =
C->getPostUpdateExpr()) {
1522 if (llvm::Value *Cond = CondGen(CGF)) {
1527 CGF.
Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1545 CodeGenBoundParametersTy;
1553 llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> PrivateDecls;
1555 for (
const Expr *Ref :
C->varlist()) {
1556 if (!Ref->getType()->isScalarType())
1558 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1561 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1566 for (
const Expr *Ref :
C->varlist()) {
1567 if (!Ref->getType()->isScalarType())
1569 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1572 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1577 for (
const Expr *Ref :
C->varlist()) {
1578 if (!Ref->getType()->isScalarType())
1580 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1583 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1592 for (
const Expr *Ref :
C->varlist()) {
1593 if (!Ref->getType()->isScalarType())
1595 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1598 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1602 CGF, S, PrivateDecls);
1608 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1609 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1610 llvm::Value *NumThreads =
nullptr;
1616 llvm::Function *OutlinedFn =
1622 NumThreads = CGF.
EmitScalarExpr(NumThreadsClause->getNumThreads(),
1624 Modifier = NumThreadsClause->getModifier();
1626 Message = MessageClause->getMessageString();
1628 Severity = SeverityClause->getSeverityKind();
1630 CGF, NumThreads, NumThreadsClause->getBeginLoc(), Modifier, Severity,
1636 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1638 const Expr *IfCond =
nullptr;
1639 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
1640 if (
C->getNameModifier() == OMPD_unknown ||
1641 C->getNameModifier() == OMPD_parallel) {
1642 IfCond =
C->getCondition();
1647 OMPParallelScope
Scope(CGF, S);
1653 CodeGenBoundParameters(CGF, S, CapturedVars);
1656 CapturedVars, IfCond, NumThreads,
1657 Modifier, Severity, Message);
1662 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
1664 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
1666 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
1667 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
1668 !AA->getAllocator());
1683 CGF, S.getBeginLoc(), OMPD_unknown,
false,
1703 Size = CGF.
Builder.CreateNUWAdd(
1712 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
1713 assert(AA->getAllocator() &&
1714 "Expected allocator expression for non-default allocator.");
1718 if (Allocator->getType()->isIntegerTy())
1720 else if (Allocator->getType()->isPointerTy())
1724 llvm::Value *
Addr = OMPBuilder.createOMPAlloc(
1727 llvm::CallInst *FreeCI =
1728 OMPBuilder.createOMPFree(CGF.
Builder,
Addr, Allocator);
1752 std::string Suffix = getNameWithSeparators({
"cache",
""});
1755 llvm::CallInst *ThreadPrivateCacheCall =
1756 OMPBuilder.createCachedThreadPrivate(CGF.
Builder,
Data, Size, CacheName);
1764 llvm::raw_svector_ostream OS(Buffer);
1765 StringRef Sep = FirstSeparator;
1766 for (StringRef Part : Parts) {
1770 return OS.str().str();
1778 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
Builder,
false,
1779 "." + RegionName +
".after");
1795 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
Builder,
false,
1796 "." + RegionName +
".after");
1811 llvm::Value *IfCond =
nullptr;
1816 llvm::Value *NumThreads =
nullptr;
1821 ProcBindKind ProcBind = OMP_PROC_BIND_default;
1823 ProcBind = ProcBindClause->getProcBindKind();
1825 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1829 auto FiniCB = [
this](InsertPointTy IP) {
1831 return llvm::Error::success();
1838 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1839 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
1847 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1850 auto BodyGenCB = [&,
this](InsertPointTy AllocaIP,
1851 InsertPointTy CodeGenIP) {
1853 *
this, ParallelRegionBodyStmt, AllocaIP, CodeGenIP,
"parallel");
1854 return llvm::Error::success();
1859 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
1861 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
1862 OMPBuilder.createParallel(
Builder, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1863 IfCond, NumThreads, ProcBind, S.hasCancel()));
1877 CGF.
EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1898class OMPTransformDirectiveScopeRAII {
1899 OMPLoopScope *
Scope =
nullptr;
1903 OMPTransformDirectiveScopeRAII(
const OMPTransformDirectiveScopeRAII &) =
1905 OMPTransformDirectiveScopeRAII &
1906 operator=(
const OMPTransformDirectiveScopeRAII &) =
delete;
1910 if (
const auto *Dir = dyn_cast<OMPLoopBasedDirective>(S)) {
1911 Scope =
new OMPLoopScope(CGF, *Dir);
1916 ~OMPTransformDirectiveScopeRAII() {
1927 int MaxLevel,
int Level = 0) {
1928 assert(Level < MaxLevel &&
"Too deep lookup during loop body codegen.");
1929 const Stmt *SimplifiedS = S->IgnoreContainers();
1930 if (
const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {
1933 "LLVM IR generation of compound statement ('{}')");
1937 for (
const Stmt *CurStmt : CS->body())
1938 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
1941 if (SimplifiedS == NextLoop) {
1942 if (
auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
1943 SimplifiedS = Dir->getTransformedStmt();
1944 if (
const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
1945 SimplifiedS = CanonLoop->getLoopStmt();
1946 if (
const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
1949 assert(isa<CXXForRangeStmt>(SimplifiedS) &&
1950 "Expected canonical for loop or range-based for loop.");
1951 const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS);
1952 CGF.
EmitStmt(CXXFor->getLoopVarStmt());
1953 S = CXXFor->getBody();
1955 if (Level + 1 < MaxLevel) {
1958 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
1969 for (
const Expr *UE :
D.updates())
1977 for (
const Expr *UE :
C->updates())
1984 BreakContinueStack.push_back(BreakContinue(
D,
LoopExit, Continue));
1985 for (
const Expr *
E :
D.finals_conditions()) {
1998 bool IsInscanRegion = InscanScope.
Privatize();
1999 if (IsInscanRegion) {
2009 if (EKind != OMPD_simd && !
getLangOpts().OpenMPSimd)
2018 D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();
2023 D.getLoopsNumber());
2031 BreakContinueStack.pop_back();
2042 std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =
2043 std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(*S);
2051static llvm::CallInst *
2056 EffectiveArgs.reserve(Args.size() + 1);
2057 llvm::append_range(EffectiveArgs, Args);
2058 EffectiveArgs.push_back(Cap.second);
2063llvm::CanonicalLoopInfo *
2065 assert(Depth == 1 &&
"Nested loops with OpenMPIRBuilder not yet implemented");
2091 const Stmt *SyntacticalLoop = S->getLoopStmt();
2102 const Stmt *BodyStmt;
2103 if (
const auto *For = dyn_cast<ForStmt>(SyntacticalLoop)) {
2104 if (
const Stmt *InitStmt = For->getInit())
2106 BodyStmt = For->getBody();
2107 }
else if (
const auto *RangeFor =
2108 dyn_cast<CXXForRangeStmt>(SyntacticalLoop)) {
2109 if (
const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
2111 if (
const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
2113 if (
const DeclStmt *EndStmt = RangeFor->getEndStmt())
2115 if (
const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
2117 BodyStmt = RangeFor->getBody();
2119 llvm_unreachable(
"Expected for-stmt or range-based for-stmt");
2122 const CapturedStmt *DistanceFunc = S->getDistanceFunc();
2139 auto BodyGen = [&,
this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,
2140 llvm::Value *IndVar) {
2145 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
2153 return llvm::Error::success();
2156 llvm::CanonicalLoopInfo *
CL =
2157 cantFail(OMPBuilder.createCanonicalLoop(
Builder, BodyGen, DistVal));
2169 const Expr *IncExpr,
2180 const auto &OMPED = cast<OMPExecutableDirective>(S);
2181 const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();
2195 llvm::BasicBlock *ExitBlock =
LoopExit.getBlock();
2196 if (RequiresCleanup)
2203 if (ExitBlock !=
LoopExit.getBlock()) {
2213 BreakContinueStack.push_back(BreakContinue(S,
LoopExit, Continue));
2221 BreakContinueStack.pop_back();
2232 bool HasLinears =
false;
2236 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
Init)->getDecl());
2237 if (
const auto *Ref =
2240 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
2256 if (
const auto *CS = cast_or_null<BinaryOperator>(
C->getCalcStep()))
2257 if (
const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
2271 llvm::BasicBlock *DoneBB =
nullptr;
2274 auto IC =
C->varlist_begin();
2275 for (
const Expr *F :
C->finals()) {
2277 if (llvm::Value *Cond = CondGen(*
this)) {
2282 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2286 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
2297 if (
const Expr *PostUpdate =
C->getPostUpdateExpr())
2309 llvm::APInt ClauseAlignment(64, 0);
2310 if (
const Expr *AlignmentExpr = Clause->getAlignment()) {
2313 ClauseAlignment = AlignmentCI->getValue();
2315 for (
const Expr *
E : Clause->varlist()) {
2316 llvm::APInt Alignment(ClauseAlignment);
2317 if (Alignment == 0) {
2327 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2328 "alignment is not power of 2");
2329 if (Alignment != 0) {
2343 auto I = S.private_counters().begin();
2344 for (
const Expr *
E : S.counters()) {
2345 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
2346 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
2350 LocalDeclMap.erase(PrivateVD);
2365 if (!
C->getNumForLoops())
2367 for (
unsigned I = S.getLoopsNumber(),
E =
C->getLoopNumIterations().size();
2369 const auto *DRE = cast<DeclRefExpr>(
C->getLoopCounter(I));
2370 const auto *VD = cast<VarDecl>(DRE->getDecl());
2373 if (DRE->refersToEnclosingVariableOrCapture()) {
2382 const Expr *Cond, llvm::BasicBlock *TrueBlock,
2383 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
2391 for (
const Expr *I : S.inits()) {
2398 for (
const Expr *
E : S.dependent_counters()) {
2402 "dependent counter must not be an iterator.");
2403 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
2406 (void)PreCondVars.
setVarAddr(CGF, VD, CounterAddr);
2408 (void)PreCondVars.
apply(CGF);
2409 for (
const Expr *
E : S.dependent_inits()) {
2423 llvm::DenseSet<const VarDecl *> SIMDLCVs;
2426 const auto *LoopDirective = cast<OMPLoopDirective>(&
D);
2427 for (
const Expr *
C : LoopDirective->counters()) {
2433 auto CurPrivate =
C->privates().begin();
2434 for (
const Expr *
E :
C->varlist()) {
2435 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
2436 const auto *PrivateVD =
2437 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
2443 assert(IsRegistered &&
"linear var already registered as private");
2461 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2470 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2528 if (isa<OMPOrderedDirective>(S))
2531 if (
const auto *CS = dyn_cast<CapturedStmt>(S))
2534 for (
const Stmt *Child : S->children()) {
2549 if (HasOrderedDirective)
2557 const Stmt *AssociatedStmt =
D.getAssociatedStmt();
2561 if (
C->getKind() == OMPC_ORDER_concurrent)
2564 if ((EKind == OMPD_simd ||
2568 return C->getModifier() == OMPC_REDUCTION_inscan;
2579 llvm::BasicBlock *DoneBB =
nullptr;
2580 auto IC =
D.counters().begin();
2581 auto IPC =
D.private_counters().begin();
2582 for (
const Expr *F :
D.finals()) {
2583 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
2584 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
2585 const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
2587 OrigVD->hasGlobalStorage() || CED) {
2589 if (llvm::Value *Cond = CondGen(*
this)) {
2594 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2629 auto VDecl = cast<VarDecl>(Helper->
getDecl());
2637 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](
CodeGenFunction &CGF,
2651 const Expr *IfCond =
nullptr;
2654 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
2656 (
C->getNameModifier() == OMPD_unknown ||
2657 C->getNameModifier() == OMPD_simd)) {
2658 IfCond =
C->getCondition();
2674 OMPLoopScope PreInitScope(CGF, S);
2696 llvm::BasicBlock *ContBlock =
nullptr;
2703 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
2710 const Expr *IVExpr = S.getIterationVariable();
2711 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
2718 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2719 CGF.
EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2733 CGF, S, CGF.
EmitLValue(S.getIterationVariable()));
2748 emitOMPLoopBodyWithStopPoint(CGF, S,
2749 CodeGenFunction::JumpDest());
2755 if (HasLastprivateClause)
2776 if (!(isa<OMPSimdlenClause>(
C) || isa<OMPSafelenClause>(
C) ||
2777 isa<OMPOrderClause>(
C) || isa<OMPAlignedClause>(
C)))
2784 if (
const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S.getRawStmt())) {
2785 if (
const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
2786 for (
const Stmt *SubStmt : SyntacticalLoop->
children()) {
2789 if (
const CompoundStmt *CS = dyn_cast<CompoundStmt>(SubStmt)) {
2793 if (isa<OMPOrderedDirective>(CSSubStmt)) {
2804static llvm::MapVector<llvm::Value *, llvm::Value *>
2806 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars;
2808 llvm::APInt ClauseAlignment(64, 0);
2809 if (
const Expr *AlignmentExpr = Clause->getAlignment()) {
2812 ClauseAlignment = AlignmentCI->getValue();
2814 for (
const Expr *
E : Clause->varlist()) {
2815 llvm::APInt Alignment(ClauseAlignment);
2816 if (Alignment == 0) {
2826 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2827 "alignment is not power of 2");
2829 AlignedVars[PtrValue] = CGF.
Builder.getInt64(Alignment.getSExtValue());
2839 bool UseOMPIRBuilder =
2841 if (UseOMPIRBuilder) {
2845 if (UseOMPIRBuilder) {
2846 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars =
2849 const Stmt *Inner = S.getRawStmt();
2850 llvm::CanonicalLoopInfo *CLI =
2851 CGF.EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
2853 llvm::OpenMPIRBuilder &OMPBuilder =
2856 llvm::ConstantInt *Simdlen =
nullptr;
2860 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2863 llvm::ConstantInt *Safelen =
nullptr;
2867 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2870 llvm::omp::OrderKind Order = llvm::omp::OrderKind::OMP_ORDER_unknown;
2872 if (
C->getKind() == OpenMPOrderClauseKind::OMPC_ORDER_concurrent) {
2873 Order = llvm::omp::OrderKind::OMP_ORDER_concurrent;
2878 OMPBuilder.applySimd(CLI, AlignedVars,
2879 nullptr, Order, Simdlen, Safelen);
2886 OMPLexicalScope
Scope(CGF, S, OMPD_unknown);
2901 OMPLexicalScope
Scope(CGF, S, OMPD_unknown);
2914 OMPTransformDirectiveScopeRAII TileScope(*
this, &S);
2920 OMPTransformDirectiveScopeRAII StripeScope(*
this, &S);
2926 OMPTransformDirectiveScopeRAII ReverseScope(*
this, &S);
2933 OMPTransformDirectiveScopeRAII InterchangeScope(*
this, &S);
2940 if (UseOMPIRBuilder) {
2942 const Stmt *Inner = S.getRawStmt();
2953 llvm::CanonicalLoopInfo *UnrolledCLI =
nullptr;
2957 OMPBuilder.unrollLoopFull(DL, CLI);
2959 uint64_t Factor = 0;
2960 if (
Expr *FactorExpr = PartialClause->getFactor()) {
2961 Factor = FactorExpr->EvaluateKnownConstInt(
getContext()).getZExtValue();
2962 assert(Factor >= 1 &&
"Only positive factors are valid");
2964 OMPBuilder.unrollLoopPartial(DL, CLI, Factor,
2965 NeedsUnrolledCLI ? &UnrolledCLI :
nullptr);
2967 OMPBuilder.unrollLoopHeuristic(DL, CLI);
2970 assert((!NeedsUnrolledCLI || UnrolledCLI) &&
2971 "NeedsUnrolledCLI implies UnrolledCLI to be set");
2988 if (
Expr *FactorExpr = PartialClause->getFactor()) {
2990 FactorExpr->EvaluateKnownConstInt(
getContext()).getZExtValue();
2991 assert(Factor >= 1 &&
"Only positive factors are valid");
2999void CodeGenFunction::EmitOMPOuterLoop(
3002 const CodeGenFunction::OMPLoopArguments &LoopArgs,
3007 const Expr *IVExpr = S.getIterationVariable();
3021 llvm::Value *BoolCondVal =
nullptr;
3022 if (!DynamicOrOrdered) {
3033 RT.
emitForNext(*
this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
3034 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
3039 llvm::BasicBlock *ExitBlock =
LoopExit.getBlock();
3044 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
3045 if (ExitBlock !=
LoopExit.getBlock()) {
3053 if (DynamicOrOrdered)
3058 BreakContinueStack.push_back(BreakContinue(S,
LoopExit, Continue));
3069 if (
C->getKind() == OMPC_ORDER_concurrent)
3075 [&S, &LoopArgs,
LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
3083 CGF.EmitOMPInnerLoop(
3086 CodeGenLoop(CGF, S, LoopExit);
3089 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
3094 BreakContinueStack.pop_back();
3095 if (!DynamicOrOrdered) {
3108 auto &&CodeGen = [DynamicOrOrdered, &S, &LoopArgs](
CodeGenFunction &CGF) {
3109 if (!DynamicOrOrdered)
3110 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3113 OMPCancelStack.emitExit(*
this, EKind, CodeGen);
3116void CodeGenFunction::EmitOMPForOuterLoop(
3119 const OMPLoopArguments &LoopArgs,
3127 LoopArgs.Chunk !=
nullptr)) &&
3128 "static non-chunked schedule does not need outer loop");
3182 const Expr *IVExpr = S.getIterationVariable();
3186 if (DynamicOrOrdered) {
3187 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
3188 CGDispatchBounds(*
this, S, LoopArgs.LB, LoopArgs.UB);
3189 llvm::Value *LBVal = DispatchBounds.first;
3190 llvm::Value *UBVal = DispatchBounds.second;
3194 IVSigned, Ordered, DipatchRTInputValues);
3197 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
3198 LoopArgs.ST, LoopArgs.Chunk);
3205 const unsigned IVSize,
3206 const bool IVSigned) {
3213 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
3214 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
3215 OuterLoopArgs.IncExpr = S.getInc();
3216 OuterLoopArgs.Init = S.getInit();
3217 OuterLoopArgs.Cond = S.getCond();
3218 OuterLoopArgs.NextLB = S.getNextLowerBound();
3219 OuterLoopArgs.NextUB = S.getNextUpperBound();
3220 OuterLoopArgs.DKind = LoopArgs.DKind;
3221 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
3223 if (DynamicOrOrdered) {
3229 const unsigned IVSize,
const bool IVSigned) {}
3231void CodeGenFunction::EmitOMPDistributeOuterLoop(
3233 OMPPrivateScope &LoopScope,
const OMPLoopArguments &LoopArgs,
3243 const Expr *IVExpr = S.getIterationVariable();
3249 IVSize, IVSigned,
false, LoopArgs.IL, LoopArgs.LB,
3250 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
3257 IncExpr = S.getDistInc();
3259 IncExpr = S.getInc();
3264 OMPLoopArguments OuterLoopArgs;
3265 OuterLoopArgs.LB = LoopArgs.LB;
3266 OuterLoopArgs.UB = LoopArgs.UB;
3267 OuterLoopArgs.ST = LoopArgs.ST;
3268 OuterLoopArgs.IL = LoopArgs.IL;
3269 OuterLoopArgs.Chunk = LoopArgs.Chunk;
3271 ? S.getCombinedEnsureUpperBound()
3272 : S.getEnsureUpperBound();
3273 OuterLoopArgs.IncExpr = IncExpr;
3275 ? S.getCombinedInit()
3278 ? S.getCombinedCond()
3281 ? S.getCombinedNextLowerBound()
3282 : S.getNextLowerBound();
3284 ? S.getCombinedNextUpperBound()
3285 : S.getNextUpperBound();
3286 OuterLoopArgs.DKind = OMPD_distribute;
3288 EmitOMPOuterLoop(
false,
false, S,
3289 LoopScope, OuterLoopArgs, CodeGenLoopContent,
3293static std::pair<LValue, LValue>
3336static std::pair<llvm::Value *, llvm::Value *>
3347 llvm::Value *LBVal =
3349 llvm::Value *UBVal =
3351 return {LBVal, UBVal};
3357 const auto &Dir = cast<OMPLoopDirective>(S);
3359 CGF.
EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
3360 llvm::Value *LBCast = CGF.
Builder.CreateIntCast(
3362 CapturedVars.push_back(LBCast);
3364 CGF.
EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
3366 llvm::Value *UBCast = CGF.
Builder.CreateIntCast(
3368 CapturedVars.push_back(UBCast);
3379 bool HasCancel =
false;
3381 if (
const auto *
D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
3382 HasCancel =
D->hasCancel();
3383 else if (
const auto *
D = dyn_cast<OMPDistributeParallelForDirective>(&S))
3384 HasCancel =
D->hasCancel();
3385 else if (
const auto *
D =
3386 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
3387 HasCancel =
D->hasCancel();
3397 CGInlinedWorksharingLoop,
3407 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
3417 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
3426 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3437 llvm::Constant *
Addr;
3440 S, ParentName, Fn,
Addr,
true, CodeGen);
3441 assert(Fn &&
Addr &&
"Target device function emission failed.");
3453struct ScheduleKindModifiersTy {
3469 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3470 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3476 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3484 bool HasLastprivateClause;
3487 OMPLoopScope PreInitScope(*
this, S);
3492 llvm::BasicBlock *ContBlock =
nullptr;
3499 emitPreCond(*
this, S, S.getPreCond(), ThenBlock, ContBlock,
3506 bool Ordered =
false;
3508 if (OrderedClause->getNumForLoops())
3518 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*
this, S);
3519 LValue LB = Bounds.first;
3520 LValue UB = Bounds.second;
3535 *
this, S.getBeginLoc(), OMPD_unknown,
false,
3540 *
this, S,
EmitLValue(S.getIterationVariable()));
3550 const Expr *ChunkExpr =
nullptr;
3553 ScheduleKind.
Schedule =
C->getScheduleKind();
3554 ScheduleKind.
M1 =
C->getFirstScheduleModifier();
3555 ScheduleKind.
M2 =
C->getSecondScheduleModifier();
3556 ChunkExpr =
C->getChunkSize();
3560 *
this, S, ScheduleKind.
Schedule, ChunkExpr);
3562 bool HasChunkSizeOne =
false;
3563 llvm::Value *Chunk =
nullptr;
3567 S.getIterationVariable()->getType(),
3571 llvm::APSInt EvaluatedChunk =
Result.Val.getInt();
3572 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
3581 bool StaticChunkedOne =
3583 Chunk !=
nullptr) &&
3587 (ScheduleKind.
Schedule == OMPC_SCHEDULE_static &&
3588 !(ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
3589 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
3590 ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
3591 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
3593 Chunk !=
nullptr) ||
3594 StaticChunkedOne) &&
3604 if (C->getKind() == OMPC_ORDER_concurrent)
3605 CGF.LoopStack.setParallel(true);
3608 [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
3617 IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
3618 UB.getAddress(), ST.getAddress(),
3619 StaticChunkedOne ? Chunk :
nullptr);
3621 CGF, S.getBeginLoc(), EKind, ScheduleKind, StaticInit);
3623 if (!StaticChunkedOne)
3642 StaticChunkedOne ? S.getCombinedParForInDistCond()
3644 StaticChunkedOne ? S.getDistInc() : S.getInc(),
3646 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
3656 OMPCancelStack.emitExit(*
this, EKind, CodeGen);
3663 LoopArguments.DKind = OMPD_for;
3664 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
3665 LoopArguments, CGDispatchBounds);
3669 return CGF.
Builder.CreateIsNotNull(
3675 ? OMPD_parallel_for_simd
3680 return CGF.
Builder.CreateIsNotNull(
3684 if (HasLastprivateClause)
3690 return CGF.
Builder.CreateIsNotNull(
3701 return HasLastprivateClause;
3707static std::pair<LValue, LValue>
3709 const auto &LS = cast<OMPLoopDirective>(S);
3721static std::pair<llvm::Value *, llvm::Value *>
3724 const auto &LS = cast<OMPLoopDirective>(S);
3725 const Expr *IVExpr = LS.getIterationVariable();
3727 llvm::Value *LBVal = CGF.
Builder.getIntN(IVSize, 0);
3729 return {LBVal, UBVal};
3741 llvm::function_ref<llvm::Value *(
CodeGenFunction &)> NumIteratorsGen) {
3742 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3743 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3749 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
3750 "Only inscan reductions are expected.");
3751 Shareds.append(
C->varlist_begin(),
C->varlist_end());
3752 Privates.append(
C->privates().begin(),
C->privates().end());
3753 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
3754 CopyArrayTemps.append(
C->copy_array_temps().begin(),
3755 C->copy_array_temps().end());
3763 auto *ITA = CopyArrayTemps.begin();
3764 for (
const Expr *IRef : Privates) {
3765 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
3768 if (PrivateVD->getType()->isVariablyModifiedType()) {
3774 cast<OpaqueValueExpr>(
3775 cast<VariableArrayType>((*ITA)->getType()->getAsArrayTypeUnsafe())
3779 CGF.
EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl()));
3793 llvm::function_ref<llvm::Value *(
CodeGenFunction &)> NumIteratorsGen) {
3794 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3795 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3803 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
3804 "Only inscan reductions are expected.");
3805 Shareds.append(
C->varlist_begin(),
C->varlist_end());
3806 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
3807 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
3808 Privates.append(
C->privates().begin(),
C->privates().end());
3809 CopyOps.append(
C->copy_ops().begin(),
C->copy_ops().end());
3810 CopyArrayElems.append(
C->copy_array_elems().begin(),
3811 C->copy_array_elems().end());
3815 llvm::Value *OMPLast = CGF.
Builder.CreateNSWSub(
3816 OMPScanNumIterations,
3817 llvm::ConstantInt::get(CGF.
SizeTy, 1,
false));
3818 for (
unsigned I = 0,
E = CopyArrayElems.size(); I <
E; ++I) {
3819 const Expr *PrivateExpr = Privates[I];
3820 const Expr *OrigExpr = Shareds[I];
3821 const Expr *CopyArrayElem = CopyArrayElems[I];
3824 cast<OpaqueValueExpr>(
3825 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3831 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
3832 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);
3860 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3861 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3868 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
3869 "Only inscan reductions are expected.");
3870 Privates.append(
C->privates().begin(),
C->privates().end());
3871 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
3872 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
3873 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
3874 CopyArrayElems.append(
C->copy_array_elems().begin(),
3875 C->copy_array_elems().end());
3890 auto &&CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,
3897 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
3898 llvm::BasicBlock *LoopBB = CGF.createBasicBlock(
"omp.outer.log.scan.body");
3899 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
"omp.outer.log.scan.exit");
3901 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy);
3903 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);
3904 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);
3905 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy);
3906 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);
3907 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);
3908 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
3909 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));
3911 CGF.EmitBlock(LoopBB);
3912 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);
3914 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3915 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);
3916 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);
3919 llvm::BasicBlock *InnerLoopBB =
3920 CGF.createBasicBlock(
"omp.inner.log.scan.body");
3921 llvm::BasicBlock *InnerExitBB =
3922 CGF.createBasicBlock(
"omp.inner.log.scan.exit");
3923 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);
3924 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3925 CGF.EmitBlock(InnerLoopBB);
3926 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3927 IVal->addIncoming(NMin1, LoopBB);
3930 auto *ILHS = LHSs.begin();
3931 auto *IRHS = RHSs.begin();
3932 for (
const Expr *CopyArrayElem : CopyArrayElems) {
3933 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
3934 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
3939 cast<OpaqueValueExpr>(
3940 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3942 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();
3947 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);
3950 cast<OpaqueValueExpr>(
3951 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3953 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();
3960 CGF.CGM.getOpenMPRuntime().emitReduction(
3961 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
3965 llvm::Value *NextIVal =
3966 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));
3967 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());
3968 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);
3969 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3970 CGF.EmitBlock(InnerExitBB);
3972 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));
3973 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock());
3975 llvm::Value *NextPow2K =
3976 CGF.Builder.CreateShl(Pow2K, 1,
"",
true);
3977 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());
3978 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal);
3979 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB);
3981 CGF.EmitBlock(ExitBB);
3987 CGF, S.getBeginLoc(), OMPD_unknown,
false,
4001 bool HasLastprivates;
4005 return C->getModifier() == OMPC_REDUCTION_inscan;
4009 OMPLoopScope LoopScope(CGF, S);
4012 const auto &&FirstGen = [&S, HasCancel, EKind](
CodeGenFunction &CGF) {
4021 const auto &&SecondGen = [&S, HasCancel, EKind,
4039 return HasLastprivates;
4049 if (isa<OMPNowaitClause, OMPBindClause>(
C))
4052 if (
auto *SC = dyn_cast<OMPScheduleClause>(
C)) {
4057 switch (SC->getScheduleKind()) {
4058 case OMPC_SCHEDULE_auto:
4059 case OMPC_SCHEDULE_dynamic:
4060 case OMPC_SCHEDULE_runtime:
4061 case OMPC_SCHEDULE_guided:
4062 case OMPC_SCHEDULE_static:
4075static llvm::omp::ScheduleKind
4077 switch (ScheduleClauseKind) {
4079 return llvm::omp::OMP_SCHEDULE_Default;
4080 case OMPC_SCHEDULE_auto:
4081 return llvm::omp::OMP_SCHEDULE_Auto;
4082 case OMPC_SCHEDULE_dynamic:
4083 return llvm::omp::OMP_SCHEDULE_Dynamic;
4084 case OMPC_SCHEDULE_guided:
4085 return llvm::omp::OMP_SCHEDULE_Guided;
4086 case OMPC_SCHEDULE_runtime:
4087 return llvm::omp::OMP_SCHEDULE_Runtime;
4088 case OMPC_SCHEDULE_static:
4089 return llvm::omp::OMP_SCHEDULE_Static;
4091 llvm_unreachable(
"Unhandled schedule kind");
4098 bool HasLastprivates =
false;
4101 auto &&CodeGen = [&S, &
CGM, HasCancel, &HasLastprivates,
4104 if (UseOMPIRBuilder) {
4107 llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;
4108 llvm::Value *ChunkSize =
nullptr;
4112 if (
const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
4117 const Stmt *Inner = S.getRawStmt();
4118 llvm::CanonicalLoopInfo *CLI =
4121 llvm::OpenMPIRBuilder &OMPBuilder =
4123 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4125 cantFail(OMPBuilder.applyWorkshareLoop(
4126 CGF.
Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
4127 SchedKind, ChunkSize,
false,
4138 OMPLexicalScope
Scope(CGF, S, OMPD_unknown);
4143 if (!UseOMPIRBuilder) {
4157 bool HasLastprivates =
false;
4165 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4178 llvm::Value *
Init =
nullptr) {
4186 const Stmt *
CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4188 bool HasLastprivates =
false;
4194 C.getIntTypeForBitwidth(32, 1);
4197 CGF.Builder.getInt32(0));
4198 llvm::ConstantInt *GlobalUBVal = CS !=
nullptr
4199 ? CGF.Builder.getInt32(CS->size() - 1)
4200 : CGF.Builder.getInt32(0);
4204 CGF.Builder.getInt32(1));
4206 CGF.Builder.getInt32(0));
4233 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
".omp.sections.exit");
4235 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
4236 ExitBB, CS ==
nullptr ? 1 : CS->size());
4238 unsigned CaseNumber = 0;
4240 auto CaseBB = CGF.createBasicBlock(
".omp.sections.case");
4241 CGF.EmitBlock(CaseBB);
4242 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
4243 CGF.EmitStmt(SubStmt);
4244 CGF.EmitBranch(ExitBB);
4248 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(
".omp.sections.case");
4249 CGF.EmitBlock(CaseBB);
4250 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
4252 CGF.EmitBranch(ExitBB);
4254 CGF.EmitBlock(ExitBB,
true);
4258 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
4262 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4263 CGF, S.getBeginLoc(), OMPD_unknown,
false,
4266 CGF.EmitOMPPrivateClause(S, LoopScope);
4268 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4269 CGF.EmitOMPReductionClauseInit(S, LoopScope);
4270 (void)LoopScope.Privatize();
4272 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4276 ScheduleKind.
Schedule = OMPC_SCHEDULE_static;
4280 CGF.CGM.getOpenMPRuntime().emitForStaticInit(CGF, S.getBeginLoc(), EKind,
4281 ScheduleKind, StaticInit);
4283 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
4284 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4285 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
4286 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
4288 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
4290 CGF.EmitOMPInnerLoop(S,
false, Cond, Inc, BodyGen,
4294 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
4297 CGF.OMPCancelStack.emitExit(CGF, EKind, CodeGen);
4298 CGF.EmitOMPReductionClauseFinal(S, OMPD_parallel);
4301 return CGF.
Builder.CreateIsNotNull(
4306 if (HasLastprivates)
4313 bool HasCancel =
false;
4314 if (
auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
4315 HasCancel = OSD->hasCancel();
4316 else if (
auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
4317 HasCancel = OPSD->hasCancel();
4318 OMPCancelStackRAII CancelRegion(*
this, EKind, HasCancel);
4342 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4347 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4361 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4362 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4364 auto FiniCB = [](InsertPointTy IP) {
4367 return llvm::Error::success();
4370 const CapturedStmt *ICS = S.getInnermostCapturedStmt();
4371 const Stmt *
CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4376 auto SectionCB = [
this, SubStmt](InsertPointTy AllocaIP,
4377 InsertPointTy CodeGenIP) {
4379 *
this, SubStmt, AllocaIP, CodeGenIP,
"section");
4380 return llvm::Error::success();
4382 SectionCBVector.push_back(SectionCB);
4385 auto SectionCB = [
this,
CapturedStmt](InsertPointTy AllocaIP,
4386 InsertPointTy CodeGenIP) {
4389 return llvm::Error::success();
4391 SectionCBVector.push_back(SectionCB);
4398 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4399 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
4409 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4411 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4412 cantFail(OMPBuilder.createSections(
4413 Builder, AllocaIP, SectionCBVector, PrivCB, FiniCB, S.hasCancel(),
4421 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4436 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4438 const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
4439 auto FiniCB = [
this](InsertPointTy IP) {
4441 return llvm::Error::success();
4444 auto BodyGenCB = [SectionRegionBodyStmt,
this](InsertPointTy AllocaIP,
4445 InsertPointTy CodeGenIP) {
4447 *
this, SectionRegionBodyStmt, AllocaIP, CodeGenIP,
"section");
4448 return llvm::Error::success();
4453 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4454 cantFail(OMPBuilder.createSection(
Builder, BodyGenCB, FiniCB));
4474 CopyprivateVars.append(
C->varlist_begin(),
C->varlist_end());
4475 DestExprs.append(
C->destination_exprs().begin(),
4476 C->destination_exprs().end());
4477 SrcExprs.append(
C->source_exprs().begin(),
C->source_exprs().end());
4478 AssignmentOps.append(
C->assignment_ops().begin(),
4479 C->assignment_ops().end());
4488 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4493 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4495 CopyprivateVars, DestExprs,
4496 SrcExprs, AssignmentOps);
4500 if (!S.getSingleClause<
OMPNowaitClause>() && CopyprivateVars.empty()) {
4502 *
this, S.getBeginLoc(),
4520 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4522 const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();
4524 auto FiniCB = [
this](InsertPointTy IP) {
4526 return llvm::Error::success();
4529 auto BodyGenCB = [MasterRegionBodyStmt,
this](InsertPointTy AllocaIP,
4530 InsertPointTy CodeGenIP) {
4532 *
this, MasterRegionBodyStmt, AllocaIP, CodeGenIP,
"master");
4533 return llvm::Error::success();
4538 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4539 cantFail(OMPBuilder.createMaster(
Builder, BodyGenCB, FiniCB));
4554 Expr *Filter =
nullptr;
4556 Filter = FilterClause->getThreadID();
4564 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4566 const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();
4567 const Expr *Filter =
nullptr;
4569 Filter = FilterClause->getThreadID();
4570 llvm::Value *FilterVal = Filter
4574 auto FiniCB = [
this](InsertPointTy IP) {
4576 return llvm::Error::success();
4579 auto BodyGenCB = [MaskedRegionBodyStmt,
this](InsertPointTy AllocaIP,
4580 InsertPointTy CodeGenIP) {
4582 *
this, MaskedRegionBodyStmt, AllocaIP, CodeGenIP,
"masked");
4583 return llvm::Error::success();
4588 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
4589 OMPBuilder.createMasked(
Builder, BodyGenCB, FiniCB, FilterVal));
4602 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4604 const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();
4605 const Expr *Hint =
nullptr;
4606 if (
const auto *HintClause = S.getSingleClause<
OMPHintClause>())
4607 Hint = HintClause->getHint();
4612 llvm::Value *HintInst =
nullptr;
4617 auto FiniCB = [
this](InsertPointTy IP) {
4619 return llvm::Error::success();
4622 auto BodyGenCB = [CriticalRegionBodyStmt,
this](InsertPointTy AllocaIP,
4623 InsertPointTy CodeGenIP) {
4625 *
this, CriticalRegionBodyStmt, AllocaIP, CodeGenIP,
"critical");
4626 return llvm::Error::success();
4631 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4632 cantFail(OMPBuilder.createCritical(
Builder, BodyGenCB, FiniCB,
4633 S.getDirectiveName().getAsString(),
4642 CGF.
EmitStmt(S.getAssociatedStmt());
4644 const Expr *Hint =
nullptr;
4645 if (
const auto *HintClause = S.getSingleClause<
OMPHintClause>())
4646 Hint = HintClause->getHint();
4650 S.getDirectiveName().getAsString(),
4651 CodeGen, S.getBeginLoc(), Hint);
4668 OMPLoopScope LoopScope(CGF, S);
4673 return C->getModifier() == OMPC_REDUCTION_inscan;
4702 OMPLoopScope LoopScope(CGF, S);
4707 return C->getModifier() == OMPC_REDUCTION_inscan;
4783 CGF.EmitSections(S);
4797class CheckVarsEscapingUntiedTaskDeclContext final
4802 explicit CheckVarsEscapingUntiedTaskDeclContext() =
default;
4803 ~CheckVarsEscapingUntiedTaskDeclContext() =
default;
4804 void VisitDeclStmt(
const DeclStmt *S) {
4808 for (
const Decl *
D : S->decls()) {
4809 if (
const auto *VD = dyn_cast_or_null<VarDecl>(
D))
4811 PrivateDecls.push_back(VD);
4817 void VisitBlockExpr(
const BlockExpr *) {}
4818 void VisitStmt(
const Stmt *S) {
4821 for (
const Stmt *Child : S->children())
4835 bool OmpAllMemory =
false;
4838 return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||
4839 C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;
4841 OmpAllMemory =
true;
4846 Data.Dependences.emplace_back(OMPC_DEPEND_outallmemory,
4855 if (Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)
4857 if (OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))
4860 Data.Dependences.emplace_back(
C->getDependencyKind(),
C->getModifier());
4861 DD.
DepExprs.append(
C->varlist_begin(),
C->varlist_end());
4870 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
4872 auto PartId = std::next(I);
4873 auto TaskT = std::next(I, 4);
4878 const Expr *Cond = Clause->getCondition();
4881 Data.Final.setInt(CondConstant);
4886 Data.Final.setInt(
false);
4890 const Expr *Prio = Clause->getPriority();
4891 Data.Priority.setInt(
true);
4899 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
4902 auto IRef =
C->varlist_begin();
4903 for (
const Expr *IInit :
C->private_copies()) {
4904 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4905 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4906 Data.PrivateVars.push_back(*IRef);
4907 Data.PrivateCopies.push_back(IInit);
4912 EmittedAsPrivate.clear();
4915 auto IRef =
C->varlist_begin();
4916 auto IElemInitRef =
C->inits().begin();
4917 for (
const Expr *IInit :
C->private_copies()) {
4918 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4919 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4920 Data.FirstprivateVars.push_back(*IRef);
4921 Data.FirstprivateCopies.push_back(IInit);
4922 Data.FirstprivateInits.push_back(*IElemInitRef);
4929 llvm::MapVector<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
4931 auto IRef =
C->varlist_begin();
4932 auto ID =
C->destination_exprs().begin();
4933 for (
const Expr *IInit :
C->private_copies()) {
4934 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4935 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4936 Data.LastprivateVars.push_back(*IRef);
4937 Data.LastprivateCopies.push_back(IInit);
4939 LastprivateDstsOrigs.insert(
4940 std::make_pair(cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
4941 cast<DeclRefExpr>(*IRef)));
4949 Data.ReductionVars.append(
C->varlist_begin(),
C->varlist_end());
4950 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
4951 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
4952 Data.ReductionOps.append(
C->reduction_ops().begin(),
4953 C->reduction_ops().end());
4954 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
4955 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
4958 *
this, S.getBeginLoc(), LHSs, RHSs,
Data);
4963 CheckVarsEscapingUntiedTaskDeclContext Checker;
4964 Checker.Visit(S.getInnermostCapturedStmt()->getCapturedStmt());
4965 Data.PrivateLocals.append(Checker.getPrivateDecls().begin(),
4966 Checker.getPrivateDecls().end());
4968 auto &&CodeGen = [&
Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
4971 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
4972 std::pair<Address, Address>>
4977 if (
auto *DI = CGF.getDebugInfo()) {
4978 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
4979 CGF.CapturedStmtInfo->getCaptureFields();
4980 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
4981 if (CaptureFields.size() && ContextValue) {
4982 unsigned CharWidth = CGF.getContext().getCharWidth();
4996 for (
auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
4997 const VarDecl *SharedVar = It->first;
5000 CGF.getContext().getASTRecordLayout(CaptureRecord);
5003 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
5004 (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,
5005 CGF.Builder,
false);
5008 auto UpdateExpr = [](llvm::LLVMContext &Ctx,
auto *
Declare,
5013 Ops.push_back(llvm::dwarf::DW_OP_plus_uconst);
5014 Ops.push_back(Offset);
5016 Ops.push_back(llvm::dwarf::DW_OP_deref);
5017 Declare->setExpression(llvm::DIExpression::get(Ctx, Ops));
5019 llvm::Instruction &
Last = CGF.Builder.GetInsertBlock()->back();
5020 if (
auto DDI = dyn_cast<llvm::DbgVariableIntrinsic>(&
Last))
5021 UpdateExpr(DDI->getContext(), DDI, Offset);
5024 assert(!
Last.isTerminator() &&
"unexpected terminator");
5026 CGF.Builder.GetInsertBlock()->getTrailingDbgRecords()) {
5027 for (llvm::DbgVariableRecord &DVR : llvm::reverse(
5028 llvm::filterDbgVars(Marker->getDbgRecordRange()))) {
5029 UpdateExpr(
Last.getContext(), &DVR, Offset);
5037 if (!
Data.PrivateVars.empty() || !
Data.FirstprivateVars.empty() ||
5038 !
Data.LastprivateVars.empty() || !
Data.PrivateLocals.empty()) {
5039 enum { PrivatesParam = 2, CopyFnParam = 3 };
5040 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5042 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
5048 CallArgs.push_back(PrivatesPtr);
5049 ParamTypes.push_back(PrivatesPtr->getType());
5050 for (
const Expr *
E :
Data.PrivateVars) {
5051 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
5053 CGF.getContext().getPointerType(
E->
getType()),
".priv.ptr.addr");
5054 PrivatePtrs.emplace_back(VD, PrivatePtr);
5056 ParamTypes.push_back(PrivatePtr.
getType());
5058 for (
const Expr *
E :
Data.FirstprivateVars) {
5059 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
5061 CGF.CreateMemTemp(CGF.getContext().getPointerType(
E->
getType()),
5062 ".firstpriv.ptr.addr");
5063 PrivatePtrs.emplace_back(VD, PrivatePtr);
5064 FirstprivatePtrs.emplace_back(VD, PrivatePtr);
5066 ParamTypes.push_back(PrivatePtr.
getType());
5068 for (
const Expr *
E :
Data.LastprivateVars) {
5069 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
5071 CGF.CreateMemTemp(CGF.getContext().getPointerType(
E->
getType()),
5072 ".lastpriv.ptr.addr");
5073 PrivatePtrs.emplace_back(VD, PrivatePtr);
5075 ParamTypes.push_back(PrivatePtr.
getType());
5080 Ty = CGF.getContext().getPointerType(Ty);
5082 Ty = CGF.getContext().getPointerType(Ty);
5084 CGF.getContext().getPointerType(Ty),
".local.ptr.addr");
5085 auto Result = UntiedLocalVars.insert(
5088 if (
Result.second ==
false)
5089 *
Result.first = std::make_pair(
5092 ParamTypes.push_back(PrivatePtr.
getType());
5094 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
5096 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5097 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
5098 for (
const auto &Pair : LastprivateDstsOrigs) {
5099 const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
5102 CGF.CapturedStmtInfo->lookup(OrigVD) !=
nullptr,
5104 Pair.second->getExprLoc());
5105 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress());
5107 for (
const auto &Pair : PrivatePtrs) {
5109 CGF.Builder.CreateLoad(Pair.second),
5110 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5111 CGF.getContext().getDeclAlign(Pair.first));
5112 Scope.addPrivate(Pair.first, Replacement);
5113 if (
auto *DI = CGF.getDebugInfo())
5114 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
5115 (void)DI->EmitDeclareOfAutoVariable(
5116 Pair.first, Pair.second.getBasePointer(), CGF.Builder,
5121 for (
auto &Pair : UntiedLocalVars) {
5122 QualType VDType = Pair.first->getType().getNonReferenceType();
5123 if (Pair.first->getType()->isLValueReferenceType())
5124 VDType = CGF.getContext().getPointerType(VDType);
5126 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
5129 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)),
5130 CGF.getPointerAlign());
5131 Pair.second.first = Replacement;
5132 Ptr = CGF.Builder.CreateLoad(Replacement);
5133 Replacement =
Address(Ptr, CGF.ConvertTypeForMem(VDType),
5134 CGF.getContext().getDeclAlign(Pair.first));
5135 Pair.second.second = Replacement;
5137 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
5138 Address Replacement(Ptr, CGF.ConvertTypeForMem(VDType),
5139 CGF.getContext().getDeclAlign(Pair.first));
5140 Pair.second.first = Replacement;
5144 if (
Data.Reductions) {
5146 for (
const auto &Pair : FirstprivatePtrs) {
5148 CGF.Builder.CreateLoad(Pair.second),
5149 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5150 CGF.getContext().getDeclAlign(Pair.first));
5151 FirstprivateScope.
addPrivate(Pair.first, Replacement);
5154 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5156 Data.ReductionCopies,
Data.ReductionOps);
5157 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
5159 for (
unsigned Cnt = 0,
E =
Data.ReductionVars.size(); Cnt <
E; ++Cnt) {
5165 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5167 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5170 CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),
5171 CGF.getContext().VoidPtrTy,
5172 CGF.getContext().getPointerType(
5173 Data.ReductionCopies[Cnt]->getType()),
5174 Data.ReductionCopies[Cnt]->getExprLoc()),
5175 CGF.ConvertTypeForMem(
Data.ReductionCopies[Cnt]->getType()),
5176 Replacement.getAlignment());
5182 (void)
Scope.Privatize();
5188 auto IPriv =
C->privates().begin();
5189 auto IRed =
C->reduction_ops().begin();
5190 auto ITD =
C->taskgroup_descriptors().begin();
5191 for (
const Expr *Ref :
C->varlist()) {
5192 InRedVars.emplace_back(Ref);
5193 InRedPrivs.emplace_back(*IPriv);
5194 InRedOps.emplace_back(*IRed);
5195 TaskgroupDescriptors.emplace_back(*ITD);
5196 std::advance(IPriv, 1);
5197 std::advance(IRed, 1);
5198 std::advance(ITD, 1);
5204 if (!InRedVars.empty()) {
5206 for (
unsigned Cnt = 0,
E = InRedVars.size(); Cnt <
E; ++Cnt) {
5214 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5216 llvm::Value *ReductionsPtr;
5217 if (
const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
5218 ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr),
5219 TRExpr->getExprLoc());
5221 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5223 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5226 CGF.EmitScalarConversion(
5227 Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,
5228 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
5229 InRedPrivs[Cnt]->getExprLoc()),
5230 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
5231 Replacement.getAlignment());
5245 S, *I, *PartId, *TaskT, EKind, CodeGen,
Data.Tied,
Data.NumberOfParts);
5246 OMPLexicalScope
Scope(*
this, S, std::nullopt,
5249 TaskGen(*
this, OutlinedFn,
Data);
5266 QualType ElemType =
C.getBaseElementType(Ty);
5276 Data.FirstprivateVars.emplace_back(OrigRef);
5277 Data.FirstprivateCopies.emplace_back(PrivateRef);
5278 Data.FirstprivateInits.emplace_back(InitRef);
5291 auto PartId = std::next(I);
5292 auto TaskT = std::next(I, 4);
5295 Data.Final.setInt(
false);
5298 auto IRef =
C->varlist_begin();
5299 auto IElemInitRef =
C->inits().begin();
5300 for (
auto *IInit :
C->private_copies()) {
5301 Data.FirstprivateVars.push_back(*IRef);
5302 Data.FirstprivateCopies.push_back(IInit);
5303 Data.FirstprivateInits.push_back(*IElemInitRef);
5311 Data.ReductionVars.append(
C->varlist_begin(),
C->varlist_end());
5312 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
5313 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
5314 Data.ReductionOps.append(
C->reduction_ops().begin(),
5315 C->reduction_ops().end());
5316 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
5317 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
5332 getContext(),
Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5334 getContext(),
Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5346 if (!isa_and_nonnull<llvm::ConstantPointerNull>(
5349 getContext(),
Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5356 auto &&CodeGen = [&
Data, &S, CS, &BodyGen, BPVD, PVD, SVD, MVD, EKind,
5360 if (!
Data.FirstprivateVars.empty()) {
5361 enum { PrivatesParam = 2, CopyFnParam = 3 };
5362 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5364 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
5370 CallArgs.push_back(PrivatesPtr);
5371 ParamTypes.push_back(PrivatesPtr->getType());
5372 for (
const Expr *
E :
Data.FirstprivateVars) {
5373 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
5375 CGF.CreateMemTemp(CGF.getContext().getPointerType(
E->
getType()),
5376 ".firstpriv.ptr.addr");
5377 PrivatePtrs.emplace_back(VD, PrivatePtr);
5379 ParamTypes.push_back(PrivatePtr.
getType());
5381 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
5383 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5384 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
5385 for (
const auto &Pair : PrivatePtrs) {
5387 CGF.Builder.CreateLoad(Pair.second),
5388 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5389 CGF.getContext().getDeclAlign(Pair.first));
5390 Scope.addPrivate(Pair.first, Replacement);
5393 CGF.processInReduction(S,
Data, CGF, CS,
Scope);
5396 CGF.GetAddrOfLocalVar(BPVD), 0);
5398 CGF.GetAddrOfLocalVar(PVD), 0);
5399 InputInfo.
SizesArray = CGF.Builder.CreateConstArrayGEP(
5400 CGF.GetAddrOfLocalVar(SVD), 0);
5403 InputInfo.
MappersArray = CGF.Builder.CreateConstArrayGEP(
5404 CGF.GetAddrOfLocalVar(MVD), 0);
5408 OMPLexicalScope LexScope(CGF, S, OMPD_task,
false);
5410 if (CGF.CGM.getLangOpts().OpenMP >= 51 &&
5415 CGF.CGM.getOpenMPRuntime().emitThreadLimitClause(
5416 CGF, TL->getThreadLimit().front(), S.getBeginLoc());
5421 S, *I, *PartId, *TaskT, EKind, CodeGen,
true,
5422 Data.NumberOfParts);
5423 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<
OMPNowaitClause>() ? 1 : 0);
5428 SharedsTy, CapturedStruct, &IfCond,
Data);
5437 if (
Data.Reductions) {
5439 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5441 Data.ReductionCopies,
Data.ReductionOps);
5444 for (
unsigned Cnt = 0,
E =
Data.ReductionVars.size(); Cnt <
E; ++Cnt) {
5458 Data.ReductionCopies[Cnt]->getType()),
5459 Data.ReductionCopies[Cnt]->getExprLoc()),
5461 Replacement.getAlignment());
5466 (void)
Scope.Privatize();
5472 auto IPriv =
C->privates().begin();
5473 auto IRed =
C->reduction_ops().begin();
5474 auto ITD =
C->taskgroup_descriptors().begin();
5475 for (
const Expr *Ref :
C->varlist()) {
5476 InRedVars.emplace_back(Ref);
5477 InRedPrivs.emplace_back(*IPriv);
5478 InRedOps.emplace_back(*IRed);
5479 TaskgroupDescriptors.emplace_back(*ITD);
5480 std::advance(IPriv, 1);
5481 std::advance(IRed, 1);
5482 std::advance(ITD, 1);
5486 if (!InRedVars.empty()) {
5488 for (
unsigned Cnt = 0,
E = InRedVars.size(); Cnt <
E; ++Cnt) {
5496 llvm::Value *ReductionsPtr;
5497 if (
const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
5501 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
5509 InRedPrivs[Cnt]->getExprLoc()),
5511 Replacement.getAlignment());
5525 const Expr *IfCond =
nullptr;
5526 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
5527 if (
C->getNameModifier() == OMPD_unknown ||
5528 C->getNameModifier() == OMPD_task) {
5529 IfCond =
C->getCondition();
5540 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
5544 SharedsTy, CapturedStruct, IfCond,
5561 bool IsFatal =
false;
5580 return T.clauses().empty();
5585 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
5588 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5592 auto BodyGenCB = [&,
this](InsertPointTy AllocaIP,
5593 InsertPointTy CodeGenIP) {
5595 EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5596 return llvm::Error::success();
5601 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5602 cantFail(OMPBuilder.createTaskgroup(
Builder, AllocaIP, BodyGenCB));
5608 if (
const Expr *
E = S.getReductionRef()) {
5613 Data.ReductionVars.append(
C->varlist_begin(),
C->varlist_end());
5614 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
5615 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
5616 Data.ReductionOps.append(
C->reduction_ops().begin(),
5617 C->reduction_ops().end());
5618 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
5619 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
5621 llvm::Value *ReductionDesc =
5624 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
5629 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5636 ? llvm::AtomicOrdering::NotAtomic
5637 : llvm::AtomicOrdering::AcquireRelease;
5641 if (
const auto *FlushClause = S.getSingleClause<
OMPFlushClause>())
5643 FlushClause->varlist_end());
5646 S.getBeginLoc(), AO);
5656 for (
auto &Dep :
Data.Dependences) {
5658 *
this, Dep, DC->getBeginLoc());
5669 *
this, DOLVal, UC->getDependencyKind(), UC->getBeginLoc());
5688 if (
C->getModifier() != OMPC_REDUCTION_inscan)
5690 Shareds.append(
C->varlist_begin(),
C->varlist_end());
5691 Privates.append(
C->privates().begin(),
C->privates().end());
5692 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
5693 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
5694 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
5695 CopyOps.append(
C->copy_ops().begin(),
C->copy_ops().end());
5696 CopyArrayTemps.append(
C->copy_array_temps().begin(),
5697 C->copy_array_temps().end());
5698 CopyArrayElems.append(
C->copy_array_elems().begin(),
5699 C->copy_array_elems().end());
5743 : BreakContinueStack.back().ContinueBlock.getBlock());
5754 for (
unsigned I = 0,
E = CopyArrayElems.size(); I <
E; ++I) {
5755 const Expr *PrivateExpr = Privates[I];
5756 const Expr *TempExpr = CopyArrayTemps[I];
5758 *cast<VarDecl>(cast<DeclRefExpr>(TempExpr)->getDecl()));
5763 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5764 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
5769 *
this, ParentDir.
getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
5772 for (
unsigned I = 0,
E = CopyArrayElems.size(); I <
E; ++I) {
5773 const Expr *PrivateExpr = Privates[I];
5780 const Expr *TempExpr = CopyArrayTemps[I];
5786 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5787 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);
5792 ? BreakContinueStack.back().ContinueBlock.getBlock()
5798 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5803 const auto *IVExpr = cast<OMPLoopDirective>(ParentDir)
5804 .getIterationVariable()
5809 for (
unsigned I = 0,
E = CopyArrayElems.size(); I <
E; ++I) {
5810 const Expr *PrivateExpr = Privates[I];
5811 const Expr *OrigExpr = Shareds[I];
5812 const Expr *CopyArrayElem = CopyArrayElems[I];
5813 OpaqueValueMapping IdxMapping(
5815 cast<OpaqueValueExpr>(
5816 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
5822 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5823 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);
5826 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5829 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5834 const auto *IVExpr = cast<OMPLoopDirective>(ParentDir)
5835 .getIterationVariable()
5840 llvm::BasicBlock *ExclusiveExitBB =
nullptr;
5845 Builder.CreateCondBr(Cmp, ExclusiveExitBB, ContBB);
5848 IdxVal =
Builder.CreateNUWSub(IdxVal, llvm::ConstantInt::get(
SizeTy, 1));
5850 for (
unsigned I = 0,
E = CopyArrayElems.size(); I <
E; ++I) {
5851 const Expr *PrivateExpr = Privates[I];
5852 const Expr *OrigExpr = Shareds[I];
5853 const Expr *CopyArrayElem = CopyArrayElems[I];
5854 OpaqueValueMapping IdxMapping(
5856 cast<OpaqueValueExpr>(
5857 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
5863 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5864 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);
5879 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
5880 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
5886 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
5894 bool HasLastprivateClause =
false;
5897 OMPLoopScope PreInitScope(*
this, S);
5902 llvm::BasicBlock *ContBlock =
nullptr;
5909 emitPreCond(*
this, S, S.getPreCond(), ThenBlock, ContBlock,
5921 *
this, cast<DeclRefExpr>(
5923 ? S.getCombinedLowerBoundVariable()
5924 : S.getLowerBoundVariable())));
5926 *
this, cast<DeclRefExpr>(
5928 ? S.getCombinedUpperBoundVariable()
5929 : S.getUpperBoundVariable())));
5941 *
this, S.getBeginLoc(), OMPD_unknown,
false,
5956 llvm::Value *Chunk =
nullptr;
5959 ScheduleKind =
C->getDistScheduleKind();
5960 if (
const Expr *Ch =
C->getChunkSize()) {
5963 S.getIterationVariable()->getType(),
5969 *
this, S, ScheduleKind, Chunk);
5982 bool StaticChunked =
5986 Chunk !=
nullptr) ||
5991 StaticChunked ? Chunk :
nullptr);
5998 ? S.getCombinedEnsureUpperBound()
5999 : S.getEnsureUpperBound());
6002 ? S.getCombinedInit()
6007 ? S.getCombinedCond()
6011 Cond = S.getCombinedDistCond();
6043 [&S, &LoopScope, Cond, IncExpr,
LoopExit, &CodeGenLoop,
6046 S, LoopScope.requiresCleanups(), Cond, IncExpr,
6048 CodeGenLoop(CGF, S, LoopExit);
6051 if (StaticChunked) {
6052 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
6053 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
6054 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
6055 CGF.EmitIgnoredExpr(S.getCombinedInit());
6065 const OMPLoopArguments LoopArguments = {
6068 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
6073 return CGF.
Builder.CreateIsNotNull(
6084 return CGF.
Builder.CreateIsNotNull(
6089 if (HasLastprivateClause) {
6112 OMPLexicalScope
Scope(CGF, S, OMPD_unknown);
6128 Fn->setDoesNotRecurse();
6132template <
typename T>
6134 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
6135 llvm::OpenMPIRBuilder &OMPBuilder) {
6137 unsigned NumLoops =
C->getNumLoops();
6141 for (
unsigned I = 0; I < NumLoops; I++) {
6142 const Expr *CounterVal =
C->getLoopData(I);
6147 StoreValues.emplace_back(StoreValue);
6149 OMPDoacrossKind<T> ODK;
6150 bool IsDependSource = ODK.isSource(
C);
6152 OMPBuilder.createOrderedDepend(CGF.
Builder, AllocaIP, NumLoops,
6153 StoreValues,
".cnt.addr", IsDependSource));
6159 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
6164 assert(!S.hasAssociatedStmt() &&
"No associated statement must be in "
6165 "ordered depend|doacross construct.");
6177 auto FiniCB = [
this](InsertPointTy IP) {
6179 return llvm::Error::success();
6182 auto BodyGenCB = [&S,
C,
this](InsertPointTy AllocaIP,
6183 InsertPointTy CodeGenIP) {
6188 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
6189 Builder,
false,
".ordered.after");
6192 llvm::Function *OutlinedFn =
6194 assert(S.getBeginLoc().isValid() &&
6195 "Outlined function call location must be valid.");
6198 OutlinedFn, CapturedVars);
6203 return llvm::Error::success();
6206 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
6207 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
6208 OMPBuilder.createOrderedThreadsSimd(
Builder, BodyGenCB, FiniCB, !
C));
6215 assert(!S.hasAssociatedStmt() &&
6216 "No associated statement must be in ordered depend construct.");
6222 assert(!S.hasAssociatedStmt() &&
6223 "No associated statement must be in ordered doacross construct.");
6234 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
6235 llvm::Function *OutlinedFn =
6238 OutlinedFn, CapturedVars);
6244 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
6252 "DestType must have scalar evaluation kind.");
6253 assert(!Val.
isAggregate() &&
"Must be a scalar or complex.");
6264 "DestType must have complex evaluation kind.");
6273 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
6275 assert(Val.
isComplex() &&
"Must be a scalar or complex.");
6296 llvm::AtomicOrdering AO,
LValue LVal,
6301 LVal,
Loc, llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO),
6319 llvm_unreachable(
"Must be a scalar or complex.");
6327 assert(
V->isLValue() &&
"V of 'omp atomic read' is not lvalue");
6328 assert(
X->isLValue() &&
"X of 'omp atomic read' is not lvalue");
6337 case llvm::AtomicOrdering::Acquire:
6338 case llvm::AtomicOrdering::AcquireRelease:
6339 case llvm::AtomicOrdering::SequentiallyConsistent:
6341 llvm::AtomicOrdering::Acquire);
6343 case llvm::AtomicOrdering::Monotonic:
6344 case llvm::AtomicOrdering::Release:
6346 case llvm::AtomicOrdering::NotAtomic:
6347 case llvm::AtomicOrdering::Unordered:
6348 llvm_unreachable(
"Unexpected ordering.");
6355 llvm::AtomicOrdering AO,
const Expr *
X,
6358 assert(
X->isLValue() &&
"X of 'omp atomic write' is not lvalue");
6366 case llvm::AtomicOrdering::Release:
6367 case llvm::AtomicOrdering::AcquireRelease:
6368 case llvm::AtomicOrdering::SequentiallyConsistent:
6370 llvm::AtomicOrdering::Release);
6372 case llvm::AtomicOrdering::Acquire:
6373 case llvm::AtomicOrdering::Monotonic:
6375 case llvm::AtomicOrdering::NotAtomic:
6376 case llvm::AtomicOrdering::Unordered:
6377 llvm_unreachable(
"Unexpected ordering.");
6384 llvm::AtomicOrdering AO,
6385 bool IsXLHSInRHSPart) {
6390 if (BO == BO_Comma || !
Update.isScalar() || !
X.isSimple() ||
6391 (!isa<llvm::ConstantInt>(
Update.getScalarVal()) &&
6392 (
Update.getScalarVal()->getType() !=
X.getAddress().getElementType())) ||
6395 return std::make_pair(
false,
RValue::get(
nullptr));
6398 if (
T->isIntegerTy())
6401 if (
T->isFloatingPointTy() && (BO == BO_Add || BO == BO_Sub))
6407 if (!CheckAtomicSupport(
Update.getScalarVal()->getType(), BO) ||
6408 !CheckAtomicSupport(
X.getAddress().getElementType(), BO))
6409 return std::make_pair(
false,
RValue::get(
nullptr));
6411 bool IsInteger =
X.getAddress().getElementType()->isIntegerTy();
6412 llvm::AtomicRMWInst::BinOp RMWOp;
6415 RMWOp = IsInteger ? llvm::AtomicRMWInst::Add : llvm::AtomicRMWInst::FAdd;
6418 if (!IsXLHSInRHSPart)
6419 return std::make_pair(
false,
RValue::get(
nullptr));
6420 RMWOp = IsInteger ? llvm::AtomicRMWInst::Sub : llvm::AtomicRMWInst::FSub;
6423 RMWOp = llvm::AtomicRMWInst::And;
6426 RMWOp = llvm::AtomicRMWInst::Or;
6429 RMWOp = llvm::AtomicRMWInst::Xor;
6433 RMWOp =
X.getType()->hasSignedIntegerRepresentation()
6434 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
6435 : llvm::AtomicRMWInst::Max)
6436 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
6437 : llvm::AtomicRMWInst::UMax);
6439 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMin
6440 : llvm::AtomicRMWInst::FMax;
6444 RMWOp =
X.getType()->hasSignedIntegerRepresentation()
6445 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
6446 : llvm::AtomicRMWInst::Min)
6447 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
6448 : llvm::AtomicRMWInst::UMin);
6450 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMax
6451 : llvm::AtomicRMWInst::FMin;
6454 RMWOp = llvm::AtomicRMWInst::Xchg;
6463 return std::make_pair(
false,
RValue::get(
nullptr));
6482 llvm_unreachable(
"Unsupported atomic update operation");
6484 llvm::Value *UpdateVal =
Update.getScalarVal();
6485 if (
auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
6487 UpdateVal = CGF.
Builder.CreateIntCast(
6488 IC,
X.getAddress().getElementType(),
6489 X.getType()->hasSignedIntegerRepresentation());
6491 UpdateVal = CGF.
Builder.CreateCast(llvm::Instruction::CastOps::UIToFP, IC,
6492 X.getAddress().getElementType());
6494 llvm::AtomicRMWInst *Res =
6511 if (
X.isGlobalReg()) {
6524 llvm::AtomicOrdering AO,
const Expr *
X,
6528 "Update expr in 'atomic update' must be a binary operator.");
6536 assert(
X->isLValue() &&
"X of 'omp atomic update' is not lvalue");
6539 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
6540 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
6543 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](
RValue XRValue) {
6549 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO,
Loc, Gen);
6556 case llvm::AtomicOrdering::Release:
6557 case llvm::AtomicOrdering::AcquireRelease:
6558 case llvm::AtomicOrdering::SequentiallyConsistent:
6560 llvm::AtomicOrdering::Release);
6562 case llvm::AtomicOrdering::Acquire:
6563 case llvm::AtomicOrdering::Monotonic:
6565 case llvm::AtomicOrdering::NotAtomic:
6566 case llvm::AtomicOrdering::Unordered:
6567 llvm_unreachable(
"Unexpected ordering.");
6585 llvm_unreachable(
"Must be a scalar or complex.");
6589 llvm::AtomicOrdering AO,
6590 bool IsPostfixUpdate,
const Expr *
V,
6592 const Expr *UE,
bool IsXLHSInRHSPart,
6594 assert(
X->isLValue() &&
"X of 'omp atomic capture' is not lvalue");
6595 assert(
V->isLValue() &&
"V of 'omp atomic capture' is not lvalue");
6604 "Update expr in 'atomic capture' must be a binary operator.");
6612 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
6613 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
6615 NewVValType = XRValExpr->
getType();
6617 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
6618 IsPostfixUpdate](
RValue XRValue) {
6622 NewVVal = IsPostfixUpdate ? XRValue : Res;
6626 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO,
Loc, Gen);
6630 if (IsPostfixUpdate) {
6632 NewVVal = Res.second;
6643 NewVValType =
X->getType().getNonReferenceType();
6645 X->getType().getNonReferenceType(),
Loc);
6646 auto &&Gen = [&NewVVal, ExprRValue](
RValue XRValue) {
6652 XLValue, ExprRValue, BO_Assign,
false, AO,
6657 NewVVal = IsPostfixUpdate ? Res.
second : ExprRValue;
6673 case llvm::AtomicOrdering::Release:
6675 llvm::AtomicOrdering::Release);
6677 case llvm::AtomicOrdering::Acquire:
6679 llvm::AtomicOrdering::Acquire);
6681 case llvm::AtomicOrdering::AcquireRelease:
6682 case llvm::AtomicOrdering::SequentiallyConsistent:
6684 CGF, {},
Loc, llvm::AtomicOrdering::AcquireRelease);
6686 case llvm::AtomicOrdering::Monotonic:
6688 case llvm::AtomicOrdering::NotAtomic:
6689 case llvm::AtomicOrdering::Unordered:
6690 llvm_unreachable(
"Unexpected ordering.");
6696 CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO,
6698 const Expr *CE,
bool IsXBinopExpr,
bool IsPostfixUpdate,
bool IsFailOnly,
6700 llvm::OpenMPIRBuilder &OMPBuilder =
6703 OMPAtomicCompareOp Op;
6704 assert(isa<BinaryOperator>(CE) &&
"CE is not a BinaryOperator");
6705 switch (cast<BinaryOperator>(CE)->getOpcode()) {
6707 Op = OMPAtomicCompareOp::EQ;
6710 Op = OMPAtomicCompareOp::MIN;
6713 Op = OMPAtomicCompareOp::MAX;
6716 llvm_unreachable(
"unsupported atomic compare binary operator");
6722 auto EmitRValueWithCastIfNeeded = [&CGF,
Loc](
const Expr *
X,
const Expr *
E) {
6727 if (NewE->
getType() ==
X->getType())
6732 llvm::Value *EVal = EmitRValueWithCastIfNeeded(
X,
E);
6733 llvm::Value *DVal =
D ? EmitRValueWithCastIfNeeded(
X,
D) :
nullptr;
6734 if (
auto *CI = dyn_cast<llvm::ConstantInt>(EVal))
6735 EVal = CGF.
Builder.CreateIntCast(
6739 if (
auto *CI = dyn_cast<llvm::ConstantInt>(DVal))
6740 DVal = CGF.
Builder.CreateIntCast(
6742 D->getType()->hasSignedIntegerRepresentation());
6744 llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{
6746 X->getType()->hasSignedIntegerRepresentation(),
6747 X->getType().isVolatileQualified()};
6748 llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal;
6752 VOpVal = {
Addr.emitRawPointer(CGF),
Addr.getElementType(),
6753 V->getType()->hasSignedIntegerRepresentation(),
6754 V->getType().isVolatileQualified()};
6759 ROpVal = {
Addr.emitRawPointer(CGF),
Addr.getElementType(),
6764 if (FailAO == llvm::AtomicOrdering::NotAtomic) {
6767 CGF.
Builder.restoreIP(OMPBuilder.createAtomicCompare(
6768 CGF.
Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
6769 IsPostfixUpdate, IsFailOnly));
6771 CGF.
Builder.restoreIP(OMPBuilder.createAtomicCompare(
6772 CGF.
Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
6773 IsPostfixUpdate, IsFailOnly, FailAO));
6777 llvm::AtomicOrdering AO,
6778 llvm::AtomicOrdering FailAO,
bool IsPostfixUpdate,
6781 const Expr *CE,
bool IsXLHSInRHSPart,
6796 IsXLHSInRHSPart,
Loc);
6798 case OMPC_compare: {
6800 IsXLHSInRHSPart, IsPostfixUpdate, IsFailOnly,
Loc);
6804 llvm_unreachable(
"Clause is not allowed in 'omp atomic'.");
6811 llvm::AtomicOrdering FailAO = llvm::AtomicOrdering::NotAtomic;
6812 bool MemOrderingSpecified =
false;
6814 AO = llvm::AtomicOrdering::SequentiallyConsistent;
6815 MemOrderingSpecified =
true;
6817 AO = llvm::AtomicOrdering::AcquireRelease;
6818 MemOrderingSpecified =
true;
6820 AO = llvm::AtomicOrdering::Acquire;
6821 MemOrderingSpecified =
true;
6823 AO = llvm::AtomicOrdering::Release;
6824 MemOrderingSpecified =
true;
6826 AO = llvm::AtomicOrdering::Monotonic;
6827 MemOrderingSpecified =
true;
6829 llvm::SmallSet<OpenMPClauseKind, 2> KindsEncountered;
6838 if (K == OMPC_seq_cst || K == OMPC_acq_rel || K == OMPC_acquire ||
6839 K == OMPC_release || K == OMPC_relaxed || K == OMPC_hint)
6842 KindsEncountered.insert(K);
6847 if (KindsEncountered.contains(OMPC_compare) &&
6848 KindsEncountered.contains(OMPC_capture))
6849 Kind = OMPC_compare;
6850 if (!MemOrderingSpecified) {
6851 llvm::AtomicOrdering DefaultOrder =
6853 if (DefaultOrder == llvm::AtomicOrdering::Monotonic ||
6854 DefaultOrder == llvm::AtomicOrdering::SequentiallyConsistent ||
6855 (DefaultOrder == llvm::AtomicOrdering::AcquireRelease &&
6856 Kind == OMPC_capture)) {
6858 }
else if (DefaultOrder == llvm::AtomicOrdering::AcquireRelease) {
6859 if (Kind == OMPC_unknown || Kind == OMPC_update || Kind == OMPC_write) {
6860 AO = llvm::AtomicOrdering::Release;
6861 }
else if (Kind == OMPC_read) {
6862 assert(Kind == OMPC_read &&
"Unexpected atomic kind.");
6863 AO = llvm::AtomicOrdering::Acquire;
6868 if (KindsEncountered.contains(OMPC_compare) &&
6869 KindsEncountered.contains(OMPC_fail)) {
6870 Kind = OMPC_compare;
6871 const auto *FailClause = S.getSingleClause<
OMPFailClause>();
6874 if (FailParameter == llvm::omp::OMPC_relaxed)
6875 FailAO = llvm::AtomicOrdering::Monotonic;
6876 else if (FailParameter == llvm::omp::OMPC_acquire)
6877 FailAO = llvm::AtomicOrdering::Acquire;
6878 else if (FailParameter == llvm::omp::OMPC_seq_cst)
6879 FailAO = llvm::AtomicOrdering::SequentiallyConsistent;
6886 S.getV(), S.getR(), S.getExpr(), S.getUpdateExpr(),
6887 S.getD(), S.getCondExpr(), S.isXLHSInRHSPart(),
6888 S.isFailOnly(), S.getBeginLoc());
6899 OMPLexicalScope
Scope(CGF, S, OMPD_target);
6902 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
6908 llvm::Function *Fn =
nullptr;
6909 llvm::Constant *FnID =
nullptr;
6911 const Expr *IfCond =
nullptr;
6913 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
6914 if (
C->getNameModifier() == OMPD_unknown ||
6915 C->getNameModifier() == OMPD_target) {
6916 IfCond =
C->getCondition();
6922 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device(
6925 Device.setPointerAndInt(
C->getDevice(),
C->getModifier());
6930 bool IsOffloadEntry =
true;
6934 IsOffloadEntry =
false;
6937 IsOffloadEntry =
false;
6939 if (
CGM.
getLangOpts().OpenMPOffloadMandatory && !IsOffloadEntry) {
6942 "No offloading entry generated while offloading is mandatory.");
6946 assert(CGF.
CurFuncDecl &&
"No parent declaration for target region!");
6947 StringRef ParentName;
6950 if (
const auto *
D = dyn_cast<CXXConstructorDecl>(CGF.
CurFuncDecl))
6952 else if (
const auto *
D = dyn_cast<CXXDestructorDecl>(CGF.
CurFuncDecl))
6960 IsOffloadEntry, CodeGen);
6961 OMPLexicalScope
Scope(CGF, S, OMPD_task);
6962 auto &&SizeEmitter =
6965 if (IsOffloadEntry) {
6966 OMPLoopScope(CGF,
D);
6968 llvm::Value *NumIterations = CGF.
EmitScalarExpr(
D.getNumIterations());
6969 NumIterations = CGF.
Builder.CreateIntCast(NumIterations, CGF.
Int64Ty,
6971 return NumIterations;
6989 CGF.
EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
6994 StringRef ParentName,
7000 llvm::Constant *
Addr;
7003 S, ParentName, Fn,
Addr,
true, CodeGen);
7004 assert(Fn &&
Addr &&
"Target device function emission failed.");
7018 const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
7019 llvm::Function *OutlinedFn =
7027 const Expr *NumTeams = NT ? NT->getNumTeams().front() :
nullptr;
7028 const Expr *ThreadLimit = TL ? TL->getThreadLimit().front() :
nullptr;
7034 OMPTeamsScope
Scope(CGF, S);
7050 CGF.
EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
7060 auto *CS = S.getCapturedStmt(OMPD_teams);
7072 CGF.
EmitStmt(CS->getCapturedStmt());
7087 llvm::Constant *
Addr;
7090 S, ParentName, Fn,
Addr,
true, CodeGen);
7091 assert(Fn &&
Addr &&
"Target device function emission failed.");
7133 llvm::Constant *
Addr;
7136 S, ParentName, Fn,
Addr,
true, CodeGen);
7137 assert(Fn &&
Addr &&
"Target device function emission failed.");
7179 llvm::Constant *
Addr;
7182 S, ParentName, Fn,
Addr,
true, CodeGen);
7183 assert(Fn &&
Addr &&
"Target device function emission failed.");
7277 CGF, OMPD_distribute, CodeGenDistribute,
false);
7288 llvm::Value *
Device =
nullptr;
7289 llvm::Value *NumDependences =
nullptr;
7290 llvm::Value *DependenceList =
nullptr;
7298 if (!
Data.Dependences.empty()) {
7300 std::tie(NumDependences, DependenciesArray) =
7310 "OMPNowaitClause clause is used separately in OMPInteropDirective.");
7313 if (!ItOMPInitClause.empty()) {
7316 llvm::Value *InteropvarPtr =
7318 llvm::omp::OMPInteropType InteropType =
7319 llvm::omp::OMPInteropType::Unknown;
7320 if (
C->getIsTarget()) {
7321 InteropType = llvm::omp::OMPInteropType::Target;
7323 assert(
C->getIsTargetSync() &&
7324 "Expected interop-type target/targetsync");
7325 InteropType = llvm::omp::OMPInteropType::TargetSync;
7327 OMPBuilder.createOMPInteropInit(
Builder, InteropvarPtr, InteropType,
7328 Device, NumDependences, DependenceList,
7329 Data.HasNowaitClause);
7333 if (!ItOMPDestroyClause.empty()) {
7336 llvm::Value *InteropvarPtr =
7338 OMPBuilder.createOMPInteropDestroy(
Builder, InteropvarPtr,
Device,
7339 NumDependences, DependenceList,
7340 Data.HasNowaitClause);
7343 auto ItOMPUseClause = S.getClausesOfKind<
OMPUseClause>();
7344 if (!ItOMPUseClause.empty()) {
7347 llvm::Value *InteropvarPtr =
7349 OMPBuilder.createOMPInteropUse(
Builder, InteropvarPtr,
Device,
7350 NumDependences, DependenceList,
7351 Data.HasNowaitClause);
7373 CGF, OMPD_distribute, CodeGenDistribute,
false);
7392 llvm::Constant *
Addr;
7395 S, ParentName, Fn,
Addr,
true, CodeGen);
7396 assert(Fn &&
Addr &&
"Target device function emission failed.");
7425 CGF, OMPD_distribute, CodeGenDistribute,
false);
7444 llvm::Constant *
Addr;
7447 S, ParentName, Fn,
Addr,
true, CodeGen);
7448 assert(Fn &&
Addr &&
"Target device function emission failed.");
7462 S.getCancelRegion());
7466 const Expr *IfCond =
nullptr;
7467 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
7468 if (
C->getNameModifier() == OMPD_unknown ||
7469 C->getNameModifier() == OMPD_cancel) {
7470 IfCond =
C->getCondition();
7478 if (S.getCancelRegion() == OMPD_parallel ||
7479 S.getCancelRegion() == OMPD_sections ||
7480 S.getCancelRegion() == OMPD_section) {
7481 llvm::Value *IfCondition =
nullptr;
7485 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
7486 OMPBuilder.createCancel(
Builder, IfCondition, S.getCancelRegion()));
7487 return Builder.restoreIP(AfterIP);
7492 S.getCancelRegion());
7497 if (Kind == OMPD_parallel || Kind == OMPD_task ||
7498 Kind == OMPD_target_parallel || Kind == OMPD_taskloop ||
7499 Kind == OMPD_master_taskloop || Kind == OMPD_parallel_master_taskloop)
7501 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
7502 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
7503 Kind == OMPD_distribute_parallel_for ||
7504 Kind == OMPD_target_parallel_for ||
7505 Kind == OMPD_teams_distribute_parallel_for ||
7506 Kind == OMPD_target_teams_distribute_parallel_for);
7507 return OMPCancelStack.getExitBlock();
7512 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
7513 CaptureDeviceAddrMap) {
7514 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
7515 for (
const Expr *OrigVarIt :
C.varlist()) {
7516 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(OrigVarIt)->getDecl());
7517 if (!Processed.insert(OrigVD).second)
7524 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
7527 const auto *ME = cast<MemberExpr>(OED->getInit());
7528 assert(isa<CXXThisExpr>(ME->getBase()->IgnoreImpCasts()) &&
7529 "Base should be the current struct!");
7530 MatchingVD = ME->getMemberDecl();
7535 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
7536 if (InitAddrIt == CaptureDeviceAddrMap.end())
7544 Address(InitAddrIt->second, Ty,
7546 assert(IsRegistered &&
"firstprivate var already registered as private");
7554 while (
const auto *OASE = dyn_cast<ArraySectionExpr>(
Base))
7555 Base = OASE->getBase()->IgnoreParenImpCasts();
7556 while (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(
Base))
7557 Base = ASE->getBase()->IgnoreParenImpCasts();
7558 return cast<VarDecl>(cast<DeclRefExpr>(
Base)->getDecl());
7563 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
7564 CaptureDeviceAddrMap) {
7565 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
7566 for (
const Expr *Ref :
C.varlist()) {
7568 if (!Processed.insert(OrigVD).second)
7574 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
7577 const auto *ME = cast<MemberExpr>(OED->getInit());
7578 assert(isa<CXXThisExpr>(ME->getBase()) &&
7579 "Base should be the current struct!");
7580 MatchingVD = ME->getMemberDecl();
7585 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
7586 if (InitAddrIt == CaptureDeviceAddrMap.end())
7592 Address(InitAddrIt->second, Ty,
7596 if (isa<DeclRefExpr>(Ref->IgnoreParenImpCasts()) ||
7605 (void)PrivateScope.
addPrivate(OrigVD, PrivAddr);
7618 bool PrivatizeDevicePointers =
false;
7620 bool &PrivatizeDevicePointers;
7623 explicit DevicePointerPrivActionTy(
bool &PrivatizeDevicePointers)
7624 : PrivatizeDevicePointers(PrivatizeDevicePointers) {}
7626 PrivatizeDevicePointers =
true;
7629 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
7633 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
7639 PrivatizeDevicePointers =
false;
7645 if (PrivatizeDevicePointers) {
7659 std::optional<OpenMPDirectiveKind> CaptureRegion;
7663 for (
const Expr *
E :
C->varlist()) {
7664 const Decl *
D = cast<DeclRefExpr>(
E)->getDecl();
7665 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(
D))
7669 for (
const Expr *
E :
C->varlist()) {
7671 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(
D))
7675 CaptureRegion = OMPD_unknown;
7678 OMPLexicalScope
Scope(CGF, S, CaptureRegion);
7690 OMPLexicalScope
Scope(CGF, S);
7705 const Expr *IfCond =
nullptr;
7707 IfCond =
C->getCondition();
7730 const Expr *IfCond =
nullptr;
7732 IfCond =
C->getCondition();
7739 OMPLexicalScope
Scope(*
this, S, OMPD_task);
7751 const Expr *IfCond =
nullptr;
7753 IfCond =
C->getCondition();
7760 OMPLexicalScope
Scope(*
this, S, OMPD_task);
7768 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
7796 llvm::Constant *
Addr;
7799 S, ParentName, Fn,
Addr,
true, CodeGen);
7800 assert(Fn &&
Addr &&
"Target device function emission failed.");
7820 CGF, OMPD_target_parallel_for, S.hasCancel());
7836 llvm::Constant *
Addr;
7839 S, ParentName, Fn,
Addr,
true, CodeGen);
7840 assert(Fn &&
Addr &&
"Target device function emission failed.");
7875 llvm::Constant *
Addr;
7878 S, ParentName, Fn,
Addr,
true, CodeGen);
7879 assert(Fn &&
Addr &&
"Target device function emission failed.");
7894 const auto *VDecl = cast<VarDecl>(Helper->
getDecl());
7901 const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
7904 OMPLexicalScope
Scope(*
this, S, OMPD_taskloop,
false);
7909 const Expr *IfCond =
nullptr;
7910 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
7911 if (
C->getNameModifier() == OMPD_unknown ||
7912 C->getNameModifier() == OMPD_taskloop) {
7913 IfCond =
C->getCondition();
7926 Data.Schedule.setInt(
false);
7929 (Clause->getModifier() == OMPC_GRAINSIZE_strict) ?
true :
false;
7932 Data.Schedule.setInt(
true);
7935 (Clause->getModifier() == OMPC_NUMTASKS_strict) ?
true :
false;
7949 llvm::BasicBlock *ContBlock =
nullptr;
7950 OMPLoopScope PreInitScope(CGF, S);
7951 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
7955 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock(
"taskloop.if.then");
7956 ContBlock = CGF.createBasicBlock(
"taskloop.if.end");
7957 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
7958 CGF.getProfileCount(&S));
7959 CGF.EmitBlock(ThenBlock);
7960 CGF.incrementProfileCounter(&S);
7963 (void)CGF.EmitOMPLinearClauseInit(S);
7967 enum { LowerBound = 5, UpperBound, Stride, LastIter };
7969 auto *LBP = std::next(I, LowerBound);
7970 auto *UBP = std::next(I, UpperBound);
7971 auto *STP = std::next(I, Stride);
7972 auto *LIP = std::next(I, LastIter);
7973 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
7975 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
7977 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
7978 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
7980 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
7981 CGF.EmitOMPLinearClause(S, LoopScope);
7982 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
7985 const Expr *IVExpr = S.getIterationVariable();
7986 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
7987 CGF.EmitVarDecl(*IVDecl);
7988 CGF.EmitIgnoredExpr(S.getInit());
7993 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
7994 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
7996 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
8000 OMPLexicalScope
Scope(CGF, S, OMPD_taskloop,
false);
8011 emitOMPLoopBodyWithStopPoint(CGF, S,
8012 CodeGenFunction::JumpDest());
8019 CGF.EmitBranch(ContBlock);
8020 CGF.EmitBlock(ContBlock,
true);
8023 if (HasLastprivateClause) {
8024 CGF.EmitOMPLastprivateClauseFinal(
8026 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
8027 CGF.GetAddrOfLocalVar(*LIP),
false,
8028 (*LIP)->getType(), S.getBeginLoc())));
8032 return CGF.
Builder.CreateIsNotNull(
8034 (*LIP)->
getType(), S.getBeginLoc()));
8037 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
8040 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
8042 OMPLoopScope PreInitScope(CGF, S);
8043 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
8044 OutlinedFn, SharedsTy,
8045 CapturedStruct, IfCond,
Data);
8047 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
8075 OMPLexicalScope
Scope(*
this, S);
8087 OMPLexicalScope
Scope(*
this, S, std::nullopt,
false);
8099 OMPLexicalScope
Scope(*
this, S, std::nullopt,
false);
8111 OMPLexicalScope
Scope(*
this, S);
8123 OMPLexicalScope
Scope(*
this, S);
8135 OMPLexicalScope
Scope(CGF, S, OMPD_parallel,
false);
8153 OMPLexicalScope
Scope(CGF, S, OMPD_parallel,
false);
8171 OMPLexicalScope
Scope(CGF, S, OMPD_parallel,
false);
8189 OMPLexicalScope
Scope(CGF, S, OMPD_parallel,
false);
8208 const Expr *IfCond =
nullptr;
8210 IfCond =
C->getCondition();
8217 OMPLexicalScope
Scope(*
this, S, OMPD_task);
8228 BindKind =
C->getBindKind();
8231 case OMPC_BIND_parallel:
8233 case OMPC_BIND_teams:
8235 case OMPC_BIND_thread:
8245 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt();
8246 const auto *ForS = dyn_cast<ForStmt>(CS);
8247 if (ForS && !isa<DeclStmt>(ForS->getInit())) {
8257 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
8306 std::string StatusMsg,
8310 StatusMsg +=
": DEVICE";
8312 StatusMsg +=
": HOST";
8319 llvm::dbgs() << StatusMsg <<
": " <<
FileName <<
": " << LineNo <<
"\n";
8342 CGF, OMPD_distribute, CodeGenDistribute,
false);
8371 CGF, OMPD_distribute, CodeGenDistribute,
false);
8385 if (S.canBeParallelFor())
8398 if (S.canBeParallelFor())
8404 llvm::Constant *
Addr;
8407 S, ParentName, Fn,
Addr,
true, CodeGen);
8408 assert(Fn &&
Addr &&
8409 "Target device function emission failed for 'target teams loop'.");
8420 CGF, OMPD_target_parallel_loop,
false);
8436 llvm::Constant *
Addr;
8439 S, ParentName, Fn,
Addr,
true, CodeGen);
8440 assert(Fn &&
Addr &&
"Target device function emission failed.");
8455 if (
const auto *SD = dyn_cast<OMPScanDirective>(&
D)) {
8459 if (!
D.hasAssociatedStmt() || !
D.getAssociatedStmt())
8466 for (
const Expr *Ref :
C->varlist()) {
8467 const auto *DRE = cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
8470 const auto *VD = dyn_cast<VarDecl>(DRE->getDecl());
8473 if (!CGF.LocalDeclMap.count(VD)) {
8485 if (
const auto *LD = dyn_cast<OMPLoopDirective>(&
D)) {
8486 for (
const Expr *
E : LD->counters()) {
8487 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
8492 if (isa<OMPCapturedExprDecl>(VD)) {
8494 if (!CGF.LocalDeclMap.count(VD))
8499 if (!
C->getNumForLoops())
8501 for (
unsigned I = LD->getLoopsNumber(),
8502 E =
C->getLoopNumIterations().size();
8504 if (
const auto *VD = dyn_cast<OMPCapturedExprDecl>(
8505 cast<DeclRefExpr>(
C->getLoopCounter(I))->getDecl())) {
8507 if (!CGF.LocalDeclMap.count(VD))
8514 CGF.
EmitStmt(
D.getInnermostCapturedStmt()->getCapturedStmt());
8517 if (
D.getDirectiveKind() == OMPD_atomic ||
8518 D.getDirectiveKind() == OMPD_critical ||
8519 D.getDirectiveKind() == OMPD_section ||
8520 D.getDirectiveKind() == OMPD_master ||
8521 D.getDirectiveKind() == OMPD_masked ||
8522 D.getDirectiveKind() == OMPD_unroll ||
8523 D.getDirectiveKind() == OMPD_assume) {
8528 OMPSimdLexicalScope
Scope(*
this,
D);
8532 :
D.getDirectiveKind(),
Defines the clang::ASTContext interface.
static bool isAllocatableDecl(const VarDecl *VD)
static const VarDecl * getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE)
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, PrePostActionTy &Action)
static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action)
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S, SourceLocation Loc)
static const VarDecl * getBaseDecl(const Expr *Ref)
static void emitTargetTeamsGenericLoopRegionAsParallel(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsGenericLoopDirective &S)
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *V, SourceLocation Loc)
static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
static void emitScanBasedDirective(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen, llvm::function_ref< void(CodeGenFunction &)> FirstGen, llvm::function_ref< void(CodeGenFunction &)> SecondGen)
Emits the code for the directive with inscan reductions.
static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO, LValue LVal, RValue RVal)
static bool isSupportedByOpenMPIRBuilder(const OMPTaskgroupDirective &T)
static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, QualType DstType, StringRef Name, LValue AddrLV)
static void emitDistributeParallelForDistributeInnerBoundParams(CodeGenFunction &CGF, const OMPExecutableDirective &S, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars)
static void emitScanBasedDirectiveFinals(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen)
Copies final inscan reductions values to the original variables.
static void checkForLastprivateConditionalUpdate(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static std::pair< LValue, LValue > emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
The following two functions generate expressions for the loop lower and upper bounds in case of stati...
static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action)
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitTargetTeamsGenericLoopRegionAsDistribute(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsGenericLoopDirective &S)
static void emitTargetParallelRegion(CodeGenFunction &CGF, const OMPTargetParallelDirective &S, PrePostActionTy &Action)
static std::pair< llvm::Value *, llvm::Value * > emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
When dealing with dispatch schedules (e.g.
static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitRestoreIP(CodeGenFunction &CGF, const T *C, llvm::OpenMPIRBuilder::InsertPointTy AllocaIP, llvm::OpenMPIRBuilder &OMPBuilder)
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen)
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
static bool isSimdSupportedByOpenMPIRBuilder(const OMPLoopDirective &S)
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
static void applyConservativeSimdOrderedDirective(const Stmt &AssociatedStmt, LoopInfoStack &LoopStack)
static bool emitWorksharingDirective(CodeGenFunction &CGF, const OMPLoopDirective &S, bool HasCancel)
static void emitPostUpdateForReductionClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, const unsigned IVSize, const bool IVSigned)
static void emitTargetTeamsLoopCodegenStatus(CodeGenFunction &CGF, std::string StatusMsg, const OMPExecutableDirective &D)
static bool isForSupportedByOpenMPIRBuilder(const OMPLoopDirective &S, bool HasCancel)
static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF, llvm::AtomicOrdering AO, LValue LVal, SourceLocation Loc)
static std::pair< llvm::Value *, llvm::Value * > emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
if the 'for' loop has a dispatch schedule (e.g.
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *UE, const Expr *D, const Expr *CE, bool IsXLHSInRHSPart, bool IsFailOnly, SourceLocation Loc)
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD, SourceLocation Loc)
static EmittedClosureTy emitCapturedStmtFunc(CodeGenFunction &ParentCGF, const CapturedStmt *S)
Emit a captured statement and return the function as well as its captured closure context.
static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static void emitOMPDistributeDirective(const OMPLoopDirective &S, CodeGenFunction &CGF, CodeGenModule &CGM)
static void emitOMPCopyinClause(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitTargetTeamsDistributeParallelForRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, PrePostActionTy &Action)
static bool hasOrderedDirective(const Stmt *S)
static llvm::CallInst * emitCapturedStmtCall(CodeGenFunction &ParentCGF, EmittedClosureTy Cap, llvm::ArrayRef< llvm::Value * > Args)
Emit a call to a previously captured closure.
static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop, int MaxLevel, int Level=0)
static void emitOMPForDirective(const OMPLoopDirective &S, CodeGenFunction &CGF, CodeGenModule &CGM, bool HasCancel)
static void emitEmptyBoundParameters(CodeGenFunction &, const OMPExecutableDirective &, llvm::SmallVectorImpl< llvm::Value * > &)
static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action)
static void emitOMPSimdDirective(const OMPLoopDirective &S, CodeGenFunction &CGF, CodeGenModule &CGM)
static void emitOMPAtomicCompareExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *D, const Expr *CE, bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly, SourceLocation Loc)
std::pair< llvm::Function *, llvm::Value * > EmittedClosureTy
static OpenMPDirectiveKind getEffectiveDirectiveKind(const OMPExecutableDirective &S)
static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDirective &S)
static void buildDependences(const OMPExecutableDirective &S, OMPTaskDataTy &Data)
static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc)
static void emitScanBasedDirectiveDecls(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen)
Emits internal temp array declarations for the directive with inscan reductions.
static void emitTargetTeamsDistributeParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForSimdDirective &S, PrePostActionTy &Action)
static void emitTargetTeamsDistributeSimdRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S)
static llvm::MapVector< llvm::Value *, llvm::Value * > GetAlignedMapping(const OMPLoopDirective &S, CodeGenFunction &CGF)
static llvm::omp::ScheduleKind convertClauseKindToSchedKind(OpenMPScheduleClauseKind ScheduleClauseKind)
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
static void emitTargetParallelGenericLoopRegion(CodeGenFunction &CGF, const OMPTargetParallelGenericLoopDirective &S, PrePostActionTy &Action)
static QualType getCanonicalParamType(ASTContext &C, QualType T)
static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S, const RegionCodeGenTy &SimdInitGen, const RegionCodeGenTy &BodyCodeGen)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *E, SourceLocation Loc)
static llvm::Function * emitOutlinedFunctionPrologue(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address > > &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value * > > &VLASizes, llvm::Value *&CXXThisValue, const FunctionOptions &FO)
static void emitInnerParallelForWhenCombined(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
enum clang::sema::@1840::IndirectLocalPathEntry::EntryKind Kind
This file defines OpenMP nodes for declarative directives.
static const Decl * getCanonicalDecl(const Decl *D)
This file defines OpenMP AST classes for clauses.
Defines some OpenMP-specific enums and functions.
Defines the PrettyStackTraceEntry class, which is used to make crashes give more contextual informati...
Defines the SourceManager interface.
This file defines OpenMP AST classes for executable directives and clauses.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
SourceManager & getSourceManager()
TranslationUnitDecl * getTranslationUnitDecl() const
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location,...
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
const TargetInfo & getTargetInfo() const
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
CanQualType getCanonicalTagType(const TagDecl *TD) const
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Represents an attribute applied to a statement.
ArrayRef< const Attr * > getAttrs() const
A builtin binary operation expression such as "x + y" or "x <= y".
static BinaryOperator * Create(const ASTContext &C, Expr *lhs, Expr *rhs, Opcode opc, QualType ResTy, ExprValueKind VK, ExprObjectKind OK, SourceLocation opLoc, FPOptionsOverride FPFeatures)
BlockExpr - Adaptor class for mixing a BlockDecl with expressions.
Represents the body of a CapturedStmt, and serves as its DeclContext.
unsigned getContextParamPosition() const
static CapturedDecl * Create(ASTContext &C, DeclContext *DC, unsigned NumParams)
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
ImplicitParamDecl * getParam(unsigned i) const
This captures a statement into a function.
CapturedDecl * getCapturedDecl()
Retrieve the outlined function declaration.
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
CharUnits - This is an opaque type for sizes expressed in character units.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
CharUnits alignTo(const CharUnits &Align) const
alignTo - Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is ...
bool hasReducedDebugInfo() const
Check if type and variable info should be emitted.
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
llvm::PointerType * getType() const
Return the type of the pointer value.
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored.
static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, SourceLocation TemporaryLocation)
Apply TemporaryLocation if it is valid.
llvm::Value * CreateIsNull(Address Addr, const Twine &Name="")
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Address CreateInBoundsGEP(Address Addr, ArrayRef< llvm::Value * > IdxList, llvm::Type *ElementType, CharUnits Align, const Twine &Name="")
CGFunctionInfo - Class to encapsulate the information about a function definition.
Manages list of lastprivate conditional decls for the specified directive.
static LastprivateConditionalRAII disable(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Manages list of nontemporal decls for the specified directive.
Struct that keeps all the relevant information that should be kept throughout a 'target data' region.
llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap
Map between the a declaration of a capture and the corresponding new llvm address where the runtime r...
Manages list of nontemporal decls for the specified directive.
virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the task directive.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, const Expr *Message=nullptr)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device)
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
virtual void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
Emits a single region.
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc)
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF, const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const
Choose default schedule type and chunk value for the schedule clause.
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
virtual void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancellation point' construct.
virtual const VarDecl * translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const
Translates the native parameter of outlined function if this is required for target.
virtual llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations)
Emit initialization for doacross loop nesting support.
virtual void adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, const OMPExecutableDirective &D) const
Adjust some parameters for the target-based directives, like addresses of the variables captured by r...
virtual void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info)
Emit the target data mapping code associated with D.
virtual Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const
Gets the address of the native argument basing on the address of the target-specific parameter.
virtual void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc)
Emit a taskgroup region.
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams,...
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, OpenMPDependClauseKind NewDepKind, SourceLocation Loc)
Updates the dependency kind in the specified depobj object.
virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, SourceLocation Loc)
Gets the address of the global copy used for lastprivate conditional update, if any.
virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME, bool IsFatal)
Emit __kmpc_error call for error directive extern void __kmpc_error(ident_t *loc, int severity,...
virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc)
Emits code for a taskyield directive.
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO)
Emit flush of the variables specified in 'omp flush' directive.
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data)
Emit code for 'taskwait' directive.
virtual void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values)
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind)
Call the appropriate runtime routine to notify that we finished all the work with current loop.
void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
Address emitDepobjDependClause(CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs) for depob...
virtual llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc, OpenMPNumThreadsClauseModifier Modifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, const Expr *Message=nullptr)
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values)
Call the appropriate runtime routine to initialize it before start of loop.
llvm::AtomicOrdering getDefaultMemoryOrdering() const
Gets default memory ordering as specified in requires directive.
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancel' construct.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
virtual llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts)
Emits outlined function for the OpenMP task directive D.
void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Emits the code to destroy the dependency object provided in depobj directive.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
virtual void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C)
Emit code for doacross ordered directive with 'depend' clause.
virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const Expr *LHS)
Checks if the provided LVal is lastprivate conditional and emits the code to update the value of the ...
virtual void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF, const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind, llvm::Value *&Chunk) const
Choose default schedule type and chunk value for the dist_schedule clause.
llvm::OpenMPIRBuilder & getOMPBuilder()
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr)
Emits a critical region.
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
virtual void emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, ArrayRef< llvm::Value * > Args={}) const
Emits call of the outlined function with the provided arguments, translating these arguments to corre...
virtual void checkAndEmitSharedLastprivateConditional(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::DenseSet< CanonicalDeclPtr< const VarDecl > > &IgnoredDecls)
Checks if the lastprivate conditional was updated in inner region and writes the value.
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static chunked.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Emit the target offloading code associated with D.
virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options)
Emit a code for reduction clause.
std::pair< llvm::Value *, Address > emitDependClause(CodeGenFunction &CGF, ArrayRef< OMPTaskDataTy::DependData > Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs).
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
Address emitLastprivateConditionalInit(CodeGenFunction &CGF, const VarDecl *VD)
Create specialized alloca to handle lastprivate conditionals.
virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads)
Emit an ordered region.
virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction)
Emits the following code for reduction clause with task modifier:
virtual void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr)
Emits a masked region.
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
Address getAllocatedAddress() const
Returns the raw, allocated address, which is not necessarily the address of the object itself.
API for captured statement code generation.
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
virtual StringRef getHelperName() const
Get the name of the capture helper.
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
RAII for correct setting/restoring of CapturedStmtInfo.
LValue getReferenceLValue(CodeGenFunction &CGF, const Expr *RefExpr) const
void ForceCleanup()
Force the emission of cleanups now, instead of waiting until this object is destroyed.
RAII for preserving necessary info during inlined region body codegen.
Cleanup action for allocate support.
RAII for preserving necessary info during Outlined region body codegen.
Controls insertion of cancellation exit blocks in worksharing constructs.
Save/restore original map of previously emitted local vars in case when we need to duplicate emission...
The class used to assign some variables some temporarily addresses.
bool apply(CodeGenFunction &CGF)
Applies new addresses to the list of the variables.
void restore(CodeGenFunction &CGF)
Restores original addresses of the variables.
bool setVarAddr(CodeGenFunction &CGF, const VarDecl *LocalVD, Address TempAddr)
Sets the address of the variable LocalVD to be TempAddr in function CGF.
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
void restoreMap()
Restore all mapped variables w/o clean up.
bool Privatize()
Privatizes local variables previously registered as private.
bool isGlobalVarCaptured(const VarDecl *VD) const
Checks if the global variable is captured in current function.
bool addPrivate(const VarDecl *LocalVD, Address Addr)
Registers LocalVD variable as a private with Addr as the address of the corresponding private variabl...
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Manages parent directive for scan directives.
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited.
void ForceCleanup(std::initializer_list< llvm::Value ** > ValuesToReload={})
Force the emission of cleanups now, instead of waiting until this object is destroyed.
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S, SourceLocation Loc)
void EmitOMPParallelMaskedTaskLoopDirective(const OMPParallelMaskedTaskLoopDirective &S)
void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S)
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S)
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, Stmt::Likelihood LH=Stmt::LH_None, const Expr *ConditionalOp=nullptr, const VarDecl *ConditionalDecl=nullptr)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
void processInReduction(const OMPExecutableDirective &S, OMPTaskDataTy &Data, CodeGenFunction &CGF, const CapturedStmt *CS, OMPPrivateScope &Scope)
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S)
Emit an OMPCanonicalLoop using the OpenMPIRBuilder.
void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S)
void EmitOMPScanDirective(const OMPScanDirective &S)
static bool hasScalarEvaluationKind(QualType T)
llvm::function_ref< std::pair< llvm::Value *, llvm::Value * >(CodeGenFunction &, const OMPExecutableDirective &S, Address LB, Address UB)> CodeGenDispatchBoundsTy
LValue InitCapturedStruct(const CapturedStmt &S)
CGCapturedStmtInfo * CapturedStmtInfo
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
void EmitOMPParallelForDirective(const OMPParallelForDirective &S)
void EmitOMPMasterDirective(const OMPMasterDirective &S)
void EmitOMPParallelMasterTaskLoopSimdDirective(const OMPParallelMasterTaskLoopSimdDirective &S)
void EmitOMPSimdInit(const OMPLoopDirective &D)
Helpers for the OpenMP loop directives.
const OMPExecutableDirective * OMPParentLoopDirectiveForScan
Parent loop-based directive for scan directive.
void EmitOMPFlushDirective(const OMPFlushDirective &S)
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S)
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope, bool ForInscan=false)
Emit initial code for reduction variables.
void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S)
void EmitAutoVarDecl(const VarDecl &D)
EmitAutoVarDecl - Emit an auto variable declaration.
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S)
const LangOptions & getLangOpts() const
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Load a pointer with type PtrTy stored at address Ptr.
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind)
Emit final update of reduction values to original variables at the end of the directive.
void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit)
Helper for the OpenMP loop directives.
void EmitOMPScopeDirective(const OMPScopeDirective &S)
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy)
Emit an aggregate assignment.
CodeGenFunction * ParentCGF
JumpDest ReturnBlock
ReturnBlock - Unified return block.
void EmitOMPTargetTeamsDistributeSimdDirective(const OMPTargetTeamsDistributeSimdDirective &S)
const llvm::function_ref< void(CodeGenFunction &, llvm::Function *, const OMPTaskDataTy &)> TaskGenTy
llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location)
Converts Location to a DebugLoc, if debug information is enabled.
bool EmitOMPCopyinClause(const OMPExecutableDirective &D)
Emit code for copyin clause in D directive.
void EmitOMPLinearClause(const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope)
Emit initial code for linear clauses.
llvm::BasicBlock * OMPBeforeScanBlock
void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S)
void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S, OMPPrivateScope &LoopScope)
Emit initial code for loop counters of loop-based directives.
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value * > &CapturedVars)
void EmitOMPDepobjDirective(const OMPDepobjDirective &S)
void EmitOMPMetaDirective(const OMPMetaDirective &S)
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S)
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
void EmitOMPCancelDirective(const OMPCancelDirective &S)
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type,...
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
llvm::CanonicalLoopInfo * EmitOMPCollapsedCanonicalLoopNest(const Stmt *S, int Depth)
Emit the Stmt S and return its topmost canonical loop, if any.
void EmitOMPSectionsDirective(const OMPSectionsDirective &S)
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
void EmitOMPInteropDirective(const OMPInteropDirective &S)
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S)
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
void EmitOMPTargetParallelForSimdDirective(const OMPTargetParallelForSimdDirective &S)
void EmitOMPTargetParallelGenericLoopDirective(const OMPTargetParallelGenericLoopDirective &S)
Emit combined directive 'target parallel loop' as if its constituent constructs are 'target',...
void EmitOMPUseDeviceAddrClause(const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap)
void EmitOMPTeamsDistributeParallelForSimdDirective(const OMPTeamsDistributeParallelForSimdDirective &S)
void EmitOMPMaskedDirective(const OMPMaskedDirective &S)
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S)
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
CGDebugInfo * getDebugInfo()
void EmitOMPDistributeLoop(const OMPLoopDirective &S, const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr)
Emit code for the distribute loop-based directive.
void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S)
void EmitOMPReverseDirective(const OMPReverseDirective &S)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
void EmitOMPTargetTeamsDistributeParallelForDirective(const OMPTargetTeamsDistributeParallelForDirective &S)
void EmitOMPMaskedTaskLoopDirective(const OMPMaskedTaskLoopDirective &S)
llvm::BasicBlock * OMPScanDispatch
llvm::function_ref< std::pair< LValue, LValue >(CodeGenFunction &, const OMPExecutableDirective &S)> CodeGenLoopBoundsTy
void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S)
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S)
void EmitOMPMaskedTaskLoopSimdDirective(const OMPMaskedTaskLoopSimdDirective &S)
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
VlaSizePair getVLASize(const VariableArrayType *vla)
Returns an LLVM value that corresponds to the size, in non-variably-sized elements,...
void EmitOMPParallelDirective(const OMPParallelDirective &S)
void EmitOMPTaskDirective(const OMPTaskDirective &S)
void EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S)
void EmitOMPDistributeParallelForDirective(const OMPDistributeParallelForDirective &S)
void EmitOMPAssumeDirective(const OMPAssumeDirective &S)
int ExpectedOMPLoopDepth
Number of nested loop to be consumed by the last surrounding loop-associated directive.
void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
ASTContext & getContext() const
void EmitStopPoint(const Stmt *S)
EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S)
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
void EmitOMPTargetTeamsGenericLoopDirective(const OMPTargetTeamsGenericLoopDirective &S)
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
void EmitAutoVarCleanups(const AutoVarEmission &emission)
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
SmallVector< llvm::CanonicalLoopInfo *, 4 > OMPLoopNestStack
List of recently emitted OMPCanonicalLoops.
void EmitOMPTeamsDistributeParallelForDirective(const OMPTeamsDistributeParallelForDirective &S)
llvm::AtomicRMWInst * emitAtomicRMWInst(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Order=llvm::AtomicOrdering::SequentiallyConsistent, llvm::SyncScope::ID SSID=llvm::SyncScope::System, const AtomicExpr *AE=nullptr)
Emit an atomicrmw instruction, and applying relevant metadata when applicable.
void EmitOMPTargetTeamsDistributeDirective(const OMPTargetTeamsDistributeDirective &S)
void EmitOMPUseDevicePtrClause(const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap)
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
void EmitStmt(const Stmt *S, ArrayRef< const Attr * > Attrs={})
EmitStmt - Emit the code for the statement.
llvm::DenseMap< const ValueDecl *, FieldDecl * > LambdaCaptureFields
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
llvm::Type * ConvertTypeForMem(QualType T)
void EmitOMPInnerLoop(const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> BodyGen, const llvm::function_ref< void(CodeGenFunction &)> PostIncGen)
Emit inner loop of the worksharing/simd construct.
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
void EmitOMPTargetDirective(const OMPTargetDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
llvm::BasicBlock * OMPScanExitBlock
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D)
Emit simple code for OpenMP directives in Simd-only mode.
void EmitOMPErrorDirective(const OMPErrorDirective &S)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
void EmitOMPParallelMaskedTaskLoopSimdDirective(const OMPParallelMaskedTaskLoopSimdDirective &S)
void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S)
void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S)
Address GenerateCapturedStmtArgument(const CapturedStmt &S)
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block,...
void EmitOMPSimdDirective(const OMPSimdDirective &S)
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S)
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S)
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
bool EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
static void EmitOMPTargetParallelGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelGenericLoopDirective &S)
Emit device code for the target parallel loop directive.
void EmitOMPUnrollDirective(const OMPUnrollDirective &S)
void EmitOMPStripeDirective(const OMPStripeDirective &S)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void EmitOMPSingleDirective(const OMPSingleDirective &S)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
llvm::function_ref< void(CodeGenFunction &, SourceLocation, const unsigned, const bool)> CodeGenOrderedTy
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
llvm::Function * GenerateCapturedStmtFunction(const CapturedStmt &S)
Creates the outlined function for a CapturedStmt.
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
uint64_t getProfileCount(const Stmt *S)
Get the profiler's count for the given statement.
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant,...
static void EmitOMPTargetTeamsGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsGenericLoopDirective &S)
Emit device code for the target teams loop directive.
void EmitOMPTileDirective(const OMPTileDirective &S)
void EmitDecl(const Decl &D, bool EvaluateConditionDecl=false)
EmitDecl - Emit a declaration.
void EmitOMPAtomicDirective(const OMPAtomicDirective &S)
llvm::BasicBlock * OMPAfterScanBlock
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
ConstantEmission tryEmitAsConstant(const DeclRefExpr *RefExpr)
Try to emit a reference to the given value without producing it as an l-value.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
void EmitOMPParallelMasterTaskLoopDirective(const OMPParallelMasterTaskLoopDirective &S)
void EmitOMPDistributeParallelForSimdDirective(const OMPDistributeParallelForSimdDirective &S)
void EmitOMPSectionDirective(const OMPSectionDirective &S)
void EnsureInsertPoint()
EnsureInsertPoint - Ensure that an insertion point is defined so that emitted IR has a place to go.
void EmitOMPForSimdDirective(const OMPForSimdDirective &S)
llvm::LLVMContext & getLLVMContext()
void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
llvm::function_ref< void(CodeGenFunction &, const OMPLoopDirective &, JumpDest)> CodeGenLoopTy
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S)
void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitOMPForDirective(const OMPForDirective &S)
void EmitOMPLinearClauseFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Emit final code for linear clauses.
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
void EmitOMPSimdFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
This class organizes the cross-function state that is used while generating LLVM code.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
llvm::Module & getModule() const
CharUnits GetTargetTypeStoreSize(llvm::Type *Ty) const
Return the store size, in character units, of the given LLVM type.
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
CharUnits getNaturalTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, bool forPointeeType=false)
CodeGenTypes & getTypes()
const llvm::DataLayout & getDataLayout() const
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
llvm::Constant * GetAddrOfGlobal(GlobalDecl GD, ForDefinition_t IsForDefinition=NotForDefinition)
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
llvm::Constant * getStaticLocalDeclAddress(const VarDecl *D)
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
llvm::Type * ConvertTypeForMem(QualType T)
ConvertTypeForMem - Convert type T into a llvm::Type.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
llvm::Value * getPointer(CodeGenFunction &CGF) const
Address getAddress() const
void setAddress(Address address)
A stack of loop information corresponding to loop nesting levels.
void setUnrollCount(unsigned C)
Set the unroll count for the next loop pushed.
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
void setUnrollState(const LoopAttributes::LVEnableState &State)
Set the next pushed loop unroll state.
void pop()
End the current loop.
void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
Begin a new structured loop.
void setVectorizeEnable(bool Enable=true)
Set the next pushed loop 'vectorize.enable'.
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual void Enter(CodeGenFunction &CGF)
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue get(llvm::Value *V)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
An abstract representation of an aligned address.
llvm::PointerType * getType() const
Return the type of the pointer value.
llvm::Value * getPointer() const
Class intended to support codegen of all kind of the reduction clauses.
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N)
Emits lvalue for the shared and original reduction item.
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
void setAction(PrePostActionTy &Action) const
Complex values, per C99 6.2.5p11.
CompoundStmt - This represents a group of statements like { stmt stmt }.
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
DeclContext * getParent()
getParent - Returns the containing DeclContext.
A reference to a declared variable, function, enum, etc.
static DeclRefExpr * Create(const ASTContext &Context, NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc, ValueDecl *D, bool RefersToEnclosingVariableOrCapture, SourceLocation NameLoc, QualType T, ExprValueKind VK, NamedDecl *FoundD=nullptr, const TemplateArgumentListInfo *TemplateArgs=nullptr, NonOdrUseReason NOUR=NOUR_None)
DeclStmt - Adaptor class for mixing declarations with statements and expressions.
Decl - This represents one declaration (or definition), e.g.
SourceLocation getEndLoc() const LLVM_READONLY
virtual Stmt * getBody() const
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
SourceLocation getBodyRBrace() const
getBodyRBrace - Gets the right brace of the body, if a body exists.
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
SourceLocation getLocation() const
SourceLocation getBeginLoc() const LLVM_READONLY
virtual Decl * getCanonicalDecl()
Retrieves the "canonical" declaration of the given declaration.
The name of a declaration.
SourceLocation getBeginLoc() const LLVM_READONLY
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
This represents one expression.
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
Expr * IgnoreImplicitAsWritten() LLVM_READONLY
Skip past any implicit AST nodes which might surround this expression until reaching a fixed point.
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Represents difference between two FPOptions values.
Represents a member of a struct/union/class.
Represents a function declaration or definition.
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool UsesFPIntrin=false, bool isInlineSpecified=false, bool hasWrittenPrototype=true, ConstexprSpecKind ConstexprKind=ConstexprSpecKind::Unspecified, const AssociatedConstraint &TrailingRequiresClause={})
GlobalDecl - represents a global declaration.
One of these records is kept for each identifier that is lexed.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
static ImplicitCastExpr * Create(const ASTContext &Context, QualType T, CastKind Kind, Expr *Operand, const CXXCastPath *BasePath, ExprValueKind Cat, FPOptionsOverride FPO)
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
A C++ lambda expression, which produces a function object (of unspecified type) that can be invoked l...
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
Represents a point when we exit a loop.
IdentifierInfo * getIdentifier() const
Get the identifier that names this declaration, if there is one.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
A C++ nested-name-specifier augmented with source location information.
This represents 'acq_rel' clause in the '#pragma omp atomic|flush' directives.
This represents 'acquire' clause in the '#pragma omp atomic|flush' directives.
This represents clause 'aligned' in the '#pragma omp ...' directives.
This represents '#pragma omp atomic' directive.
This represents '#pragma omp barrier' directive.
This represents 'bind' clause in the '#pragma omp ...' directives.
This represents '#pragma omp cancel' directive.
This represents '#pragma omp cancellation point' directive.
Representation of an OpenMP canonical loop.
static OMPClauseWithPreInit * get(OMPClause *C)
This is a basic class for representing single OpenMP clause.
This represents clause 'copyin' in the '#pragma omp ...' directives.
This represents clause 'copyprivate' in the '#pragma omp ...' directives.
This represents '#pragma omp critical' directive.
This represents implicit clause 'depend' for the '#pragma omp task' directive.
This represents implicit clause 'depobj' for the '#pragma omp depobj' directive.
This represents '#pragma omp depobj' directive.
This represents 'destroy' clause in the '#pragma omp depobj' directive or the '#pragma omp interop' d...
This represents 'device' clause in the '#pragma omp ...' directive.
This represents 'dist_schedule' clause in the '#pragma omp ...' directive.
This represents '#pragma omp distribute' directive.
This represents '#pragma omp distribute parallel for' composite directive.
This represents '#pragma omp distribute parallel for simd' composite directive.
This represents '#pragma omp distribute simd' composite directive.
This represents the 'doacross' clause for the '#pragma omp ordered' directive.
This represents '#pragma omp error' directive.
This is a basic class for representing single OpenMP executable directive.
OpenMPDirectiveKind getDirectiveKind() const
SourceLocation getEndLoc() const
Returns ending location of directive.
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause * > Clauses)
This represents 'fail' clause in the '#pragma omp atomic' directive.
OpenMPClauseKind getFailParameter() const
Gets the parameter (type memory-order-clause) in Fail clause.
This represents 'filter' clause in the '#pragma omp ...' directive.
This represents 'final' clause in the '#pragma omp ...' directive.
This represents clause 'firstprivate' in the '#pragma omp ...' directives.
This represents implicit clause 'flush' for the '#pragma omp flush' directive.
This represents '#pragma omp flush' directive.
This represents '#pragma omp for' directive.
This represents '#pragma omp for simd' directive.
Representation of the 'full' clause of the '#pragma omp unroll' directive.
This represents '#pragma omp loop' directive.
This represents 'grainsize' clause in the '#pragma omp ...' directive.
This represents 'hint' clause in the '#pragma omp ...' directive.
This represents 'if' clause in the '#pragma omp ...' directive.
This represents clause 'in_reduction' in the '#pragma omp task' directives.
This represents clause 'inclusive' in the '#pragma omp scan' directive.
This represents the 'init' clause in '#pragma omp ...' directives.
Represents the '#pragma omp interchange' loop transformation directive.
This represents '#pragma omp interop' directive.
This represents clause 'lastprivate' in the '#pragma omp ...' directives.
This represents clause 'linear' in the '#pragma omp ...' directives.
The base class for all loop-based directives, including loop transformation directives.
static Stmt * tryToFindNextInnerLoop(Stmt *CurStmt, bool TryImperfectlyNestedLoops)
Try to find the next loop sub-statement in the specified statement CurStmt.
static bool doForAllLoops(Stmt *CurStmt, bool TryImperfectlyNestedLoops, unsigned NumLoops, llvm::function_ref< bool(unsigned, Stmt *)> Callback, llvm::function_ref< void(OMPLoopTransformationDirective *)> OnTransformationCallback)
Calls the specified callback function for all the loops in CurStmt, from the outermost to the innermo...
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc....
Expr * getPrevUpperBoundVariable() const
Expr * getPrevLowerBoundVariable() const
Expr * getIterationVariable() const
Expr * getUpperBoundVariable() const
Expr * getLowerBoundVariable() const
This represents '#pragma omp masked' directive.
This represents '#pragma omp masked taskloop' directive.
This represents '#pragma omp masked taskloop simd' directive.
This represents '#pragma omp master' directive.
This represents '#pragma omp master taskloop' directive.
This represents '#pragma omp master taskloop simd' directive.
This represents the 'message' clause in the '#pragma omp error' and the '#pragma omp parallel' direct...
Expr * getMessageString() const
Returns message string of the clause.
This represents 'nogroup' clause in the '#pragma omp ...' directive.
This represents 'nowait' clause in the '#pragma omp ...' directive.
This represents 'num_tasks' clause in the '#pragma omp ...' directive.
This represents 'num_teams' clause in the '#pragma omp ...' directive.
This represents 'num_threads' clause in the '#pragma omp ...' directive.
This represents 'order' clause in the '#pragma omp ...' directive.
This represents 'ordered' clause in the '#pragma omp ...' directive.
This represents '#pragma omp ordered' directive.
This represents '#pragma omp parallel' directive.
This represents '#pragma omp parallel for' directive.
This represents '#pragma omp parallel for simd' directive.
This represents '#pragma omp parallel masked' directive.
This represents '#pragma omp parallel masked taskloop' directive.
This represents '#pragma omp parallel masked taskloop simd' directive.
This represents '#pragma omp parallel master' directive.
This represents '#pragma omp parallel master taskloop' directive.
This represents '#pragma omp parallel master taskloop simd' directive.
This represents '#pragma omp parallel sections' directive.
Representation of the 'partial' clause of the '#pragma omp unroll' directive.
This represents 'priority' clause in the '#pragma omp ...' directive.
This represents clause 'private' in the '#pragma omp ...' directives.
This represents 'proc_bind' clause in the '#pragma omp ...' directive.
This represents clause 'reduction' in the '#pragma omp ...' directives.
This represents 'relaxed' clause in the '#pragma omp atomic' directives.
This represents 'release' clause in the '#pragma omp atomic|flush' directives.
Represents the '#pragma omp reverse' loop transformation directive.
This represents 'simd' clause in the '#pragma omp ...' directive.
This represents 'safelen' clause in the '#pragma omp ...' directive.
This represents '#pragma omp scan' directive.
This represents 'schedule' clause in the '#pragma omp ...' directive.
This represents '#pragma omp scope' directive.
This represents '#pragma omp section' directive.
This represents '#pragma omp sections' directive.
This represents 'seq_cst' clause in the '#pragma omp atomic|flush' directives.
This represents the 'severity' clause in the '#pragma omp error' and the '#pragma omp parallel' direc...
OpenMPSeverityClauseKind getSeverityKind() const
Returns kind of the clause.
This represents '#pragma omp simd' directive.
This represents 'simdlen' clause in the '#pragma omp ...' directive.
This represents '#pragma omp single' directive.
This represents the '#pragma omp stripe' loop transformation directive.
This represents '#pragma omp target data' directive.
This represents '#pragma omp target' directive.
This represents '#pragma omp target enter data' directive.
This represents '#pragma omp target exit data' directive.
This represents '#pragma omp target parallel' directive.
This represents '#pragma omp target parallel for' directive.
This represents '#pragma omp target parallel for simd' directive.
This represents '#pragma omp target parallel loop' directive.
This represents '#pragma omp target simd' directive.
This represents '#pragma omp target teams' directive.
This represents '#pragma omp target teams distribute' combined directive.
This represents '#pragma omp target teams distribute parallel for' combined directive.
This represents '#pragma omp target teams distribute parallel for simd' combined directive.
This represents '#pragma omp target teams distribute simd' combined directive.
This represents '#pragma omp target teams loop' directive.
This represents '#pragma omp target update' directive.
This represents '#pragma omp task' directive.
This represents '#pragma omp taskloop' directive.
This represents '#pragma omp taskloop simd' directive.
This represents clause 'task_reduction' in the '#pragma omp taskgroup' directives.
This represents '#pragma omp taskgroup' directive.
This represents '#pragma omp taskwait' directive.
This represents '#pragma omp taskyield' directive.
This represents '#pragma omp teams' directive.
This represents '#pragma omp teams distribute' directive.
This represents '#pragma omp teams distribute parallel for' composite directive.
This represents '#pragma omp teams distribute parallel for simd' composite directive.
This represents '#pragma omp teams distribute simd' combined directive.
This represents '#pragma omp teams loop' directive.
This represents 'thread_limit' clause in the '#pragma omp ...' directive.
This represents the '#pragma omp tile' loop transformation directive.
This represents the '#pragma omp unroll' loop transformation directive.
This represents 'untied' clause in the '#pragma omp ...' directive.
This represents 'update' clause in the '#pragma omp atomic' directive.
This represents the 'use' clause in '#pragma omp ...' directives.
This represents clause 'use_device_addr' in the '#pragma omp ...' directives.
This represents clause 'use_device_ptr' in the '#pragma omp ...' directives.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class.
static ParmVarDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, const IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, StorageClass S, Expr *DefArg)
PointerType - C99 6.7.5.1 - Pointer Declarators.
Represents an unpacked "presumed" location which can be presented to the user.
const char * getFilename() const
Return the presumed filename of this location.
unsigned getLine() const
Return the presumed line number of this location.
If a crash happens while one of these objects are live, the message is printed out along with the spe...
A (possibly-)qualified type.
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Represents a struct/union/class.
field_range fields() const
field_iterator field_begin() const
Base for LValueReferenceType and RValueReferenceType.
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
A trivial tuple used to represent a source range.
SourceLocation getEnd() const
SourceLocation getBegin() const
Stmt - This represents one statement.
SourceLocation getBeginLoc() const LLVM_READONLY
SwitchStmt - This represents a 'switch' stmt.
virtual bool hasBuiltinAtomic(uint64_t AtomicSizeInBits, uint64_t AlignmentInBits) const
Returns true if the given target supports lock-free atomic operations at the specified width and alig...
bool isTLSSupported() const
Whether the target supports thread-local storage.
The base class of the type hierarchy.
bool isPointerType() const
const T * castAs() const
Member-template castAs<specific type>.
bool isReferenceType() const
bool isScalarType() const
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isLValueReferenceType() const
bool isAnyComplexType() const
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g....
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type.
bool isAnyPointerType() const
bool isRecordType() const
UnaryOperator - This represents the unary-expression's (except sizeof and alignof),...
static UnaryOperator * Create(const ASTContext &C, Expr *input, Opcode opc, QualType type, ExprValueKind VK, ExprObjectKind OK, SourceLocation l, bool CanOverflow, FPOptionsOverride FPFeatures)
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a variable declaration or definition.
TLSKind getTLSKind() const
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
@ CInit
C-style initialization with assignment.
bool hasGlobalStorage() const
Returns true for all variables that do not have local storage.
bool isStaticLocal() const
Returns true if a variable with function scope is a static local variable.
const Expr * getInit() const
bool hasLocalStorage() const
Returns true if a variable with function scope is a non-static local variable.
@ TLS_None
Not a TLS variable.
Represents a C array with a specified size that is not an integer-constant-expression.
Expr * getSizeExpr() const
@ Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
bool Inc(InterpState &S, CodePtr OpPC, bool CanOverflow)
1) Pops a pointer from the stack 2) Load the value from the pointer 3) Writes the value increased by ...
The JSON file list parser is used to communicate input to InstallAPI.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
bool needsTaskBasedThreadLimit(OpenMPDirectiveKind DKind)
Checks if the specified target directive, combined or not, needs task based thread_limit.
@ Ctor_Complete
Complete object ctor.
if(T->getSizeExpr()) TRY_TO(TraverseStmt(const_cast< Expr * >(T -> getSizeExpr())))
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
@ OK_Ordinary
An ordinary object is located at an address in memory.
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
@ Tile
'tile' clause, allowed on 'loop' and Combined constructs.
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
@ OMPC_SCHEDULE_MODIFIER_unknown
llvm::omp::Clause OpenMPClauseKind
OpenMP clauses.
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
@ OMPC_DIST_SCHEDULE_unknown
bool isOpenMPTaskingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of tasking directives - task, taskloop,...
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
@ Result
The result type of a method or function.
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
bool isOpenMPGenericLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive constitutes a 'loop' directive in the outermost nest.
OpenMPBindClauseKind
OpenMP bindings for the 'bind' clause.
OpenMPDependClauseKind
OpenMP attributes for 'depend' clause.
@ Dtor_Complete
Complete object dtor.
OpenMPSeverityClauseKind
OpenMP attributes for 'severity' clause.
bool isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of the composite or combined directives that need loop ...
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
@ VK_PRValue
A pr-value expression (in the C++11 taxonomy) produces a temporary value.
@ VK_LValue
An l-value expression is a reference to an object with independent storage.
const FunctionProtoType * T
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
OpenMPNumThreadsClauseModifier
@ OMPC_NUMTHREADS_unknown
@ ThreadPrivateVar
Parameter for Thread private variable.
@ Other
Other implicit parameter.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
llvm::BasicBlock * getBlock() const
static Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
llvm::OpenMPIRBuilder::InsertPointTy InsertPointTy
static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Twine RegionName)
Emit the body of an OMP region that will be outlined in OpenMPIRBuilder::finalize().
static Address getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD)
Gets the OpenMP-specific address of the local variable /p VD.
static void EmitCaptureStmt(CodeGenFunction &CGF, InsertPointTy CodeGenIP, llvm::BasicBlock &FiniBB, llvm::Function *Fn, ArrayRef< llvm::Value * > Args)
static std::string getNameWithSeparators(ArrayRef< StringRef > Parts, StringRef FirstSeparator=".", StringRef Separator=".")
Get the platform-specific name separator.
static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP)
Emit the Finalization for an OMP region.
static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Twine RegionName)
Emit the body of an OMP region.
unsigned NumberOfTargetItems
Address BasePointersArray
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::IntegerType * SizeTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntPtrTy
llvm::PointerType * Int8PtrTy
SmallVector< const Expr *, 4 > DepExprs
EvalResult is a struct with detailed info about an evaluated expression.
Extra information about a function prototype.
Scheduling data for loop-based OpenMP directives.
OpenMPScheduleClauseModifier M2
OpenMPScheduleClauseModifier M1
OpenMPScheduleClauseKind Schedule