⋮ 202 lines hidden ⋮
1
//===- Construction of pass pipelines -------------------------------------===//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7
//===----------------------------------------------------------------------===//
10
/// This file provides the implementation of the PassBuilder based on our
11
/// static pass registry as well as related functionality. It also provides
12
/// helpers to aid in analyzing, debugging, and testing passes and pass
15
//===----------------------------------------------------------------------===//
17
#include "llvm/Analysis/AliasAnalysis.h"
18
#include "llvm/Analysis/BasicAliasAnalysis.h"
19
#include "llvm/Analysis/CGSCCPassManager.h"
20
#include "llvm/Analysis/CJAliasAnalysis.h"
21
#include "llvm/Analysis/GlobalsModRef.h"
22
#include "llvm/Analysis/InlineAdvisor.h"
23
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
24
#include "llvm/Analysis/ProfileSummaryInfo.h"
25
#include "llvm/Analysis/ScopedNoAliasAA.h"
26
#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
27
#include "llvm/IR/PassManager.h"
28
#include "llvm/Passes/OptimizationLevel.h"
29
#include "llvm/Passes/PassBuilder.h"
30
#include "llvm/Support/CommandLine.h"
31
#include "llvm/Support/ErrorHandling.h"
32
#include "llvm/Support/PGOOptions.h"
33
#include "llvm/Target/TargetMachine.h"
34
#include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
35
#include "llvm/Transforms/CJCFI/PtrAuthBackwardCFI.h"
36
#include "llvm/Transforms/Coroutines/CoroCleanup.h"
37
#include "llvm/Transforms/Coroutines/CoroConditionalWrapper.h"
38
#include "llvm/Transforms/Coroutines/CoroEarly.h"
39
#include "llvm/Transforms/Coroutines/CoroElide.h"
40
#include "llvm/Transforms/Coroutines/CoroSplit.h"
41
#include "llvm/Transforms/IPO/AlwaysInliner.h"
42
#include "llvm/Transforms/IPO/Annotation2Metadata.h"
43
#include "llvm/Transforms/IPO/ArgumentPromotion.h"
44
#include "llvm/Transforms/IPO/Attributor.h"
45
#include "llvm/Transforms/IPO/CalledValuePropagation.h"
46
#include "llvm/Transforms/IPO/CJPartialEscapeAnalysis.h"
47
#include "llvm/Transforms/IPO/ConstantMerge.h"
48
#include "llvm/Transforms/IPO/CrossDSOCFI.h"
49
#include "llvm/Transforms/IPO/DeadArgumentElimination.h"
50
#include "llvm/Transforms/IPO/ElimAvailExtern.h"
51
#include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
52
#include "llvm/Transforms/IPO/FunctionAttrs.h"
53
#include "llvm/Transforms/IPO/GlobalDCE.h"
54
#include "llvm/Transforms/IPO/GlobalOpt.h"
55
#include "llvm/Transforms/IPO/GlobalSplit.h"
56
#include "llvm/Transforms/IPO/HotColdSplitting.h"
57
#include "llvm/Transforms/IPO/IROutliner.h"
58
#include "llvm/Transforms/IPO/InferFunctionAttrs.h"
59
#include "llvm/Transforms/IPO/Inliner.h"
60
#include "llvm/Transforms/IPO/LowerTypeTests.h"
61
#include "llvm/Transforms/IPO/MergeFunctions.h"
62
#include "llvm/Transforms/IPO/ModuleInliner.h"
63
#include "llvm/Transforms/IPO/OpenMPOpt.h"
64
#include "llvm/Transforms/IPO/PartialInlining.h"
65
#include "llvm/Transforms/IPO/SCCP.h"
66
#include "llvm/Transforms/IPO/SampleProfile.h"
67
#include "llvm/Transforms/IPO/SampleProfileProbe.h"
68
#include "llvm/Transforms/IPO/SyntheticCountsPropagation.h"
69
#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
70
#include "llvm/Transforms/InstCombine/InstCombine.h"
71
#include "llvm/Transforms/Instrumentation/CGProfile.h"
72
#include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
73
#include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
74
#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
75
#include "llvm/Transforms/Instrumentation/MemProfiler.h"
76
#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
77
#include "llvm/Transforms/Scalar/ADCE.h"
78
#include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
79
#include "llvm/Transforms/Scalar/AnnotationRemarks.h"
80
#include "llvm/Transforms/Scalar/BDCE.h"
81
#include "llvm/Transforms/Scalar/CJBarrierSplit.h"
82
#include "llvm/Transforms/Scalar/CJDevirtualOpt.h"
83
#include "llvm/Transforms/Scalar/CJSimpleOpt.h"
84
#include "llvm/Transforms/Scalar/CJObjectReuseOpt.h"
85
#include "llvm/Transforms/Scalar/DCE.h"
86
#include "llvm/Transforms/Scalar/CJGenericIntrinsicOpt.h"
87
#include "llvm/Transforms/Scalar/CJLoopFloatOpt.h"
88
#include "llvm/Transforms/Scalar/CJRSSCE.h"
89
#include "llvm/Transforms/Scalar/CJSimpleRangeAnalysis.h"
90
#include "llvm/Transforms/Scalar/CallSiteSplitting.h"
91
#include "llvm/Transforms/Scalar/ConstraintElimination.h"
92
#include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
93
#include "llvm/Transforms/Scalar/DFAJumpThreading.h"
94
#include "llvm/Transforms/Scalar/DeadStoreElimination.h"
95
#include "llvm/Transforms/Scalar/DivRemPairs.h"
96
#include "llvm/Transforms/Scalar/EarlyCSE.h"
97
#include "llvm/Transforms/Scalar/Float2Int.h"
98
#include "llvm/Transforms/Scalar/GVN.h"
99
#include "llvm/Transforms/Scalar/IndVarSimplify.h"
100
#include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h"
101
#include "llvm/Transforms/Scalar/InstSimplifyPass.h"
102
#include "llvm/Transforms/Scalar/JumpThreading.h"
103
#include "llvm/Transforms/Scalar/LICM.h"
104
#include "llvm/Transforms/Scalar/LoopDeletion.h"
105
#include "llvm/Transforms/Scalar/LoopDistribute.h"
106
#include "llvm/Transforms/Scalar/LoopFlatten.h"
107
#include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
108
#include "llvm/Transforms/Scalar/LoopInstSimplify.h"
109
#include "llvm/Transforms/Scalar/LoopInterchange.h"
110
#include "llvm/Transforms/Scalar/LoopLoadElimination.h"
111
#include "llvm/Transforms/Scalar/LoopPassManager.h"
112
#include "llvm/Transforms/Scalar/LoopRotation.h"
113
#include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
114
#include "llvm/Transforms/Scalar/LoopSink.h"
115
#include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
116
#include "llvm/Transforms/Scalar/LoopUnrollPass.h"
117
#include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
118
#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
119
#include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
120
#include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
121
#include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
122
#include "llvm/Transforms/Scalar/NewGVN.h"
123
#include "llvm/Transforms/Scalar/Reassociate.h"
124
#include "llvm/Transforms/Scalar/SCCP.h"
125
#include "llvm/Transforms/Scalar/SROA.h"
126
#include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
127
#include "llvm/Transforms/Scalar/SimplifyCFG.h"
128
#include "llvm/Transforms/Scalar/SpeculativeExecution.h"
129
#include "llvm/Transforms/Scalar/TailRecursionElimination.h"
130
#include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
131
#include "llvm/Transforms/Utils/AddDiscriminators.h"
132
#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
133
#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
134
#include "llvm/Transforms/Utils/InjectTLIMappings.h"
135
#include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
136
#include "llvm/Transforms/Utils/Mem2Reg.h"
137
#include "llvm/Transforms/Utils/NameAnonGlobals.h"
138
#include "llvm/Transforms/Utils/RelLookupTableConverter.h"
139
#include "llvm/Transforms/Utils/SimplifyCFGOptions.h"
140
#include "llvm/Transforms/Vectorize/LoopVectorize.h"
141
#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
142
#include "llvm/Transforms/Vectorize/VectorCombine.h"
144
using namespace llvm;
146
static cl::opt<InliningAdvisorMode> UseInlineAdvisor(
147
"enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden,
148
cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"),
149
cl::values(clEnumValN(InliningAdvisorMode::Default, "default",
150
"Heuristics-based inliner version."),
151
clEnumValN(InliningAdvisorMode::Development, "development",
152
"Use development mode (runtime-loadable model)."),
153
clEnumValN(InliningAdvisorMode::Release, "release",
154
"Use release mode (AOT-compiled model).")));
156
static cl::opt<bool> EnableSyntheticCounts(
157
"enable-npm-synthetic-counts", cl::Hidden,
158
cl::desc("Run synthetic function entry count generation "
161
/// Flag to enable inline deferral during PGO.
163
EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true),
165
cl::desc("Enable inline deferral during PGO"));
167
static cl::opt<bool> EnableMemProfiler("enable-mem-prof", cl::Hidden,
168
cl::desc("Enable memory profiler"));
170
static cl::opt<bool> EnableModuleInliner("enable-module-inliner",
171
cl::init(false), cl::Hidden,
172
cl::desc("Enable module inliner"));
174
static cl::opt<bool> PerformMandatoryInliningsFirst(
175
"mandatory-inlining-first", cl::init(true), cl::Hidden,
176
cl::desc("Perform mandatory inlinings module-wide, before performing "
179
static cl::opt<bool> EnableO3NonTrivialUnswitching(
180
"enable-npm-O3-nontrivial-unswitch", cl::init(true), cl::Hidden,
181
cl::desc("Enable non-trivial loop unswitching for -O3"));
183
static cl::opt<bool> EnableEagerlyInvalidateAnalyses(
184
"eagerly-invalidate-analyses", cl::init(true), cl::Hidden,
185
cl::desc("Eagerly invalidate more analyses in default pipelines"));
187
static cl::opt<bool> EnableNoRerunSimplificationPipeline(
188
"enable-no-rerun-simplification-pipeline", cl::init(true), cl::Hidden,
190
"Prevent running the simplification pipeline on a function more "
191
"than once in the case that SCC mutations cause a function to be "
192
"visited multiple times as long as the function has not been changed"));
194
static cl::opt<bool> EnableMergeFunctions(
195
"enable-merge-functions", cl::init(false), cl::Hidden,
196
cl::desc("Enable function merging as part of the optimization pipeline"));
198
static cl::opt<bool> EnableCJDevirtual("enable-cj-devirtual", cl::init(true),
200
cl::desc("Enable CJ Devirtual"));
202
extern cl::opt<int> MaxRecursionInl;
203
extern cl::opt<int> CountedLoopTripWidth;
204
extern cl::opt<bool> CangjieLTOPreOpt;
205
extern cl::opt<bool> EnableCJPtrAuthBackwardCFI;
206
extern cl::opt<bool> DisableCJLTOReflection;
208
PipelineTuningOptions::PipelineTuningOptions() {
209
LoopInterleaving = true;
⋮ 1289 lines hidden ⋮
210
LoopVectorization = true;
211
SLPVectorization = false;
212
LoopUnrolling = true;
213
ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
214
LicmMssaOptCap = SetLicmMssaOptCap;
215
LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
216
CallGraphProfile = true;
217
MergeFunctions = EnableMergeFunctions;
218
EagerlyInvalidateAnalyses = EnableEagerlyInvalidateAnalyses;
223
extern cl::opt<unsigned> MaxDevirtIterations;
224
extern cl::opt<bool> EnableConstraintElimination;
225
extern cl::opt<bool> EnableFunctionSpecialization;
226
extern cl::opt<bool> EnableGVNHoist;
227
extern cl::opt<bool> EnableGVNSink;
228
extern cl::opt<bool> EnableHotColdSplit;
229
extern cl::opt<bool> EnableIROutliner;
230
extern cl::opt<bool> EnableOrderFileInstrumentation;
231
extern cl::opt<bool> EnableCHR;
232
extern cl::opt<bool> EnableLoopInterchange;
233
extern cl::opt<bool> EnableUnrollAndJam;
234
extern cl::opt<bool> EnableLoopFlatten;
235
extern cl::opt<bool> EnableDFAJumpThreading;
236
extern cl::opt<bool> RunNewGVN;
237
extern cl::opt<bool> RunPartialInlining;
238
extern cl::opt<bool> ExtraVectorizerPasses;
240
extern cl::opt<bool> FlattenedProfileUsed;
242
extern cl::opt<AttributorRunOption> AttributorRun;
243
extern cl::opt<bool> EnableKnowledgeRetention;
245
extern cl::opt<bool> EnableMatrix;
247
extern cl::opt<bool> DisablePreInliner;
248
extern cl::opt<int> PreInlineThreshold;
250
extern cl::opt<bool> CJPipeline;
251
extern cl::opt<bool> CJLTOOpt;
252
extern cl::opt<bool> CJDisableEscapeAnalysis;
253
extern cl::opt<bool> EnableCJBarrierSplit;
254
extern cl::opt<bool> EnableCJIRCEPass;
255
extern cl::opt<bool> EnableCJGenericIntrinsicOpt;
258
void PassBuilder::invokePeepholeEPCallbacks(FunctionPassManager &FPM,
259
OptimizationLevel Level) {
260
for (auto &C : PeepholeEPCallbacks)
264
// Helper to add AnnotationRemarksPass.
265
static void addAnnotationRemarksPass(ModulePassManager &MPM) {
266
MPM.addPass(createModuleToFunctionPassAdaptor(AnnotationRemarksPass()));
269
// Helper to check if the current compilation phase is preparing for LTO
270
static bool isLTOPreLink(ThinOrFullLTOPhase Phase) {
271
return Phase == ThinOrFullLTOPhase::ThinLTOPreLink ||
272
Phase == ThinOrFullLTOPhase::FullLTOPreLink;
275
static void setSLPVectorization(PipelineTuningOptions &PTO) {
277
PTO.SLPVectorization = true;
281
static void addEliminationOptPasses(FunctionPassManager &MainFPM,
282
PipelineTuningOptions &PTO) {
283
if (CJPipeline && EnableCJIRCEPass) {
284
// This pass should before IndVarSimplifyPass LFTR optimization as the
285
// latter might change predicate to eq or ne, which IRCE would not
286
// recognize. Also after LICM, because the SSA that needs to be compared
287
// should be a loop invariant.
288
MainFPM.addPass(IRCEPass());
289
// The new loop created by IRCE has potential unswitch conditions. If we do
290
// not perform this pass, the subsequent LICM may not work effectively on
293
SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
294
MainFPM.addPass(createFunctionToLoopPassAdaptor(SimpleLoopUnswitchPass()));
297
MainFPM.addPass(createFunctionToLoopPassAdaptor(
298
LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
299
/*AllowSpeculation=*/true),
300
/*USeMemorySSA=*/true, /*UseBlockFrequencyInfo=*/true));
303
MainFPM.addPass(NewGVNPass());
305
MainFPM.addPass(GVNPass());
307
// Remove dead memcpy()'s.
308
MainFPM.addPass(MemCpyOptPass());
311
MainFPM.addPass(CJRSSCEPass());
314
MainFPM.addPass(DSEPass());
315
MainFPM.addPass(MergedLoadStoreMotionPass());
318
// TODO: Investigate the cost/benefit of tail call elimination on debugging.
320
PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level,
321
ThinOrFullLTOPhase Phase) {
323
FunctionPassManager FPM;
325
// Form SSA out of local memory accesses after breaking apart aggregates into
327
FPM.addPass(SROAPass());
329
// Catch trivial redundancies
330
FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
332
// Hoisting of scalars and load expressions.
334
SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
335
FPM.addPass(InstCombinePass());
337
FPM.addPass(LibCallsShrinkWrapPass());
339
invokePeepholeEPCallbacks(FPM, Level);
342
SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
344
// Form canonically associated expression trees, and simplify the trees using
345
// basic mathematical properties. For example, this will form (nearly)
346
// minimal multiplication trees.
347
FPM.addPass(ReassociatePass());
349
// Add the primary loop simplification pipeline.
350
// FIXME: Currently this is split into two loop pass pipelines because we run
351
// some function passes in between them. These can and should be removed
352
// and/or replaced by scheduling the loop pass equivalents in the correct
353
// positions. But those equivalent passes aren't powerful enough yet.
354
// Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
355
// used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
356
// fully replace `SimplifyCFGPass`, and the closest to the other we have is
357
// `LoopInstSimplify`.
358
LoopPassManager LPM1, LPM2;
360
// Simplify the loop body. We do this initially to clean up after other loop
361
// passes run, either when iterating on a loop or on inner loops with
362
// implications on the outer loop.
363
LPM1.addPass(LoopInstSimplifyPass());
364
LPM1.addPass(LoopSimplifyCFGPass());
366
// Try to remove as much code from the loop header as possible,
367
// to reduce amount of IR that will have to be duplicated. However,
368
// do not perform speculative hoisting the first time as LICM
369
// will destroy metadata that may not need to be destroyed if run
370
// after loop rotation.
371
// TODO: Investigate promotion cap for O1.
372
LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
373
/*AllowSpeculation=*/false));
375
LPM1.addPass(LoopRotatePass(/* Disable header duplication */ true,
376
isLTOPreLink(Phase)));
377
// TODO: Investigate promotion cap for O1.
378
LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
379
/*AllowSpeculation=*/true));
380
LPM1.addPass(SimpleLoopUnswitchPass());
381
if (EnableLoopFlatten)
382
LPM1.addPass(LoopFlattenPass());
384
LPM2.addPass(LoopIdiomRecognizePass());
385
LPM2.addPass(IndVarSimplifyPass());
387
for (auto &C : LateLoopOptimizationsEPCallbacks)
390
LPM2.addPass(LoopDeletionPass());
392
if (EnableLoopInterchange)
393
LPM2.addPass(LoopInterchangePass());
395
// Do not enable unrolling in PreLinkThinLTO phase during sample PGO
396
// because it changes IR to makes profile annotation in back compile
397
// inaccurate. The normal unroller doesn't pay attention to forced full unroll
398
// attributes so we need to make sure and allow the full unroll pass to pay
400
if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink || !PGOOpt ||
401
PGOOpt->Action != PGOOptions::SampleUse)
402
LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
403
/* OnlyWhenForced= */ !PTO.LoopUnrolling,
404
PTO.ForgetAllSCEVInLoopUnroll));
406
for (auto &C : LoopOptimizerEndEPCallbacks)
409
// We provide the opt remark emitter pass for LICM to use. We only need to do
410
// this once as it is immutable.
412
RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
413
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1),
414
/*UseMemorySSA=*/true,
415
/*UseBlockFrequencyInfo=*/true));
417
SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
418
FPM.addPass(InstCombinePass());
419
// The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
420
// *All* loop passes must preserve it, in order to be able to use it.
421
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2),
422
/*UseMemorySSA=*/false,
423
/*UseBlockFrequencyInfo=*/false));
425
// Delete small array after loop unroll.
426
FPM.addPass(SROAPass());
428
// Specially optimize memory movement as it doesn't look like dataflow in SSA.
429
FPM.addPass(MemCpyOptPass());
431
// Sparse conditional constant propagation.
432
// FIXME: It isn't clear why we do this *after* loop passes rather than
434
FPM.addPass(SCCPPass());
436
// Delete dead bit computations (instcombine runs after to fold away the dead
437
// computations, and then ADCE will run later to exploit any new DCE
438
// opportunities that creates).
439
FPM.addPass(BDCEPass());
441
// Run instcombine after redundancy and dead bit elimination to exploit
442
// opportunities opened up by them.
443
FPM.addPass(InstCombinePass());
444
invokePeepholeEPCallbacks(FPM, Level);
446
FPM.addPass(CoroElidePass());
448
for (auto &C : ScalarOptimizerLateEPCallbacks)
451
// Finally, do an expensive DCE pass to catch all the dead code exposed by
452
// the simplifications and basic cleanup after all the simplifications.
453
// TODO: Investigate if this is too expensive.
454
FPM.addPass(ADCEPass());
456
SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
457
FPM.addPass(InstCombinePass());
458
invokePeepholeEPCallbacks(FPM, Level);
464
PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
465
ThinOrFullLTOPhase Phase) {
466
assert(Level != OptimizationLevel::O0 && "Must request optimizations!");
468
// The O1 pipeline has a separate pipeline creation function to simplify
469
// construction readability.
470
if (Level.getSpeedupLevel() == 1)
471
return buildO1FunctionSimplificationPipeline(Level, Phase);
473
FunctionPassManager FPM;
475
if (CJPipeline && Level == OptimizationLevel::O2) {
476
FPM.addPass(CJSimpleOpt());
479
// Form SSA out of local memory accesses after breaking apart aggregates into
481
FPM.addPass(SROAPass());
483
if (CJPipeline && Level == OptimizationLevel::O2) {
484
FPM.addPass(CJObjectReuseOpt());
485
FPM.addPass(DCEPass());
488
// Catch trivial redundancies
489
FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
490
if (EnableKnowledgeRetention)
491
FPM.addPass(AssumeSimplifyPass());
493
// Hoisting of scalars and load expressions.
495
FPM.addPass(GVNHoistPass());
497
// Global value numbering based sinking.
499
FPM.addPass(GVNSinkPass());
501
SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
504
if (EnableConstraintElimination)
505
FPM.addPass(ConstraintEliminationPass());
507
// Speculative execution if the target has divergent branches; otherwise nop.
508
FPM.addPass(SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true));
510
// Optimize based on known information about branches, and cleanup afterward.
511
FPM.addPass(JumpThreadingPass());
512
FPM.addPass(CorrelatedValuePropagationPass());
514
if (CJPipeline && Level == OptimizationLevel::O2) {
515
if (EnableCJGenericIntrinsicOpt)
516
FPM.addPass(CJGenericIntrinsicOpt());
517
FPM.addPass(CJSimpleRangeAnalysisPass());
521
SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
522
FPM.addPass(InstCombinePass());
523
if (Level == OptimizationLevel::O3)
524
FPM.addPass(AggressiveInstCombinePass());
526
if (!Level.isOptimizingForSize())
527
FPM.addPass(LibCallsShrinkWrapPass());
529
invokePeepholeEPCallbacks(FPM, Level);
531
// For PGO use pipeline, try to optimize memory intrinsics such as memcpy
532
// using the size value profile. Don't perform this when optimizing for size.
533
if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse &&
534
!Level.isOptimizingForSize())
535
FPM.addPass(PGOMemOPSizeOpt());
537
if (!CangjieLTOPreOpt)
538
FPM.addPass(TailCallElimPass());
541
SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
543
// Form canonically associated expression trees, and simplify the trees using
544
// basic mathematical properties. For example, this will form (nearly)
545
// minimal multiplication trees.
546
FPM.addPass(ReassociatePass());
548
// Add the primary loop simplification pipeline.
549
// FIXME: Currently this is split into two loop pass pipelines because we run
550
// some function passes in between them. These can and should be removed
551
// and/or replaced by scheduling the loop pass equivalents in the correct
552
// positions. But those equivalent passes aren't powerful enough yet.
553
// Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
554
// used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
555
// fully replace `SimplifyCFGPass`, and the closest to the other we have is
556
// `LoopInstSimplify`.
557
LoopPassManager LPM1, LPM2;
560
LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
561
/*AllowSpeculation=*/false));
562
// 2: Opt size level, Oz. -1: argument for MaxHeaderSize
563
LPM1.addPass(LoopRotatePass(Level == OptimizationLevel::O2 ? true : false));
564
FPM.addPass(InstCombinePass());
565
LPM1.addPass(IndVarSimplifyPass());
566
LPM1.addPass(LoopDeletionPass());
569
// Simplify the loop body. We do this initially to clean up after other loop
570
// passes run, either when iterating on a loop or on inner loops with
571
// implications on the outer loop.
572
LPM1.addPass(LoopInstSimplifyPass());
573
LPM1.addPass(LoopSimplifyCFGPass());
575
// Try to remove as much code from the loop header as possible,
576
// to reduce amount of IR that will have to be duplicated. However,
577
// do not perform speculative hoisting the first time as LICM
578
// will destroy metadata that may not need to be destroyed if run
579
// after loop rotation.
580
// TODO: Investigate promotion cap for O1.
581
LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
582
/*AllowSpeculation=*/false));
584
// Disable header duplication in loop rotation at -Oz.
586
LoopRotatePass(Level != OptimizationLevel::Oz, isLTOPreLink(Phase)));
587
// TODO: Investigate promotion cap for O1.
588
LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
589
/*AllowSpeculation=*/true));
591
SimpleLoopUnswitchPass(/* NonTrivial */ Level == OptimizationLevel::O3 &&
592
EnableO3NonTrivialUnswitching));
593
if (EnableLoopFlatten)
594
LPM1.addPass(LoopFlattenPass());
596
LPM2.addPass(LoopIdiomRecognizePass());
597
LPM2.addPass(IndVarSimplifyPass());
599
for (auto &C : LateLoopOptimizationsEPCallbacks)
602
LPM2.addPass(LoopDeletionPass());
604
if (EnableLoopInterchange)
605
LPM2.addPass(LoopInterchangePass());
607
// Do not enable unrolling in PreLinkThinLTO phase during sample PGO
608
// because it changes IR to makes profile annotation in back compile
609
// inaccurate. The normal unroller doesn't pay attention to forced full unroll
610
// attributes so we need to make sure and allow the full unroll pass to pay
612
if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink || !PGOOpt ||
613
PGOOpt->Action != PGOOptions::SampleUse)
614
LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
615
/* OnlyWhenForced= */ !PTO.LoopUnrolling,
616
PTO.ForgetAllSCEVInLoopUnroll));
618
for (auto &C : LoopOptimizerEndEPCallbacks)
621
// We provide the opt remark emitter pass for LICM to use. We only need to do
622
// this once as it is immutable.
624
RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
625
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1),
626
/*UseMemorySSA=*/true,
627
/*UseBlockFrequencyInfo=*/true));
629
SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
630
FPM.addPass(InstCombinePass());
632
FPM.addPass(CJLoopFloatOpt());
634
// The loop passes in LPM2 (LoopIdiomRecognizePass, IndVarSimplifyPass,
635
// LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
636
// *All* loop passes must preserve it, in order to be able to use it.
637
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2),
638
/*UseMemorySSA=*/false,
639
/*UseBlockFrequencyInfo=*/false));
641
// Delete small array after loop unroll.
642
FPM.addPass(SROAPass());
644
// The matrix extension can introduce large vector operations early, which can
645
// benefit from running vector-combine early on.
647
FPM.addPass(VectorCombinePass(/*ScalarizationOnly=*/true));
649
// Eliminate redundancies.
650
FPM.addPass(MergedLoadStoreMotionPass());
652
FPM.addPass(NewGVNPass());
654
FPM.addPass(GVNPass());
656
// Sparse conditional constant propagation.
657
// FIXME: It isn't clear why we do this *after* loop passes rather than
659
FPM.addPass(SCCPPass());
661
// Delete dead bit computations (instcombine runs after to fold away the dead
662
// computations, and then ADCE will run later to exploit any new DCE
663
// opportunities that creates).
664
FPM.addPass(BDCEPass());
666
// Run instcombine after redundancy and dead bit elimination to exploit
667
// opportunities opened up by them.
668
FPM.addPass(InstCombinePass());
669
invokePeepholeEPCallbacks(FPM, Level);
671
// Re-consider control flow based optimizations after redundancy elimination,
673
if (EnableDFAJumpThreading && Level.getSizeLevel() == 0)
674
FPM.addPass(DFAJumpThreadingPass());
676
FPM.addPass(JumpThreadingPass());
677
FPM.addPass(CorrelatedValuePropagationPass());
679
// Finally, do an expensive DCE pass to catch all the dead code exposed by
680
// the simplifications and basic cleanup after all the simplifications.
681
// TODO: Investigate if this is too expensive.
682
FPM.addPass(ADCEPass());
684
// Specially optimize memory movement as it doesn't look like dataflow in SSA.
685
FPM.addPass(MemCpyOptPass());
687
if (CJPipeline && Level == OptimizationLevel::O2) {
688
FPM.addPass(CJRSSCEPass());
689
FPM.addPass(CJObjectReuseOpt());
692
FPM.addPass(DSEPass());
693
FPM.addPass(createFunctionToLoopPassAdaptor(
694
LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
695
/*AllowSpeculation=*/true),
696
/*UseMemorySSA=*/true, /*UseBlockFrequencyInfo=*/true));
698
FPM.addPass(CoroElidePass());
700
for (auto &C : ScalarOptimizerLateEPCallbacks)
703
FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions()
704
.convertSwitchRangeToICmp(true)
705
.hoistCommonInsts(true)
706
.sinkCommonInsts(true)));
707
FPM.addPass(InstCombinePass());
708
invokePeepholeEPCallbacks(FPM, Level);
710
if (EnableCHR && Level == OptimizationLevel::O3 && PGOOpt &&
711
(PGOOpt->Action == PGOOptions::IRUse ||
712
PGOOpt->Action == PGOOptions::SampleUse))
713
FPM.addPass(ControlHeightReductionPass());
718
void PassBuilder::addRequiredLTOPreLinkPasses(ModulePassManager &MPM) {
719
MPM.addPass(CanonicalizeAliasesPass());
720
MPM.addPass(NameAnonGlobalPass());
723
void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
724
OptimizationLevel Level, bool RunProfileGen,
725
bool IsCS, std::string ProfileFile,
726
std::string ProfileRemappingFile,
727
ThinOrFullLTOPhase LTOPhase) {
728
assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
729
if (!IsCS && !DisablePreInliner) {
732
IP.DefaultThreshold = PreInlineThreshold;
734
// FIXME: The hint threshold has the same value used by the regular inliner
735
// when not optimzing for size. This should probably be lowered after
736
// performance testing.
737
// FIXME: this comment is cargo culted from the old pass manager, revisit).
738
IP.HintThreshold = Level.isOptimizingForSize() ? PreInlineThreshold : 325;
739
ModuleInlinerWrapperPass MIWP(
740
IP, /* MandatoryFirst */ true,
741
InlineContext{LTOPhase, InlinePass::EarlyInliner});
742
CGSCCPassManager &CGPipeline = MIWP.getPM();
744
FunctionPassManager FPM;
745
FPM.addPass(SROAPass());
746
FPM.addPass(EarlyCSEPass()); // Catch trivial redundancies.
747
FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(
748
true))); // Merge & remove basic blocks.
749
FPM.addPass(InstCombinePass()); // Combine silly sequences.
750
invokePeepholeEPCallbacks(FPM, Level);
752
CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
753
std::move(FPM), PTO.EagerlyInvalidateAnalyses));
755
MPM.addPass(std::move(MIWP));
757
// Delete anything that is now dead to make sure that we don't instrument
758
// dead code. Instrumentation can end up keeping dead code around and
759
// dramatically increase code size.
760
MPM.addPass(GlobalDCEPass());
763
if (!RunProfileGen) {
764
assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
765
MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
766
// Cache ProfileSummaryAnalysis once to avoid the potential need to insert
767
// RequireAnalysisPass for PSI before subsequent non-module passes.
768
MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
772
// Perform PGO instrumentation.
773
MPM.addPass(PGOInstrumentationGen(IsCS));
775
// Disable header duplication in loop rotation at -Oz.
776
MPM.addPass(createModuleToFunctionPassAdaptor(
777
createFunctionToLoopPassAdaptor(
778
LoopRotatePass(Level != OptimizationLevel::Oz),
779
/*UseMemorySSA=*/false,
780
/*UseBlockFrequencyInfo=*/false),
781
PTO.EagerlyInvalidateAnalyses));
783
// Add the profile lowering pass.
784
InstrProfOptions Options;
785
if (!ProfileFile.empty())
786
Options.InstrProfileOutput = ProfileFile;
787
// Do counter promotion at Level greater than O0.
788
Options.DoCounterPromotion = true;
789
Options.UseBFIInPromotion = IsCS;
790
MPM.addPass(InstrProfiling(Options, IsCS));
793
void PassBuilder::addPGOInstrPassesForO0(ModulePassManager &MPM,
794
bool RunProfileGen, bool IsCS,
795
std::string ProfileFile,
796
std::string ProfileRemappingFile) {
797
if (!RunProfileGen) {
798
assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
799
MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
800
// Cache ProfileSummaryAnalysis once to avoid the potential need to insert
801
// RequireAnalysisPass for PSI before subsequent non-module passes.
802
MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
806
// Perform PGO instrumentation.
807
MPM.addPass(PGOInstrumentationGen(IsCS));
808
// Add the profile lowering pass.
809
InstrProfOptions Options;
810
if (!ProfileFile.empty())
811
Options.InstrProfileOutput = ProfileFile;
812
// Do not do counter promotion at O0.
813
Options.DoCounterPromotion = false;
814
Options.UseBFIInPromotion = IsCS;
815
MPM.addPass(InstrProfiling(Options, IsCS));
818
static InlineParams getInlineParamsFromOptLevel(OptimizationLevel Level) {
819
return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel());
822
ModuleInlinerWrapperPass
823
PassBuilder::buildInlinerPipeline(OptimizationLevel Level,
824
ThinOrFullLTOPhase Phase) {
825
InlineParams IP = getInlineParamsFromOptLevel(Level);
826
// For PreLinkThinLTO + SamplePGO, set hot-caller threshold to 0 to
827
// disable hot callsite inline (as much as possible [1]) because it makes
828
// profile annotation in the backend inaccurate.
830
// [1] Note the cost of a function could be below zero due to erased
831
// prologue / epilogue.
832
if (Phase == ThinOrFullLTOPhase::ThinLTOPreLink && PGOOpt &&
833
PGOOpt->Action == PGOOptions::SampleUse)
834
IP.HotCallSiteThreshold = 0;
837
IP.EnableDeferral = EnablePGOInlineDeferral;
839
ModuleInlinerWrapperPass MIWP(IP, PerformMandatoryInliningsFirst,
840
InlineContext{Phase, InlinePass::CGSCCInliner},
841
UseInlineAdvisor, MaxDevirtIterations);
843
// Require the GlobalsAA analysis for the module so we can query it within
844
// the CGSCC pipeline.
845
MIWP.addModulePass(RequireAnalysisPass<GlobalsAA, Module>());
846
// Invalidate AAManager so it can be recreated and pick up the newly available
849
createModuleToFunctionPassAdaptor(InvalidateAnalysisPass<AAManager>()));
851
// Require the ProfileSummaryAnalysis for the module so we can query it within
853
MIWP.addModulePass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
855
// Now begin the main postorder CGSCC pipeline.
856
// FIXME: The current CGSCC pipeline has its origins in the legacy pass
857
// manager and trying to emulate its precise behavior. Much of this doesn't
858
// make a lot of sense and we should revisit the core CGSCC structure.
859
CGSCCPassManager &MainCGPipeline = MIWP.getPM();
861
// Note: historically, the PruneEH pass was run first to deduce nounwind and
862
// generally clean up exception handling overhead. It isn't clear this is
863
// valuable as the inliner doesn't currently care whether it is inlining an
866
if (AttributorRun & AttributorRunOption::CGSCC)
867
MainCGPipeline.addPass(AttributorCGSCCPass());
869
// Now deduce any function attributes based in the current code.
870
MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
872
// When at O3 add argument promotion to the pass pipeline.
873
// FIXME: It isn't at all clear why this should be limited to O3.
874
if (Level == OptimizationLevel::O3)
875
MainCGPipeline.addPass(ArgumentPromotionPass());
877
// Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
878
// there are no OpenMP runtime calls present in the module.
879
if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3)
880
MainCGPipeline.addPass(OpenMPOptCGSCCPass());
882
for (auto &C : CGSCCOptimizerLateEPCallbacks)
883
C(MainCGPipeline, Level);
886
if (Level == OptimizationLevel::O2) {
887
if (EnableCJDevirtual)
888
MainCGPipeline.addPass(CJDevirtualOpt());
890
if (!CJDisableEscapeAnalysis) {
891
MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(SimplifyCFGPass(
892
SimplifyCFGOptions().convertSwitchRangeToICmp(true))));
893
MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(InstCombinePass()));
894
MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(SCCPPass()));
895
MainCGPipeline.addPass(CJPartialEscapeAnalysisPass());
899
// Lastly, add the core function simplification pipeline nested inside the
901
MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
902
buildFunctionSimplificationPipeline(Level, Phase),
903
PTO.EagerlyInvalidateAnalyses, EnableNoRerunSimplificationPipeline));
905
MainCGPipeline.addPass(CoroSplitPass(Level != OptimizationLevel::O0));
907
if (EnableNoRerunSimplificationPipeline)
908
MIWP.addLateModulePass(createModuleToFunctionPassAdaptor(
909
InvalidateAnalysisPass<ShouldNotRunFunctionPassesAnalysis>()));
915
PassBuilder::buildModuleInlinerPipeline(OptimizationLevel Level,
916
ThinOrFullLTOPhase Phase) {
917
ModulePassManager MPM;
919
InlineParams IP = getInlineParamsFromOptLevel(Level);
920
// For PreLinkThinLTO + SamplePGO, set hot-caller threshold to 0 to
921
// disable hot callsite inline (as much as possible [1]) because it makes
922
// profile annotation in the backend inaccurate.
924
// [1] Note the cost of a function could be below zero due to erased
925
// prologue / epilogue.
926
if (Phase == ThinOrFullLTOPhase::ThinLTOPreLink && PGOOpt &&
927
PGOOpt->Action == PGOOptions::SampleUse)
928
IP.HotCallSiteThreshold = 0;
931
IP.EnableDeferral = EnablePGOInlineDeferral;
933
// The inline deferral logic is used to avoid losing some
934
// inlining chance in future. It is helpful in SCC inliner, in which
935
// inlining is processed in bottom-up order.
936
// While in module inliner, the inlining order is a priority-based order
937
// by default. The inline deferral is unnecessary there. So we disable the
938
// inline deferral logic in module inliner.
939
IP.EnableDeferral = false;
941
MPM.addPass(ModuleInlinerPass(IP, UseInlineAdvisor, Phase));
944
if (Level == OptimizationLevel::O2) {
945
if (EnableCJDevirtual)
946
MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(CJDevirtualOpt()));
948
if (!CJDisableEscapeAnalysis) {
949
MPM.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass(
950
SimplifyCFGOptions().convertSwitchRangeToICmp(true))));
951
MPM.addPass(createModuleToFunctionPassAdaptor(InstCombinePass()));
952
MPM.addPass(createModuleToFunctionPassAdaptor(SCCPPass()));
954
createModuleToPostOrderCGSCCPassAdaptor(
955
CJPartialEscapeAnalysisPass()));
959
MPM.addPass(createModuleToFunctionPassAdaptor(
960
buildFunctionSimplificationPipeline(Level, Phase),
961
PTO.EagerlyInvalidateAnalyses));
963
MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
964
CoroSplitPass(Level != OptimizationLevel::O0)));
970
PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
971
ThinOrFullLTOPhase Phase) {
972
ModulePassManager MPM;
974
// Place pseudo probe instrumentation as the first pass of the pipeline to
975
// minimize the impact of optimization changes.
976
if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
977
Phase != ThinOrFullLTOPhase::ThinLTOPostLink)
978
MPM.addPass(SampleProfileProbePass(TM));
980
bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
982
// In ThinLTO mode, when flattened profile is used, all the available
983
// profile information will be annotated in PreLink phase so there is
984
// no need to load the profile again in PostLink.
985
bool LoadSampleProfile =
987
!(FlattenedProfileUsed && Phase == ThinOrFullLTOPhase::ThinLTOPostLink);
989
// During the ThinLTO backend phase we perform early indirect call promotion
990
// here, before globalopt. Otherwise imported available_externally functions
991
// look unreferenced and are removed. If we are going to load the sample
992
// profile then defer until later.
993
// TODO: See if we can move later and consolidate with the location where
994
// we perform ICP when we are loading a sample profile.
995
// TODO: We pass HasSampleProfile (whether there was a sample profile file
996
// passed to the compile) to the SamplePGO flag of ICP. This is used to
997
// determine whether the new direct calls are annotated with prof metadata.
998
// Ideally this should be determined from whether the IR is annotated with
999
// sample profile, and not whether the a sample profile was provided on the
1000
// command line. E.g. for flattened profiles where we will not be reloading
1001
// the sample profile in the ThinLTO backend, we ideally shouldn't have to
1002
// provide the sample profile file.
1003
if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink && !LoadSampleProfile)
1004
MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
1006
// Do basic inference of function attributes from known properties of system
1007
// libraries and other oracles.
1008
MPM.addPass(InferFunctionAttrsPass());
1009
MPM.addPass(CoroEarlyPass());
1011
// Create an early function pass manager to cleanup the output of the
1013
FunctionPassManager EarlyFPM;
1015
EarlyFPM.addPass(createFunctionToLoopPassAdaptor(LoopSimplifyCFGPass()));
1017
// Lower llvm.expect to metadata before attempting transforms.
1018
// Compare/branch metadata may alter the behavior of passes like SimplifyCFG.
1019
EarlyFPM.addPass(LowerExpectIntrinsicPass());
1020
EarlyFPM.addPass(SimplifyCFGPass());
1021
EarlyFPM.addPass(SROAPass());
1022
EarlyFPM.addPass(EarlyCSEPass());
1024
if (CJPipeline && Level == OptimizationLevel::O2) {
1025
EarlyFPM.addPass(CJObjectReuseOpt());
1026
EarlyFPM.addPass(DCEPass());
1029
if (Level == OptimizationLevel::O3)
1030
EarlyFPM.addPass(CallSiteSplittingPass());
1032
// In SamplePGO ThinLTO backend, we need instcombine before profile annotation
1033
// to convert bitcast to direct calls so that they can be inlined during the
1034
// profile annotation prepration step.
1035
// More details about SamplePGO design can be found in:
1036
// https://research.google.com/pubs/pub45290.html
1037
// FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
1038
if (LoadSampleProfile)
1039
EarlyFPM.addPass(InstCombinePass());
1040
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM),
1041
PTO.EagerlyInvalidateAnalyses));
1043
if (LoadSampleProfile) {
1044
// Annotate sample profile right after early FPM to ensure freshness of
1046
MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1047
PGOOpt->ProfileRemappingFile, Phase));
1048
// Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1049
// RequireAnalysisPass for PSI before subsequent non-module passes.
1050
MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1051
// Do not invoke ICP in the LTOPrelink phase as it makes it hard
1052
// for the profile annotation to be accurate in the LTO backend.
1053
if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink &&
1054
Phase != ThinOrFullLTOPhase::FullLTOPreLink)
1055
// We perform early indirect call promotion here, before globalopt.
1056
// This is important for the ThinLTO backend phase because otherwise
1057
// imported available_externally functions look unreferenced and are
1060
PGOIndirectCallPromotion(true /* IsInLTO */, true /* SamplePGO */));
1063
// Try to perform OpenMP specific optimizations on the module. This is a
1064
// (quick!) no-op if there are no OpenMP runtime calls present in the module.
1065
if (Level != OptimizationLevel::O0)
1066
MPM.addPass(OpenMPOptPass());
1068
if (AttributorRun & AttributorRunOption::MODULE)
1069
MPM.addPass(AttributorPass());
1071
// Lower type metadata and the type.test intrinsic in the ThinLTO
1072
// post link pipeline after ICP. This is to enable usage of the type
1073
// tests in ICP sequences.
1074
if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink)
1075
MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1077
for (auto &C : PipelineEarlySimplificationEPCallbacks)
1080
// Specialize functions with IPSCCP.
1081
if (EnableFunctionSpecialization && Level == OptimizationLevel::O3)
1082
MPM.addPass(FunctionSpecializationPass());
1084
// Interprocedural constant propagation now that basic cleanup has occurred
1085
// and prior to optimizing globals.
1086
// FIXME: This position in the pipeline hasn't been carefully considered in
1087
// years, it should be re-analyzed.
1088
MPM.addPass(IPSCCPPass());
1090
// Attach metadata to indirect call sites indicating the set of functions
1091
// they may target at run-time. This should follow IPSCCP.
1092
MPM.addPass(CalledValuePropagationPass());
1094
// Optimize globals to try and fold them into constants.
1095
MPM.addPass(GlobalOptPass());
1097
// Promote any localized globals to SSA registers.
1098
// FIXME: Should this instead by a run of SROA?
1099
// FIXME: We should probably run instcombine and simplifycfg afterward to
1100
// delete control flows that are dead once globals have been folded to
1102
MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1104
// Remove any dead arguments exposed by cleanups and constant folding
1106
MPM.addPass(DeadArgumentEliminationPass());
1108
// Create a small function pass pipeline to cleanup after all the global
1110
FunctionPassManager GlobalCleanupPM;
1111
GlobalCleanupPM.addPass(InstCombinePass());
1112
invokePeepholeEPCallbacks(GlobalCleanupPM, Level);
1114
GlobalCleanupPM.addPass(
1115
SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
1116
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM),
1117
PTO.EagerlyInvalidateAnalyses));
1119
// Add all the requested passes for instrumentation PGO, if requested.
1120
if (!CangjieLTOPreOpt && PGOOpt &&
1121
Phase != ThinOrFullLTOPhase::ThinLTOPostLink &&
1122
(PGOOpt->Action == PGOOptions::IRInstr ||
1123
PGOOpt->Action == PGOOptions::IRUse)) {
1124
addPGOInstrPasses(MPM, Level,
1125
/* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr,
1126
/* IsCS */ false, PGOOpt->ProfileFile,
1127
PGOOpt->ProfileRemappingFile, Phase);
1128
MPM.addPass(PGOIndirectCallPromotion(CJLTOOpt, false));
1130
if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink &&
1131
PGOOpt->CSAction == PGOOptions::CSIRInstr)
1132
MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile));
1134
// Synthesize function entry counts for non-PGO compilation.
1135
if (EnableSyntheticCounts && !PGOOpt)
1136
MPM.addPass(SyntheticCountsPropagation());
1138
if (EnableModuleInliner)
1139
MPM.addPass(buildModuleInlinerPipeline(Level, Phase));
1141
MPM.addPass(buildInlinerPipeline(Level, Phase));
1143
MPM.addPass(CoroCleanupPass());
1145
if (EnableMemProfiler && Phase != ThinOrFullLTOPhase::ThinLTOPreLink) {
1146
MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass()));
1147
MPM.addPass(ModuleMemProfilerPass());
1153
/// TODO: Should LTO cause any differences to this set of passes?
1154
void PassBuilder::addVectorPasses(OptimizationLevel Level,
1155
FunctionPassManager &FPM, bool IsFullLTO) {
1156
FPM.addPass(LoopVectorizePass(
1157
LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization)));
1159
// The vectorizer may have significantly shortened a loop body; unroll
1160
// again. Unroll small loops to hide loop backedge latency and saturate any
1161
// parallel execution resources of an out-of-order processor. We also then
1162
// need to clean up redundancies and loop invariant code.
1163
// FIXME: It would be really good to use a loop-integrated instruction
1164
// combiner for cleanup here so that the unrolling and LICM can be pipelined
1165
// across the loop nests.
1166
// We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1167
if (EnableUnrollAndJam && PTO.LoopUnrolling)
1168
FPM.addPass(createFunctionToLoopPassAdaptor(
1169
LoopUnrollAndJamPass(Level.getSpeedupLevel())));
1170
FPM.addPass(LoopUnrollPass(LoopUnrollOptions(
1171
Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1172
PTO.ForgetAllSCEVInLoopUnroll)));
1173
FPM.addPass(WarnMissedTransformationsPass());
1177
// Eliminate loads by forwarding stores from the previous iteration to loads
1178
// of the current iteration.
1179
FPM.addPass(LoopLoadEliminationPass());
1181
// Cleanup after the loop optimization passes.
1182
FPM.addPass(InstCombinePass());
1184
if (Level.getSpeedupLevel() > 1 && ExtraVectorizerPasses) {
1185
ExtraVectorPassManager ExtraPasses;
1186
// At higher optimization levels, try to clean up any runtime overlap and
1187
// alignment checks inserted by the vectorizer. We want to track correlated
1188
// runtime checks for two inner loops in the same outer loop, fold any
1189
// common computations, hoist loop-invariant aspects out of any outer loop,
1190
// and unswitch the runtime checks if possible. Once hoisted, we may have
1191
// dead (or speculatable) control flows or more combining opportunities.
1192
ExtraPasses.addPass(EarlyCSEPass());
1193
ExtraPasses.addPass(CorrelatedValuePropagationPass());
1194
ExtraPasses.addPass(InstCombinePass());
1195
LoopPassManager LPM;
1196
LPM.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
1197
/*AllowSpeculation=*/true));
1198
LPM.addPass(SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
1199
OptimizationLevel::O3));
1200
ExtraPasses.addPass(
1201
RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
1202
ExtraPasses.addPass(
1203
createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA=*/true,
1204
/*UseBlockFrequencyInfo=*/true));
1205
ExtraPasses.addPass(
1206
SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
1207
ExtraPasses.addPass(InstCombinePass());
1208
FPM.addPass(std::move(ExtraPasses));
1211
// Now that we've formed fast to execute loop structures, we do further
1212
// optimizations. These are run afterward as they might block doing complex
1213
// analyses and transforms such as what are needed for loop vectorization.
1215
// Cleanup after loop vectorization, etc. Simplification passes like CVP and
1216
// GVN, loop transforms, and others have already run, so it's now better to
1217
// convert to more optimized IR using more aggressive simplify CFG options.
1218
// The extra sinking transform can create larger basic blocks, so do this
1219
// before SLP vectorization.
1220
FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions()
1221
.forwardSwitchCondToPhi(true)
1222
.convertSwitchRangeToICmp(true)
1223
.convertSwitchToLookupTable(true)
1224
.needCanonicalLoops(false)
1225
.hoistCommonInsts(true)
1226
.sinkCommonInsts(true)));
1229
FPM.addPass(SCCPPass());
1230
FPM.addPass(InstCombinePass());
1231
FPM.addPass(BDCEPass());
1234
// Optimize parallel scalar instruction chains into SIMD instructions.
1235
if (PTO.SLPVectorization) {
1236
FPM.addPass(SLPVectorizerPass());
1237
if (Level.getSpeedupLevel() > 1 && ExtraVectorizerPasses) {
1238
FPM.addPass(EarlyCSEPass());
1241
// Enhance/cleanup vector code.
1242
FPM.addPass(VectorCombinePass());
1245
FPM.addPass(InstCombinePass());
1246
// Unroll small loops to hide loop backedge latency and saturate any
1247
// parallel execution resources of an out-of-order processor. We also then
1248
// need to clean up redundancies and loop invariant code.
1249
// FIXME: It would be really good to use a loop-integrated instruction
1250
// combiner for cleanup here so that the unrolling and LICM can be pipelined
1251
// across the loop nests.
1252
// We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1253
if (EnableUnrollAndJam && PTO.LoopUnrolling) {
1254
FPM.addPass(createFunctionToLoopPassAdaptor(
1255
LoopUnrollAndJamPass(Level.getSpeedupLevel())));
1257
FPM.addPass(LoopUnrollPass(LoopUnrollOptions(
1258
Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1259
PTO.ForgetAllSCEVInLoopUnroll)));
1260
FPM.addPass(WarnMissedTransformationsPass());
1261
FPM.addPass(InstCombinePass());
1263
RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
1264
FPM.addPass(createFunctionToLoopPassAdaptor(
1265
LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
1266
/*AllowSpeculation=*/true),
1267
/*UseMemorySSA=*/true, /*UseBlockFrequencyInfo=*/true));
1270
// Now that we've vectorized and unrolled loops, we may have more refined
1271
// alignment information, try to re-derive it here.
1272
FPM.addPass(AlignmentFromAssumptionsPass());
1275
FPM.addPass(InstCombinePass());
1279
PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
1280
ThinOrFullLTOPhase LTOPhase) {
1281
const bool LTOPreLink = (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink ||
1282
LTOPhase == ThinOrFullLTOPhase::FullLTOPreLink);
1283
ModulePassManager MPM;
1285
// Optimize globals now that the module is fully simplified.
1286
MPM.addPass(GlobalOptPass());
1287
MPM.addPass(GlobalDCEPass());
1289
// Run partial inlining pass to partially inline functions that have
1291
if (RunPartialInlining)
1292
MPM.addPass(PartialInlinerPass());
1294
// Remove avail extern fns and globals definitions since we aren't compiling
1295
// an object file for later LTO. For LTO we want to preserve these so they
1296
// are eligible for inlining at link-time. Note if they are unreferenced they
1297
// will be removed by GlobalDCE later, so this only impacts referenced
1298
// available externally globals. Eventually they will be suppressed during
1299
// codegen, but eliminating here enables more opportunity for GlobalDCE as it
1300
// may make globals referenced by available external functions dead and saves
1301
// running remaining passes on the eliminated functions. These should be
1302
// preserved during prelinking for link-time inlining decisions.
1303
if (!LTOPreLink && !CJPipeline) {
1304
MPM.addPass(EliminateAvailableExternallyPass());
1307
if (EnableOrderFileInstrumentation)
1308
MPM.addPass(InstrOrderFilePass());
1310
// Do RPO function attribute inference across the module to forward-propagate
1311
// attributes where applicable.
1312
// FIXME: Is this really an optimization rather than a canonicalization?
1313
MPM.addPass(ReversePostOrderFunctionAttrsPass());
1315
// Do a post inline PGO instrumentation and use pass. This is a context
1316
// sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
1317
// cross-module inline has not been done yet. The context sensitive
1318
// instrumentation is after all the inlines are done.
1319
if (!LTOPreLink && PGOOpt) {
1320
if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1321
addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true,
1322
/* IsCS */ true, PGOOpt->CSProfileGenFile,
1323
PGOOpt->ProfileRemappingFile, LTOPhase);
1324
else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1325
addPGOInstrPasses(MPM, Level, /* RunProfileGen */ false,
1326
/* IsCS */ true, PGOOpt->ProfileFile,
1327
PGOOpt->ProfileRemappingFile, LTOPhase);
1330
// Re-compute GlobalsAA here prior to function passes. This is particularly
1331
// useful as the above will have inlined, DCE'ed, and function-attr
1332
// propagated everything. We should at this point have a reasonably minimal
1333
// and richly annotated call graph. By computing aliasing and mod/ref
1334
// information for all local globals here, the late loop passes and notably
1335
// the vectorizer will be able to use them to help recognize vectorizable
1336
// memory operations.
1337
MPM.addPass(RecomputeGlobalsAAPass());
1339
for (auto &C : OptimizerEarlyEPCallbacks)
1342
FunctionPassManager OptimizePM;
1343
OptimizePM.addPass(Float2IntPass());
1344
OptimizePM.addPass(LowerConstantIntrinsicsPass());
1347
OptimizePM.addPass(LowerMatrixIntrinsicsPass());
1348
OptimizePM.addPass(EarlyCSEPass());
1351
// FIXME: We need to run some loop optimizations to re-rotate loops after
1352
// simplifycfg and others undo their rotation.
1354
// Optimize the loop execution. These passes operate on entire loop nests
1355
// rather than on each loop in an inside-out manner, and so they are actually
1358
for (auto &C : VectorizerStartEPCallbacks)
1359
C(OptimizePM, Level);
1363
createModuleToPostOrderCGSCCPassAdaptor(PostOrderFunctionAttrsPass()));
1365
FunctionPassManager MainFPM;
1366
addEliminationOptPasses(MainFPM, PTO);
1367
MPM.addPass(createModuleToFunctionPassAdaptor(
1368
std::move(MainFPM), PTO.EagerlyInvalidateAnalyses));
1371
LoopPassManager LPM;
1372
// First rotate loops that may have been un-rotated by prior passes.
1373
// Disable header duplication at -Oz.
1374
LPM.addPass(LoopRotatePass(Level != OptimizationLevel::Oz, LTOPreLink));
1375
// Some loops may have become dead by now. Try to delete them.
1376
// FIXME: see discussion in https://reviews.llvm.org/D112851,
1377
// this may need to be revisited once we run GVN before loop deletion
1378
// in the simplification pipeline.
1379
LPM.addPass(LoopDeletionPass());
1380
OptimizePM.addPass(createFunctionToLoopPassAdaptor(
1381
std::move(LPM), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false));
1383
// Distribute loops to allow partial vectorization. I.e. isolate dependences
1384
// into separate loop that would otherwise inhibit vectorization. This is
1385
// currently only performed for loops marked with the metadata
1386
// llvm.loop.distribute=true or when -enable-loop-distribute is specified.
1387
OptimizePM.addPass(LoopDistributePass());
1389
// Populates the VFABI attribute with the scalar-to-vector mappings
1390
// from the TargetLibraryInfo.
1391
OptimizePM.addPass(InjectTLIMappings());
1393
addVectorPasses(Level, OptimizePM, /* IsFullLTO */ false);
1396
if (Level == OptimizationLevel::O2) {
1397
OptimizePM.addPass(CJAfterInlineSimpleOpt());
1399
OptimizePM.addPass(createFunctionToLoopPassAdaptor(IndVarSimplifyPass()));
1400
OptimizePM.addPass(EarlyCSEPass());
1403
// LoopSink pass sinks instructions hoisted by LICM, which serves as a
1404
// canonicalization pass that enables other optimizations. As a result,
1405
// LoopSink pass needs to be a very late IR pass to avoid undoing LICM
1406
// result too early.
1407
OptimizePM.addPass(LoopSinkPass());
1409
// And finally clean up LCSSA form before generating code.
1410
OptimizePM.addPass(InstSimplifyPass());
1412
// This hoists/decomposes div/rem ops. It should run after other sink/hoist
1413
// passes to avoid re-sinking, but before SimplifyCFG because it can allow
1414
// flattening of blocks.
1415
OptimizePM.addPass(DivRemPairsPass());
1417
// Try to annotate calls that were created during optimization.
1418
if (!CangjieLTOPreOpt)
1419
OptimizePM.addPass(TailCallElimPass());
1421
// LoopSink (and other loop passes since the last simplifyCFG) might have
1422
// resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
1424
SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
1426
// Add the core optimizing pipeline.
1427
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM),
1428
PTO.EagerlyInvalidateAnalyses));
1430
for (auto &C : OptimizerLastEPCallbacks)
1433
// Split out cold code. Splitting is done late to avoid hiding context from
1434
// other optimizations and inadvertently regressing performance. The tradeoff
1435
// is that this has a higher code size cost than splitting early.
1436
if (EnableHotColdSplit && !LTOPreLink)
1437
MPM.addPass(HotColdSplittingPass());
1439
// Search the code for similar regions of code. If enough similar regions can
1440
// be found where extracting the regions into their own function will decrease
1441
// the size of the program, we extract the regions, a deduplicate the
1442
// structurally similar regions.
1443
if (EnableIROutliner)
1444
MPM.addPass(IROutlinerPass());
1446
// Merge functions if requested.
1447
if (PTO.MergeFunctions)
1448
MPM.addPass(MergeFunctionsPass());
1450
if (!LTOPreLink && CJPipeline) {
1451
MPM.addPass(EliminateAvailableExternallyPass());
1454
// Now we need to do some global optimization transforms.
1455
// FIXME: It would seem like these should come first in the optimization
1456
// pipeline and maybe be the bottom of the canonicalization pipeline? Weird
1458
MPM.addPass(GlobalDCEPass());
1459
MPM.addPass(ConstantMergePass());
1461
if (PTO.CallGraphProfile && !LTOPreLink)
1462
MPM.addPass(CGProfilePass());
1464
// TODO: Relative look table converter pass caused an issue when full lto is
1465
// enabled. See https://reviews.llvm.org/D94355 for more details.
1466
// Until the issue fixed, disable this pass during pre-linking phase.
1468
MPM.addPass(RelLookupTableConverterPass());
1474
PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
1476
assert(Level != OptimizationLevel::O0 &&
1477
"Must request optimizations for the default pipeline!");
1479
ModulePassManager MPM;
1481
setSLPVectorization(PTO);
1483
// Convert @llvm.global.annotations to !annotation metadata.
1484
MPM.addPass(Annotation2MetadataPass());
1486
// Force any function attributes we want the rest of the pipeline to observe.
1487
MPM.addPass(ForceFunctionAttrsPass());
1489
// Apply module pipeline start EP callback.
1490
for (auto &C : PipelineStartEPCallbacks)
1493
if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1494
MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1496
ThinOrFullLTOPhase LTOPhase = ThinOrFullLTOPhase::None;
1497
if (CJPipeline && LTOPreLink)
1498
LTOPhase = ThinOrFullLTOPhase::ThinLTOPreLink;
1499
else if (LTOPreLink)
1500
LTOPhase = ThinOrFullLTOPhase::FullLTOPreLink;
1502
// Import-lib reflection is trimmed at post-link by CJDisableImportLibReflection;
1503
// the main package already has TF_REFLECTION=0 under --disable-reflection.
1504
// Add the core simplification pipeline.
1505
MPM.addPass(buildModuleSimplificationPipeline(Level, LTOPhase));
⋮ 109 lines hidden ⋮
1507
// Now add the optimization pipeline.
1508
MPM.addPass(buildModuleOptimizationPipeline(Level, LTOPhase));
1510
if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
1511
PGOOpt->Action == PGOOptions::SampleUse)
1512
MPM.addPass(PseudoProbeUpdatePass());
1514
// Emit annotation remarks.
1515
addAnnotationRemarksPass(MPM);
1518
addRequiredLTOPreLinkPasses(MPM);
1524
PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
1525
assert(Level != OptimizationLevel::O0 &&
1526
"Must request optimizations for the default pipeline!");
1528
ModulePassManager MPM;
1530
setSLPVectorization(PTO);
1532
// Convert @llvm.global.annotations to !annotation metadata.
1533
MPM.addPass(Annotation2MetadataPass());
1535
// Force any function attributes we want the rest of the pipeline to observe.
1536
MPM.addPass(ForceFunctionAttrsPass());
1538
if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1539
MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1541
// Apply module pipeline start EP callback.
1542
for (auto &C : PipelineStartEPCallbacks)
1545
// If we are planning to perform ThinLTO later, we don't bloat the code with
1546
// unrolling/vectorization/... now. Just simplify the module as much as we
1548
MPM.addPass(buildModuleSimplificationPipeline(
1549
Level, ThinOrFullLTOPhase::ThinLTOPreLink));
1551
// Run partial inlining pass to partially inline functions that have
1553
// FIXME: It isn't clear whether this is really the right place to run this
1554
// in ThinLTO. Because there is another canonicalization and simplification
1555
// phase that will run after the thin link, running this here ends up with
1556
// less information than will be available later and it may grow functions in
1557
// ways that aren't beneficial.
1558
if (RunPartialInlining)
1559
MPM.addPass(PartialInlinerPass());
1561
// Reduce the size of the IR as much as possible.
1562
MPM.addPass(GlobalOptPass());
1564
if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
1565
PGOOpt->Action == PGOOptions::SampleUse)
1566
MPM.addPass(PseudoProbeUpdatePass());
1568
// Handle OptimizerLastEPCallbacks added by clang on PreLink. Actual
1569
// optimization is going to be done in PostLink stage, but clang can't
1570
// add callbacks there in case of in-process ThinLTO called by linker.
1571
for (auto &C : OptimizerLastEPCallbacks)
1574
// Emit annotation remarks.
1575
addAnnotationRemarksPass(MPM);
1577
addRequiredLTOPreLinkPasses(MPM);
1582
ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
1583
OptimizationLevel Level, const ModuleSummaryIndex *ImportSummary) {
1584
ModulePassManager MPM;
1586
setSLPVectorization(PTO);
1588
// Convert @llvm.global.annotations to !annotation metadata.
1589
MPM.addPass(Annotation2MetadataPass());
1591
if (ImportSummary) {
1592
// These passes import type identifier resolutions for whole-program
1593
// devirtualization and CFI. They must run early because other passes may
1594
// disturb the specific instruction patterns that these passes look for,
1595
// creating dependencies on resolutions that may not appear in the summary.
1597
// For example, GVN may transform the pattern assume(type.test) appearing in
1598
// two basic blocks into assume(phi(type.test, type.test)), which would
1599
// transform a dependency on a WPD resolution into a dependency on a type
1600
// identifier resolution for CFI.
1602
// Also, WPD has access to more precise information than ICP and can
1603
// devirtualize more effectively, so it should operate on the IR first.
1605
// The WPD and LowerTypeTest passes need to run at -O0 to lower type
1606
// metadata and intrinsics.
1607
MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));
1608
MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
1611
if (Level == OptimizationLevel::O0) {
1612
// Run a second time to clean up any type tests left behind by WPD for use
1614
MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1615
// Drop available_externally and unreferenced globals. This is necessary
1616
// with ThinLTO in order to avoid leaving undefined references to dead
1617
// globals in the object file.
1618
MPM.addPass(EliminateAvailableExternallyPass());
1619
if (CJPipeline && DisableCJLTOReflection)
1620
MPM.addPass(CJDisableImportLibReflection());
1621
MPM.addPass(GlobalDCEPass());
⋮ 1 lines hidden ⋮
1625
// Force any function attributes we want the rest of the pipeline to observe.
1626
MPM.addPass(ForceFunctionAttrsPass());
1628
if (CJPipeline && DisableCJLTOReflection) {
1629
MPM.addPass(CJDisableImportLibReflection());
1632
// Add the core simplification pipeline.
1633
MPM.addPass(buildModuleSimplificationPipeline(
1634
Level, ThinOrFullLTOPhase::ThinLTOPostLink));
⋮ 61 lines hidden ⋮
1636
// Now add the optimization pipeline.
1637
MPM.addPass(buildModuleOptimizationPipeline(
1638
Level, ThinOrFullLTOPhase::ThinLTOPostLink));
1640
// Emit annotation remarks.
1641
addAnnotationRemarksPass(MPM);
1647
PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
1648
assert(Level != OptimizationLevel::O0 &&
1649
"Must request optimizations for the default pipeline!");
1650
// FIXME: We should use a customized pre-link pipeline!
1651
return buildPerModuleDefaultPipeline(Level,
1652
/* LTOPreLink */ true);
1656
PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
1657
ModuleSummaryIndex *ExportSummary) {
1658
ModulePassManager MPM;
1660
setSLPVectorization(PTO);
1662
// Convert @llvm.global.annotations to !annotation metadata.
1663
MPM.addPass(Annotation2MetadataPass());
1665
for (auto &C : FullLinkTimeOptimizationEarlyEPCallbacks)
1668
// Create a function that performs CFI checks for cross-DSO calls with targets
1669
// in the current module.
1670
MPM.addPass(CrossDSOCFIPass());
1671
if (CJPipeline && EnableCJPtrAuthBackwardCFI) {
1672
MPM.addPass(PtrAuthBackwardCFI());
1675
if (Level == OptimizationLevel::O0) {
1677
if (PGOOpt && (PGOOpt->Action == PGOOptions::IRInstr ||
1678
PGOOpt->Action == PGOOptions::IRUse))
1679
addPGOInstrPassesForO0(
1681
/* RunProfileGen */ (PGOOpt->Action == PGOOptions::IRInstr),
1682
/* IsCS */ false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
1685
// The WPD and LowerTypeTest passes need to run at -O0 to lower type
1686
// metadata and intrinsics.
1687
MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1688
MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1689
// Run a second time to clean up any type tests left behind by WPD for use
1691
MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1693
for (auto &C : FullLinkTimeOptimizationLastEPCallbacks)
1696
// Emit annotation remarks.
1697
addAnnotationRemarksPass(MPM);
1699
if (CJPipeline && DisableCJLTOReflection) {
1700
MPM.addPass(CJDisableImportLibReflection());
1701
// No GlobalDCE at -O0; the pass self-erases dead reflection globals.
⋮ 8 lines hidden ⋮
1707
// Add all the requested passes for instrumentation PGO, if requested.
1708
if (CJPipeline && !CangjieLTOPreOpt && PGOOpt &&
1709
(PGOOpt->Action == PGOOptions::IRInstr ||
1710
PGOOpt->Action == PGOOptions::IRUse)) {
1711
addPGOInstrPasses(MPM, Level,
1712
/* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr,
1713
/* IsCS */ false, PGOOpt->ProfileFile,
1714
PGOOpt->ProfileRemappingFile,
1715
ThinOrFullLTOPhase::FullLTOPostLink);
1718
if (CJPipeline && DisableCJLTOReflection) {
1719
MPM.addPass(CJDisableImportLibReflection());
1722
if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
1723
// Load sample profile before running the LTO optimization pipeline.
1724
MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
⋮ 415 lines hidden ⋮
1725
PGOOpt->ProfileRemappingFile,
1726
ThinOrFullLTOPhase::FullLTOPostLink));
1727
// Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1728
// RequireAnalysisPass for PSI before subsequent non-module passes.
1729
MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1732
// Try to run OpenMP optimizations, quick no-op if no OpenMP metadata present.
1733
MPM.addPass(OpenMPOptPass());
1735
// Remove unused virtual tables to improve the quality of code generated by
1736
// whole-program devirtualization and bitset lowering.
1737
MPM.addPass(GlobalDCEPass());
1739
// Force any function attributes we want the rest of the pipeline to observe.
1740
MPM.addPass(ForceFunctionAttrsPass());
1742
// Do basic inference of function attributes from known properties of system
1743
// libraries and other oracles.
1744
MPM.addPass(InferFunctionAttrsPass());
1746
if (Level.getSpeedupLevel() > 1) {
1747
MPM.addPass(createModuleToFunctionPassAdaptor(
1748
CallSiteSplittingPass(), PTO.EagerlyInvalidateAnalyses));
1750
// Indirect call promotion. This should promote all the targets that are
1751
// left by the earlier promotion pass that promotes intra-module targets.
1752
// This two-step promotion is to save the compile time. For LTO, it should
1753
// produce the same result as if we only do promotion here.
1754
MPM.addPass(PGOIndirectCallPromotion(
1755
true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1757
if (EnableFunctionSpecialization && Level == OptimizationLevel::O3)
1758
MPM.addPass(FunctionSpecializationPass());
1759
// Propagate constants at call sites into the functions they call. This
1760
// opens opportunities for globalopt (and inlining) by substituting function
1761
// pointers passed as arguments to direct uses of functions.
1762
MPM.addPass(IPSCCPPass());
1764
// Attach metadata to indirect call sites indicating the set of functions
1765
// they may target at run-time. This should follow IPSCCP.
1766
MPM.addPass(CalledValuePropagationPass());
1769
// Now deduce any function attributes based in the current code.
1771
createModuleToPostOrderCGSCCPassAdaptor(PostOrderFunctionAttrsPass()));
1773
// Do RPO function attribute inference across the module to forward-propagate
1774
// attributes where applicable.
1775
// FIXME: Is this really an optimization rather than a canonicalization?
1776
MPM.addPass(ReversePostOrderFunctionAttrsPass());
1778
// Use in-range annotations on GEP indices to split globals where beneficial.
1779
MPM.addPass(GlobalSplitPass());
1781
// Run whole program optimization of virtual call when the list of callees
1783
MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1785
// Stop here at -O1.
1786
if (Level == OptimizationLevel::O1) {
1787
// The LowerTypeTestsPass needs to run to lower type metadata and the
1788
// type.test intrinsics. The pass does nothing if CFI is disabled.
1789
MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1790
// Run a second time to clean up any type tests left behind by WPD for use
1791
// in ICP (which is performed earlier than this in the regular LTO
1793
MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1795
for (auto &C : FullLinkTimeOptimizationLastEPCallbacks)
1798
// Emit annotation remarks.
1799
addAnnotationRemarksPass(MPM);
1804
// Optimize globals to try and fold them into constants.
1805
MPM.addPass(GlobalOptPass());
1807
// Promote any localized globals to SSA registers.
1808
MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1810
// Linking modules together can lead to duplicate global constant, only
1811
// keep one copy of each constant.
1812
MPM.addPass(ConstantMergePass());
1814
// Remove unused arguments from functions.
1815
MPM.addPass(DeadArgumentEliminationPass());
1817
// Reduce the code after globalopt and ipsccp. Both can open up significant
1818
// simplification opportunities, and both can propagate functions through
1819
// function pointers. When this happens, we often have to resolve varargs
1820
// calls, etc, so let instcombine do this.
1821
FunctionPassManager PeepholeFPM;
1822
PeepholeFPM.addPass(InstCombinePass());
1823
if (Level == OptimizationLevel::O3)
1824
PeepholeFPM.addPass(AggressiveInstCombinePass());
1825
invokePeepholeEPCallbacks(PeepholeFPM, Level);
1827
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM),
1828
PTO.EagerlyInvalidateAnalyses));
1830
// Note: historically, the PruneEH pass was run first to deduce nounwind and
1831
// generally clean up exception handling overhead. It isn't clear this is
1832
// valuable as the inliner doesn't currently care whether it is inlining an
1833
// invoke or a call.
1834
// Run the inliner now.
1835
MPM.addPass(ModuleInlinerWrapperPass(
1836
getInlineParamsFromOptLevel(Level),
1837
/* MandatoryFirst */ true,
1838
InlineContext{ThinOrFullLTOPhase::FullLTOPostLink,
1839
InlinePass::CGSCCInliner}));
1842
if (Level == OptimizationLevel::O2) {
1843
if (EnableCJDevirtual)
1844
MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(CJDevirtualOpt()));
1846
if (!CJDisableEscapeAnalysis) {
1848
createModuleToFunctionPassAdaptor(SimplifyCFGPass(
1849
SimplifyCFGOptions().convertSwitchRangeToICmp(true))));
1850
MPM.addPass(createModuleToFunctionPassAdaptor(InstCombinePass()));
1851
MPM.addPass(createModuleToFunctionPassAdaptor(SCCPPass()));
1853
createModuleToPostOrderCGSCCPassAdaptor(
1854
CJPartialEscapeAnalysisPass()));
1858
// Optimize globals again after we ran the inliner.
1859
MPM.addPass(GlobalOptPass());
1861
// Garbage collect dead functions.
1862
MPM.addPass(GlobalDCEPass());
1864
// If we didn't decide to inline a function, check to see if we can
1865
// transform it to pass arguments by value instead of by reference.
1866
MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(ArgumentPromotionPass()));
1869
MPM.addPass(createModuleToFunctionPassAdaptor(
1870
buildFunctionSimplificationPipeline(
1871
Level, ThinOrFullLTOPhase::FullLTOPostLink),
1872
PTO.EagerlyInvalidateAnalyses));
1875
if (RunPartialInlining)
1876
MPM.addPass(PartialInlinerPass());
1878
FunctionPassManager FPM;
1879
// The IPO Passes may leave cruft around. Clean up after them.
1880
FPM.addPass(InstCombinePass());
1881
invokePeepholeEPCallbacks(FPM, Level);
1883
FPM.addPass(JumpThreadingPass());
1885
// Do a post inline PGO instrumentation and use pass. This is a context
1886
// sensitive PGO pass.
1888
if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1889
addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true,
1890
/* IsCS */ true, PGOOpt->CSProfileGenFile,
1891
PGOOpt->ProfileRemappingFile,
1892
ThinOrFullLTOPhase::FullLTOPostLink);
1893
else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1894
addPGOInstrPasses(MPM, Level, /* RunProfileGen */ false,
1895
/* IsCS */ true, PGOOpt->ProfileFile,
1896
PGOOpt->ProfileRemappingFile,
1897
ThinOrFullLTOPhase::FullLTOPostLink);
1901
FPM.addPass(SROAPass());
1903
// LTO provides additional opportunities for tailcall elimination due to
1904
// link-time inlining, and visibility of nocapture attribute.
1905
if (!CangjieLTOPreOpt )
1906
FPM.addPass(TailCallElimPass());
1908
// Run a few AA driver optimizations here and now to cleanup the code.
1909
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM),
1910
PTO.EagerlyInvalidateAnalyses));
1913
createModuleToPostOrderCGSCCPassAdaptor(PostOrderFunctionAttrsPass()));
1915
// Require the GlobalsAA analysis for the module so we can query it within
1917
MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
1918
// Invalidate AAManager so it can be recreated and pick up the newly available
1921
createModuleToFunctionPassAdaptor(InvalidateAnalysisPass<AAManager>()));
1923
FunctionPassManager MainFPM;
1924
addEliminationOptPasses(MainFPM, PTO);
1926
if (EnableConstraintElimination)
1927
MainFPM.addPass(ConstraintEliminationPass());
1929
LoopPassManager LPM;
1930
if (EnableLoopFlatten && Level.getSpeedupLevel() > 1)
1931
LPM.addPass(LoopFlattenPass());
1932
LPM.addPass(IndVarSimplifyPass());
1933
LPM.addPass(LoopDeletionPass());
1934
// FIXME: Add loop interchange.
1936
// Unroll small loops and perform peeling.
1937
LPM.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
1938
/* OnlyWhenForced= */ !PTO.LoopUnrolling,
1939
PTO.ForgetAllSCEVInLoopUnroll));
1940
// The loop passes in LPM (LoopFullUnrollPass) do not preserve MemorySSA.
1941
// *All* loop passes must preserve it, in order to be able to use it.
1942
MainFPM.addPass(createFunctionToLoopPassAdaptor(
1943
std::move(LPM), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/true));
1945
MainFPM.addPass(LoopDistributePass());
1947
addVectorPasses(Level, MainFPM, /* IsFullLTO */ true);
1949
// Run the OpenMPOpt CGSCC pass again late.
1951
createModuleToPostOrderCGSCCPassAdaptor(OpenMPOptCGSCCPass()));
1953
invokePeepholeEPCallbacks(MainFPM, Level);
1954
MainFPM.addPass(JumpThreadingPass());
1955
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM),
1956
PTO.EagerlyInvalidateAnalyses));
1958
// Lower type metadata and the type.test intrinsic. This pass supports
1959
// clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
1960
// to be run at link time if CFI is enabled. This pass does nothing if
1962
MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1963
// Run a second time to clean up any type tests left behind by WPD for use
1964
// in ICP (which is performed earlier than this in the regular LTO pipeline).
1965
MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1967
// Enable splitting late in the FullLTO post-link pipeline.
1968
if (EnableHotColdSplit)
1969
MPM.addPass(HotColdSplittingPass());
1971
// Add late LTO optimization passes.
1972
// Delete basic blocks, which optimization passes may have killed.
1973
MPM.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass(
1974
SimplifyCFGOptions().convertSwitchRangeToICmp(true).hoistCommonInsts(
1977
// Drop bodies of available eternally objects to improve GlobalDCE.
1978
MPM.addPass(EliminateAvailableExternallyPass());
1980
// Now that we have optimized the program, discard unreachable functions.
1981
MPM.addPass(GlobalDCEPass());
1983
// It will conflict with VFE, so it is necessary to ensure that VFE is not
1984
// executed after MergeFunction.
1985
if (PTO.MergeFunctions || CJPipeline)
1986
MPM.addPass(MergeFunctionsPass());
1988
if (PTO.CallGraphProfile)
1989
MPM.addPass(CGProfilePass());
1991
for (auto &C : FullLinkTimeOptimizationLastEPCallbacks)
1994
// Emit annotation remarks.
1995
addAnnotationRemarksPass(MPM);
2000
ModulePassManager PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level,
2002
assert(Level == OptimizationLevel::O0 &&
2003
"buildO0DefaultPipeline should only be used with O0");
2005
ModulePassManager MPM;
2007
// Perform pseudo probe instrumentation in O0 mode. This is for the
2008
// consistency between different build modes. For example, a LTO build can be
2009
// mixed with an O0 prelink and an O2 postlink. Loading a sample profile in
2010
// the postlink will require pseudo probe instrumentation in the prelink.
2011
if (PGOOpt && PGOOpt->PseudoProbeForProfiling)
2012
MPM.addPass(SampleProfileProbePass(TM));
2014
if (!CangjieLTOPreOpt && PGOOpt &&
2015
(PGOOpt->Action == PGOOptions::IRInstr ||
2016
PGOOpt->Action == PGOOptions::IRUse))
2017
addPGOInstrPassesForO0(
2019
/* RunProfileGen */ (PGOOpt->Action == PGOOptions::IRInstr),
2020
/* IsCS */ false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
2022
for (auto &C : PipelineStartEPCallbacks)
2025
if (PGOOpt && PGOOpt->DebugInfoForProfiling)
2026
MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
2028
for (auto &C : PipelineEarlySimplificationEPCallbacks)
2031
// Build a minimal pipeline based on the semantics required by LLVM,
2032
// which is just that always inlining occurs. Further, disable generating
2033
// lifetime intrinsics to avoid enabling further optimizations during
2035
MPM.addPass(AlwaysInlinerPass(
2036
/*InsertLifetimeIntrinsics=*/false));
2038
if (PTO.MergeFunctions)
2039
MPM.addPass(MergeFunctionsPass());
2043
createModuleToFunctionPassAdaptor(LowerMatrixIntrinsicsPass(true)));
2045
if (!CGSCCOptimizerLateEPCallbacks.empty()) {
2046
CGSCCPassManager CGPM;
2047
for (auto &C : CGSCCOptimizerLateEPCallbacks)
2049
if (!CGPM.isEmpty())
2050
MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
2052
if (!LateLoopOptimizationsEPCallbacks.empty()) {
2053
LoopPassManager LPM;
2054
for (auto &C : LateLoopOptimizationsEPCallbacks)
2056
if (!LPM.isEmpty()) {
2057
MPM.addPass(createModuleToFunctionPassAdaptor(
2058
createFunctionToLoopPassAdaptor(std::move(LPM))));
2061
if (!LoopOptimizerEndEPCallbacks.empty()) {
2062
LoopPassManager LPM;
2063
for (auto &C : LoopOptimizerEndEPCallbacks)
2065
if (!LPM.isEmpty()) {
2066
MPM.addPass(createModuleToFunctionPassAdaptor(
2067
createFunctionToLoopPassAdaptor(std::move(LPM))));
2070
if (!ScalarOptimizerLateEPCallbacks.empty()) {
2071
FunctionPassManager FPM;
2072
for (auto &C : ScalarOptimizerLateEPCallbacks)
2075
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
2078
for (auto &C : OptimizerEarlyEPCallbacks)
2081
if (!VectorizerStartEPCallbacks.empty()) {
2082
FunctionPassManager FPM;
2083
for (auto &C : VectorizerStartEPCallbacks)
2086
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
2089
ModulePassManager CoroPM;
2090
CoroPM.addPass(CoroEarlyPass());
2091
CGSCCPassManager CGPM;
2092
CGPM.addPass(CoroSplitPass());
2093
CoroPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
2094
CoroPM.addPass(CoroCleanupPass());
2095
CoroPM.addPass(GlobalDCEPass());
2096
MPM.addPass(CoroConditionalWrapper(std::move(CoroPM)));
2098
for (auto &C : OptimizerLastEPCallbacks)
2102
addRequiredLTOPreLinkPasses(MPM);
2104
MPM.addPass(createModuleToFunctionPassAdaptor(AnnotationRemarksPass()));
2109
AAManager PassBuilder::buildDefaultAAPipeline() {
2112
// The order in which these are registered determines their priority when
2115
// First we register the basic alias analysis that provides the majority of
2116
// per-function local AA logic. This is a stateless, on-demand local set of
2118
AA.registerFunctionAnalysis<BasicAA>();
2120
// Next we query fast, specialized alias analyses that wrap IR-embedded
2121
// information about aliasing.
2122
AA.registerFunctionAnalysis<ScopedNoAliasAA>();
2123
AA.registerFunctionAnalysis<TypeBasedAA>();
2126
AA.registerFunctionAnalysis<CangjieAA>();
2128
// Add support for querying global aliasing information when available.
2129
// Because the `AAManager` is a function analysis and `GlobalsAA` is a module
2130
// analysis, all that the `AAManager` can do is query for any *cached*
2131
// results from `GlobalsAA` through a readonly proxy.
2132
AA.registerModuleAnalysis<GlobalsAA>();
2134
// Add target-specific alias analyses.
2136
TM->registerDefaultAliasAnalyses(AA);