LLVM 24.0.0git
AutoUpgrade.cpp
Go to the documentation of this file.
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/StringExtras.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ADT/StringSwitch.h"
20#include "llvm/BinaryFormat/Dwarf.h"
21#include "llvm/IR/AttributeMask.h"
22#include "llvm/IR/Attributes.h"
23#include "llvm/IR/CallingConv.h"
24#include "llvm/IR/Constants.h"
25#include "llvm/IR/DebugInfo.h"
26#include "llvm/IR/DebugInfoMetadata.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/Function.h"
29#include "llvm/IR/GlobalValue.h"
30#include "llvm/IR/IRBuilder.h"
31#include "llvm/IR/InstVisitor.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/IntrinsicInst.h"
34#include "llvm/IR/Intrinsics.h"
35#include "llvm/IR/IntrinsicsAArch64.h"
36#include "llvm/IR/IntrinsicsAMDGPU.h"
37#include "llvm/IR/IntrinsicsARM.h"
38#include "llvm/IR/IntrinsicsNVPTX.h"
39#include "llvm/IR/IntrinsicsRISCV.h"
40#include "llvm/IR/IntrinsicsWebAssembly.h"
41#include "llvm/IR/IntrinsicsX86.h"
42#include "llvm/IR/LLVMContext.h"
43#include "llvm/IR/MDBuilder.h"
44#include "llvm/IR/Metadata.h"
45#include "llvm/IR/Module.h"
46#include "llvm/IR/NVVMIntrinsicUtils.h"
47#include "llvm/IR/Value.h"
48#include "llvm/IR/Verifier.h"
49#include "llvm/Support/AMDGPUAddrSpace.h"
50#include "llvm/Support/CodeGen.h"
51#include "llvm/Support/CommandLine.h"
52#include "llvm/Support/ErrorHandling.h"
53#include "llvm/Support/NVPTXAddrSpace.h"
54#include "llvm/Support/NVVMAttributes.h"
55#include "llvm/Support/Regex.h"
56#include "llvm/Support/TimeProfiler.h"
57#include "llvm/TargetParser/Triple.h"
58#include <cstdint>
59#include <cstring>
60#include <numeric>
61
62using namespace llvm;
63
64static cl::opt<bool>
65 DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info",
66 cl::desc("Disable autoupgrade of debug info"));
67
68static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
69
70// Report a fatal error along with the
71// Call Instruction which caused the error
72[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
73 CallBase *CI) {
74 CI->print(llvm::errs());
75 llvm::errs() << "\n";
76 reportFatalUsageError(reason);
77}
78
79// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
80// changed their type from v4f32 to v2i64.
81static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
82 Function *&NewFn) {
83 // Check whether this is an old version of the function, which received
84 // v4f32 arguments.
85 Type *Arg0Type = F->getFunctionType()->getParamType(0);
86 if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
87 return false;
88
89 // Yes, it's old, replace it with new version.
90 rename(F);
91 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
92 return true;
93}
94
95// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
96// arguments have changed their type from i32 to i8.
97static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
98 Function *&NewFn) {
99 // Check that the last argument is an i32.
100 Type *LastArgType = F->getFunctionType()->getParamType(
101 F->getFunctionType()->getNumParams() - 1);
102 if (!LastArgType->isIntegerTy(32))
103 return false;
104
105 // Move this function aside and map down.
106 rename(F);
107 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
108 return true;
109}
110
111// Upgrade the declaration of fp compare intrinsics that change return type
112// from scalar to vXi1 mask.
113static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
114 Function *&NewFn) {
115 // Check if the return type is a vector.
116 if (F->getReturnType()->isVectorTy())
117 return false;
118
119 rename(F);
120 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
121 return true;
122}
123
124// Upgrade the declaration of multiply and add bytes intrinsics whose input
125// arguments' types have changed from vectors of i32 to vectors of i8
126static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID,
127 Function *&NewFn) {
128 // check if input argument type is a vector of i8
129 Type *Arg1Type = F->getFunctionType()->getParamType(1);
130 Type *Arg2Type = F->getFunctionType()->getParamType(2);
131 if (Arg1Type->isVectorTy() &&
132 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(8) &&
133 Arg2Type->isVectorTy() &&
134 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(8))
135 return false;
136
137 rename(F);
138 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
139 return true;
140}
141
142// Upgrade the declaration of multipy and add words intrinsics whose input
143// arguments' types have changed to vectors of i32 to vectors of i16
144static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID,
145 Function *&NewFn) {
146 // check if input argument type is a vector of i16
147 Type *Arg1Type = F->getFunctionType()->getParamType(1);
148 Type *Arg2Type = F->getFunctionType()->getParamType(2);
149 if (Arg1Type->isVectorTy() &&
150 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(16) &&
151 Arg2Type->isVectorTy() &&
152 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(16))
153 return false;
154
155 rename(F);
156 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
157 return true;
158}
159
160static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
161 Function *&NewFn) {
162 if (F->getReturnType()->getScalarType()->isBFloatTy())
163 return false;
164
165 rename(F);
166 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
167 return true;
168}
169
170static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
171 Function *&NewFn) {
172 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
173 return false;
174
175 rename(F);
176 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
177 return true;
178}
179
180static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
181 // All of the intrinsics matches below should be marked with which llvm
182 // version started autoupgrading them. At some point in the future we would
183 // like to use this information to remove upgrade code for some older
184 // intrinsics. It is currently undecided how we will determine that future
185 // point.
186 if (Name.consume_front("avx."))
187 return (Name.starts_with("blend.p") || // Added in 3.7
188 Name == "cvt.ps2.pd.256" || // Added in 3.9
189 Name == "cvtdq2.pd.256" || // Added in 3.9
190 Name == "cvtdq2.ps.256" || // Added in 7.0
191 Name.starts_with("movnt.") || // Added in 3.2
192 Name.starts_with("sqrt.p") || // Added in 7.0
193 Name.starts_with("storeu.") || // Added in 3.9
194 Name.starts_with("vbroadcast.s") || // Added in 3.5
195 Name.starts_with("vbroadcastf128") || // Added in 4.0
196 Name.starts_with("vextractf128.") || // Added in 3.7
197 Name.starts_with("vinsertf128.") || // Added in 3.7
198 Name.starts_with("vperm2f128.") || // Added in 6.0
199 Name.starts_with("vpermil.")); // Added in 3.1
200
201 if (Name.consume_front("avx2."))
202 return (Name == "movntdqa" || // Added in 5.0
203 Name.starts_with("pabs.") || // Added in 6.0
204 Name.starts_with("padds.") || // Added in 8.0
205 Name.starts_with("paddus.") || // Added in 8.0
206 Name.starts_with("pblendd.") || // Added in 3.7
207 Name == "pblendw" || // Added in 3.7
208 Name.starts_with("pbroadcast") || // Added in 3.8
209 Name.starts_with("pcmpeq.") || // Added in 3.1
210 Name.starts_with("pcmpgt.") || // Added in 3.1
211 Name.starts_with("pmax") || // Added in 3.9
212 Name.starts_with("pmin") || // Added in 3.9
213 Name.starts_with("pmovsx") || // Added in 3.9
214 Name.starts_with("pmovzx") || // Added in 3.9
215 Name == "pmul.dq" || // Added in 7.0
216 Name == "pmulu.dq" || // Added in 7.0
217 Name.starts_with("psll.dq") || // Added in 3.7
218 Name.starts_with("psrl.dq") || // Added in 3.7
219 Name.starts_with("psubs.") || // Added in 8.0
220 Name.starts_with("psubus.") || // Added in 8.0
221 Name.starts_with("vbroadcast") || // Added in 3.8
222 Name == "vbroadcasti128" || // Added in 3.7
223 Name == "vextracti128" || // Added in 3.7
224 Name == "vinserti128" || // Added in 3.7
225 Name == "vperm2i128"); // Added in 6.0
226
227 if (Name.consume_front("avx512.")) {
228 if (Name.consume_front("mask."))
229 // 'avx512.mask.*'
230 return (Name.starts_with("add.p") || // Added in 7.0. 128/256 in 4.0
231 Name.starts_with("and.") || // Added in 3.9
232 Name.starts_with("andn.") || // Added in 3.9
233 Name.starts_with("broadcast.s") || // Added in 3.9
234 Name.starts_with("broadcastf32x4.") || // Added in 6.0
235 Name.starts_with("broadcastf32x8.") || // Added in 6.0
236 Name.starts_with("broadcastf64x2.") || // Added in 6.0
237 Name.starts_with("broadcastf64x4.") || // Added in 6.0
238 Name.starts_with("broadcasti32x4.") || // Added in 6.0
239 Name.starts_with("broadcasti32x8.") || // Added in 6.0
240 Name.starts_with("broadcasti64x2.") || // Added in 6.0
241 Name.starts_with("broadcasti64x4.") || // Added in 6.0
242 Name.starts_with("cmp.b") || // Added in 5.0
243 Name.starts_with("cmp.d") || // Added in 5.0
244 Name.starts_with("cmp.q") || // Added in 5.0
245 Name.starts_with("cmp.w") || // Added in 5.0
246 Name.starts_with("compress.b") || // Added in 9.0
247 Name.starts_with("compress.d") || // Added in 9.0
248 Name.starts_with("compress.p") || // Added in 9.0
249 Name.starts_with("compress.q") || // Added in 9.0
250 Name.starts_with("compress.store.") || // Added in 7.0
251 Name.starts_with("compress.w") || // Added in 9.0
252 Name.starts_with("conflict.") || // Added in 9.0
253 Name.starts_with("cvtdq2pd.") || // Added in 4.0
254 Name.starts_with("cvtdq2ps.") || // Added in 7.0 updated 9.0
255 Name == "cvtpd2dq.256" || // Added in 7.0
256 Name == "cvtpd2ps.256" || // Added in 7.0
257 Name == "cvtps2pd.128" || // Added in 7.0
258 Name == "cvtps2pd.256" || // Added in 7.0
259 Name.starts_with("cvtqq2pd.") || // Added in 7.0 updated 9.0
260 Name == "cvtqq2ps.256" || // Added in 9.0
261 Name == "cvtqq2ps.512" || // Added in 9.0
262 Name == "cvttpd2dq.256" || // Added in 7.0
263 Name == "cvttps2dq.128" || // Added in 7.0
264 Name == "cvttps2dq.256" || // Added in 7.0
265 Name.starts_with("cvtudq2pd.") || // Added in 4.0
266 Name.starts_with("cvtudq2ps.") || // Added in 7.0 updated 9.0
267 Name.starts_with("cvtuqq2pd.") || // Added in 7.0 updated 9.0
268 Name == "cvtuqq2ps.256" || // Added in 9.0
269 Name == "cvtuqq2ps.512" || // Added in 9.0
270 Name.starts_with("dbpsadbw.") || // Added in 7.0
271 Name.starts_with("div.p") || // Added in 7.0. 128/256 in 4.0
272 Name.starts_with("expand.b") || // Added in 9.0
273 Name.starts_with("expand.d") || // Added in 9.0
274 Name.starts_with("expand.load.") || // Added in 7.0
275 Name.starts_with("expand.p") || // Added in 9.0
276 Name.starts_with("expand.q") || // Added in 9.0
277 Name.starts_with("expand.w") || // Added in 9.0
278 Name.starts_with("fpclass.p") || // Added in 7.0
279 Name.starts_with("insert") || // Added in 4.0
280 Name.starts_with("load.") || // Added in 3.9
281 Name.starts_with("loadu.") || // Added in 3.9
282 Name.starts_with("lzcnt.") || // Added in 5.0
283 Name.starts_with("max.p") || // Added in 7.0. 128/256 in 5.0
284 Name.starts_with("min.p") || // Added in 7.0. 128/256 in 5.0
285 Name.starts_with("movddup") || // Added in 3.9
286 Name.starts_with("move.s") || // Added in 4.0
287 Name.starts_with("movshdup") || // Added in 3.9
288 Name.starts_with("movsldup") || // Added in 3.9
289 Name.starts_with("mul.p") || // Added in 7.0. 128/256 in 4.0
290 Name.starts_with("or.") || // Added in 3.9
291 Name.starts_with("pabs.") || // Added in 6.0
292 Name.starts_with("packssdw.") || // Added in 5.0
293 Name.starts_with("packsswb.") || // Added in 5.0
294 Name.starts_with("packusdw.") || // Added in 5.0
295 Name.starts_with("packuswb.") || // Added in 5.0
296 Name.starts_with("padd.") || // Added in 4.0
297 Name.starts_with("padds.") || // Added in 8.0
298 Name.starts_with("paddus.") || // Added in 8.0
299 Name.starts_with("palignr.") || // Added in 3.9
300 Name.starts_with("pand.") || // Added in 3.9
301 Name.starts_with("pandn.") || // Added in 3.9
302 Name.starts_with("pavg") || // Added in 6.0
303 Name.starts_with("pbroadcast") || // Added in 6.0
304 Name.starts_with("pcmpeq.") || // Added in 3.9
305 Name.starts_with("pcmpgt.") || // Added in 3.9
306 Name.starts_with("perm.df.") || // Added in 3.9
307 Name.starts_with("perm.di.") || // Added in 3.9
308 Name.starts_with("permvar.") || // Added in 7.0
309 Name.starts_with("pmaddubs.w.") || // Added in 7.0
310 Name.starts_with("pmaddw.d.") || // Added in 7.0
311 Name.starts_with("pmax") || // Added in 4.0
312 Name.starts_with("pmin") || // Added in 4.0
313 Name == "pmov.qd.256" || // Added in 9.0
314 Name == "pmov.qd.512" || // Added in 9.0
315 Name == "pmov.wb.256" || // Added in 9.0
316 Name == "pmov.wb.512" || // Added in 9.0
317 Name.starts_with("pmovsx") || // Added in 4.0
318 Name.starts_with("pmovzx") || // Added in 4.0
319 Name.starts_with("pmul.dq.") || // Added in 4.0
320 Name.starts_with("pmul.hr.sw.") || // Added in 7.0
321 Name.starts_with("pmulh.w.") || // Added in 7.0
322 Name.starts_with("pmulhu.w.") || // Added in 7.0
323 Name.starts_with("pmull.") || // Added in 4.0
324 Name.starts_with("pmultishift.qb.") || // Added in 8.0
325 Name.starts_with("pmulu.dq.") || // Added in 4.0
326 Name.starts_with("por.") || // Added in 3.9
327 Name.starts_with("prol.") || // Added in 8.0
328 Name.starts_with("prolv.") || // Added in 8.0
329 Name.starts_with("pror.") || // Added in 8.0
330 Name.starts_with("prorv.") || // Added in 8.0
331 Name.starts_with("pshuf.b.") || // Added in 4.0
332 Name.starts_with("pshuf.d.") || // Added in 3.9
333 Name.starts_with("pshufh.w.") || // Added in 3.9
334 Name.starts_with("pshufl.w.") || // Added in 3.9
335 Name.starts_with("psll.d") || // Added in 4.0
336 Name.starts_with("psll.q") || // Added in 4.0
337 Name.starts_with("psll.w") || // Added in 4.0
338 Name.starts_with("pslli") || // Added in 4.0
339 Name.starts_with("psllv") || // Added in 4.0
340 Name.starts_with("psra.d") || // Added in 4.0
341 Name.starts_with("psra.q") || // Added in 4.0
342 Name.starts_with("psra.w") || // Added in 4.0
343 Name.starts_with("psrai") || // Added in 4.0
344 Name.starts_with("psrav") || // Added in 4.0
345 Name.starts_with("psrl.d") || // Added in 4.0
346 Name.starts_with("psrl.q") || // Added in 4.0
347 Name.starts_with("psrl.w") || // Added in 4.0
348 Name.starts_with("psrli") || // Added in 4.0
349 Name.starts_with("psrlv") || // Added in 4.0
350 Name.starts_with("psub.") || // Added in 4.0
351 Name.starts_with("psubs.") || // Added in 8.0
352 Name.starts_with("psubus.") || // Added in 8.0
353 Name.starts_with("pternlog.") || // Added in 7.0
354 Name.starts_with("punpckh") || // Added in 3.9
355 Name.starts_with("punpckl") || // Added in 3.9
356 Name.starts_with("pxor.") || // Added in 3.9
357 Name.starts_with("shuf.f") || // Added in 6.0
358 Name.starts_with("shuf.i") || // Added in 6.0
359 Name.starts_with("shuf.p") || // Added in 4.0
360 Name.starts_with("sqrt.p") || // Added in 7.0
361 Name.starts_with("store.b.") || // Added in 3.9
362 Name.starts_with("store.d.") || // Added in 3.9
363 Name.starts_with("store.p") || // Added in 3.9
364 Name.starts_with("store.q.") || // Added in 3.9
365 Name.starts_with("store.w.") || // Added in 3.9
366 Name == "store.ss" || // Added in 7.0
367 Name.starts_with("storeu.") || // Added in 3.9
368 Name.starts_with("sub.p") || // Added in 7.0. 128/256 in 4.0
369 Name.starts_with("ucmp.") || // Added in 5.0
370 Name.starts_with("unpckh.") || // Added in 3.9
371 Name.starts_with("unpckl.") || // Added in 3.9
372 Name.starts_with("valign.") || // Added in 4.0
373 Name == "vcvtph2ps.128" || // Added in 11.0
374 Name == "vcvtph2ps.256" || // Added in 11.0
375 Name.starts_with("vextract") || // Added in 4.0
376 Name.starts_with("vfmadd.") || // Added in 7.0
377 Name.starts_with("vfmaddsub.") || // Added in 7.0
378 Name.starts_with("vfnmadd.") || // Added in 7.0
379 Name.starts_with("vfnmsub.") || // Added in 7.0
380 Name.starts_with("vpdpbusd.") || // Added in 7.0
381 Name.starts_with("vpdpbusds.") || // Added in 7.0
382 Name.starts_with("vpdpwssd.") || // Added in 7.0
383 Name.starts_with("vpdpwssds.") || // Added in 7.0
384 Name.starts_with("vpermi2var.") || // Added in 7.0
385 Name.starts_with("vpermil.p") || // Added in 3.9
386 Name.starts_with("vpermilvar.") || // Added in 4.0
387 Name.starts_with("vpermt2var.") || // Added in 7.0
388 Name.starts_with("vpmadd52") || // Added in 7.0
389 Name.starts_with("vpshld.") || // Added in 7.0
390 Name.starts_with("vpshldv.") || // Added in 8.0
391 Name.starts_with("vpshrd.") || // Added in 7.0
392 Name.starts_with("vpshrdv.") || // Added in 8.0
393 Name.starts_with("vpshufbitqmb.") || // Added in 8.0
394 Name.starts_with("xor.")); // Added in 3.9
395
396 if (Name.consume_front("mask3."))
397 // 'avx512.mask3.*'
398 return (Name.starts_with("vfmadd.") || // Added in 7.0
399 Name.starts_with("vfmaddsub.") || // Added in 7.0
400 Name.starts_with("vfmsub.") || // Added in 7.0
401 Name.starts_with("vfmsubadd.") || // Added in 7.0
402 Name.starts_with("vfnmsub.")); // Added in 7.0
403
404 if (Name.consume_front("maskz."))
405 // 'avx512.maskz.*'
406 return (Name.starts_with("pternlog.") || // Added in 7.0
407 Name.starts_with("vfmadd.") || // Added in 7.0
408 Name.starts_with("vfmaddsub.") || // Added in 7.0
409 Name.starts_with("vpdpbusd.") || // Added in 7.0
410 Name.starts_with("vpdpbusds.") || // Added in 7.0
411 Name.starts_with("vpdpwssd.") || // Added in 7.0
412 Name.starts_with("vpdpwssds.") || // Added in 7.0
413 Name.starts_with("vpermt2var.") || // Added in 7.0
414 Name.starts_with("vpmadd52") || // Added in 7.0
415 Name.starts_with("vpshldv.") || // Added in 8.0
416 Name.starts_with("vpshrdv.")); // Added in 8.0
417
418 // 'avx512.*'
419 return (Name == "movntdqa" || // Added in 5.0
420 Name == "pmul.dq.512" || // Added in 7.0
421 Name == "pmulu.dq.512" || // Added in 7.0
422 Name.starts_with("broadcastm") || // Added in 6.0
423 Name.starts_with("cmp.p") || // Added in 12.0
424 Name.starts_with("cvtb2mask.") || // Added in 7.0
425 Name.starts_with("cvtd2mask.") || // Added in 7.0
426 Name.starts_with("cvtmask2") || // Added in 5.0
427 Name.starts_with("cvtq2mask.") || // Added in 7.0
428 Name == "cvtusi2sd" || // Added in 7.0
429 Name.starts_with("cvtw2mask.") || // Added in 7.0
430 Name == "kand.w" || // Added in 7.0
431 Name == "kandn.w" || // Added in 7.0
432 Name == "knot.w" || // Added in 7.0
433 Name == "kor.w" || // Added in 7.0
434 Name == "kortestc.w" || // Added in 7.0
435 Name == "kortestz.w" || // Added in 7.0
436 Name.starts_with("kunpck") || // added in 6.0
437 Name == "kxnor.w" || // Added in 7.0
438 Name == "kxor.w" || // Added in 7.0
439 Name.starts_with("padds.") || // Added in 8.0
440 Name.starts_with("pbroadcast") || // Added in 3.9
441 Name.starts_with("prol") || // Added in 8.0
442 Name.starts_with("pror") || // Added in 8.0
443 Name.starts_with("psll.dq") || // Added in 3.9
444 Name.starts_with("psrl.dq") || // Added in 3.9
445 Name.starts_with("psubs.") || // Added in 8.0
446 Name.starts_with("ptestm") || // Added in 6.0
447 Name.starts_with("ptestnm") || // Added in 6.0
448 Name.starts_with("storent.") || // Added in 3.9
449 Name.starts_with("vbroadcast.s") || // Added in 7.0
450 Name.starts_with("vpshld.") || // Added in 8.0
451 Name.starts_with("vpshrd.")); // Added in 8.0
452 }
453
454 if (Name.consume_front("fma."))
455 return (Name.starts_with("vfmadd.") || // Added in 7.0
456 Name.starts_with("vfmsub.") || // Added in 7.0
457 Name.starts_with("vfmsubadd.") || // Added in 7.0
458 Name.starts_with("vfnmadd.") || // Added in 7.0
459 Name.starts_with("vfnmsub.")); // Added in 7.0
460
461 if (Name.consume_front("fma4."))
462 return Name.starts_with("vfmadd.s"); // Added in 7.0
463
464 if (Name.consume_front("sse."))
465 return (Name == "add.ss" || // Added in 4.0
466 Name == "cvtsi2ss" || // Added in 7.0
467 Name == "cvtsi642ss" || // Added in 7.0
468 Name == "div.ss" || // Added in 4.0
469 Name == "mul.ss" || // Added in 4.0
470 Name.starts_with("sqrt.p") || // Added in 7.0
471 Name == "sqrt.ss" || // Added in 7.0
472 Name.starts_with("storeu.") || // Added in 3.9
473 Name == "sub.ss"); // Added in 4.0
474
475 if (Name.consume_front("sse2."))
476 return (Name == "add.sd" || // Added in 4.0
477 Name == "cvtdq2pd" || // Added in 3.9
478 Name == "cvtdq2ps" || // Added in 7.0
479 Name == "cvtps2pd" || // Added in 3.9
480 Name == "cvtsi2sd" || // Added in 7.0
481 Name == "cvtsi642sd" || // Added in 7.0
482 Name == "cvtss2sd" || // Added in 7.0
483 Name == "div.sd" || // Added in 4.0
484 Name == "mul.sd" || // Added in 4.0
485 Name.starts_with("padds.") || // Added in 8.0
486 Name.starts_with("paddus.") || // Added in 8.0
487 Name.starts_with("pcmpeq.") || // Added in 3.1
488 Name.starts_with("pcmpgt.") || // Added in 3.1
489 Name == "pmaxs.w" || // Added in 3.9
490 Name == "pmaxu.b" || // Added in 3.9
491 Name == "pmins.w" || // Added in 3.9
492 Name == "pminu.b" || // Added in 3.9
493 Name == "pmulu.dq" || // Added in 7.0
494 Name.starts_with("pshuf") || // Added in 3.9
495 Name.starts_with("psll.dq") || // Added in 3.7
496 Name.starts_with("psrl.dq") || // Added in 3.7
497 Name.starts_with("psubs.") || // Added in 8.0
498 Name.starts_with("psubus.") || // Added in 8.0
499 Name.starts_with("sqrt.p") || // Added in 7.0
500 Name == "sqrt.sd" || // Added in 7.0
501 Name == "storel.dq" || // Added in 3.9
502 Name.starts_with("storeu.") || // Added in 3.9
503 Name == "sub.sd"); // Added in 4.0
504
505 if (Name.consume_front("sse41."))
506 return (Name.starts_with("blendp") || // Added in 3.7
507 Name == "movntdqa" || // Added in 5.0
508 Name == "pblendw" || // Added in 3.7
509 Name == "pmaxsb" || // Added in 3.9
510 Name == "pmaxsd" || // Added in 3.9
511 Name == "pmaxud" || // Added in 3.9
512 Name == "pmaxuw" || // Added in 3.9
513 Name == "pminsb" || // Added in 3.9
514 Name == "pminsd" || // Added in 3.9
515 Name == "pminud" || // Added in 3.9
516 Name == "pminuw" || // Added in 3.9
517 Name.starts_with("pmovsx") || // Added in 3.8
518 Name.starts_with("pmovzx") || // Added in 3.9
519 Name == "pmuldq"); // Added in 7.0
520
521 if (Name.consume_front("sse42."))
522 return Name == "crc32.64.8"; // Added in 3.4
523
524 if (Name.consume_front("sse4a."))
525 return Name.starts_with("movnt."); // Added in 3.9
526
527 if (Name.consume_front("ssse3."))
528 return (Name == "pabs.b.128" || // Added in 6.0
529 Name == "pabs.d.128" || // Added in 6.0
530 Name == "pabs.w.128"); // Added in 6.0
531
532 if (Name.consume_front("xop."))
533 return (Name == "vpcmov" || // Added in 3.8
534 Name == "vpcmov.256" || // Added in 5.0
535 Name.starts_with("vpcom") || // Added in 3.2, Updated in 9.0
536 Name.starts_with("vprot")); // Added in 8.0
537
538 if (Name.consume_front("bmi."))
539 return (Name.starts_with("pdep.") || // Added in 23.0
540 Name.starts_with("pext.")); // Added in 23.0
541
542 return (Name == "addcarry.u32" || // Added in 8.0
543 Name == "addcarry.u64" || // Added in 8.0
544 Name == "addcarryx.u32" || // Added in 8.0
545 Name == "addcarryx.u64" || // Added in 8.0
546 Name == "subborrow.u32" || // Added in 8.0
547 Name == "subborrow.u64" || // Added in 8.0
548 Name.starts_with("vcvtph2ps.")); // Added in 11.0
549}
550
551static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
552 Function *&NewFn) {
553 // Only handle intrinsics that start with "x86.".
554 if (!Name.consume_front("x86."))
555 return false;
556
557 if (shouldUpgradeX86Intrinsic(F, Name)) {
558 NewFn = nullptr;
559 return true;
560 }
561
562 if (Name == "rdtscp") { // Added in 8.0
563 // If this intrinsic has 0 operands, it's the new version.
564 if (F->getFunctionType()->getNumParams() == 0)
565 return false;
566
567 rename(F);
568 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
569 Intrinsic::x86_rdtscp);
570 return true;
571 }
572
573 Intrinsic::ID ID;
574
575 // SSE4.1 ptest functions may have an old signature.
576 if (Name.consume_front("sse41.ptest")) { // Added in 3.2
577 ID = StringSwitch<Intrinsic::ID>(Name)
578 .Case("c", Intrinsic::x86_sse41_ptestc)
579 .Case("z", Intrinsic::x86_sse41_ptestz)
580 .Case("nzc", Intrinsic::x86_sse41_ptestnzc)
581 .Default(Intrinsic::not_intrinsic);
582 if (ID != Intrinsic::not_intrinsic)
583 return upgradePTESTIntrinsic(F, ID, NewFn);
584
585 return false;
586 }
587
588 // Several blend and other instructions with masks used the wrong number of
589 // bits.
590
591 // Added in 3.6
592 ID = StringSwitch<Intrinsic::ID>(Name)
593 .Case("sse41.insertps", Intrinsic::x86_sse41_insertps)
594 .Case("sse41.dppd", Intrinsic::x86_sse41_dppd)
595 .Case("sse41.dpps", Intrinsic::x86_sse41_dpps)
596 .Case("sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
597 .Case("avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
598 .Case("avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
599 .Default(Intrinsic::not_intrinsic);
600 if (ID != Intrinsic::not_intrinsic)
601 return upgradeX86IntrinsicsWith8BitMask(F, ID, NewFn);
602
603 if (Name.consume_front("avx512.")) {
604 if (Name.consume_front("mask.cmp.")) {
605 // Added in 7.0
606 ID = StringSwitch<Intrinsic::ID>(Name)
607 .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
608 .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
609 .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
610 .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
611 .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
612 .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
613 .Default(Intrinsic::not_intrinsic);
614 if (ID != Intrinsic::not_intrinsic)
615 return upgradeX86MaskedFPCompare(F, ID, NewFn);
616 } else if (Name.starts_with("vpdpbusd.") ||
617 Name.starts_with("vpdpbusds.")) {
618 // Added in 21.1
619 ID = StringSwitch<Intrinsic::ID>(Name)
620 .Case("vpdpbusd.128", Intrinsic::x86_avx512_vpdpbusd_128)
621 .Case("vpdpbusd.256", Intrinsic::x86_avx512_vpdpbusd_256)
622 .Case("vpdpbusd.512", Intrinsic::x86_avx512_vpdpbusd_512)
623 .Case("vpdpbusds.128", Intrinsic::x86_avx512_vpdpbusds_128)
624 .Case("vpdpbusds.256", Intrinsic::x86_avx512_vpdpbusds_256)
625 .Case("vpdpbusds.512", Intrinsic::x86_avx512_vpdpbusds_512)
626 .Default(Intrinsic::not_intrinsic);
627 if (ID != Intrinsic::not_intrinsic)
628 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
629 } else if (Name.starts_with("vpdpwssd.") ||
630 Name.starts_with("vpdpwssds.")) {
631 // Added in 21.1
632 ID = StringSwitch<Intrinsic::ID>(Name)
633 .Case("vpdpwssd.128", Intrinsic::x86_avx512_vpdpwssd_128)
634 .Case("vpdpwssd.256", Intrinsic::x86_avx512_vpdpwssd_256)
635 .Case("vpdpwssd.512", Intrinsic::x86_avx512_vpdpwssd_512)
636 .Case("vpdpwssds.128", Intrinsic::x86_avx512_vpdpwssds_128)
637 .Case("vpdpwssds.256", Intrinsic::x86_avx512_vpdpwssds_256)
638 .Case("vpdpwssds.512", Intrinsic::x86_avx512_vpdpwssds_512)
639 .Default(Intrinsic::not_intrinsic);
640 if (ID != Intrinsic::not_intrinsic)
641 return upgradeX86MultiplyAddWords(F, ID, NewFn);
642 }
643 return false; // No other 'x86.avx512.*'.
644 }
645
646 if (Name.consume_front("avx2.")) {
647 if (Name.consume_front("vpdpb")) {
648 // Added in 21.1
649 ID = StringSwitch<Intrinsic::ID>(Name)
650 .Case("ssd.128", Intrinsic::x86_avx2_vpdpbssd_128)
651 .Case("ssd.256", Intrinsic::x86_avx2_vpdpbssd_256)
652 .Case("ssds.128", Intrinsic::x86_avx2_vpdpbssds_128)
653 .Case("ssds.256", Intrinsic::x86_avx2_vpdpbssds_256)
654 .Case("sud.128", Intrinsic::x86_avx2_vpdpbsud_128)
655 .Case("sud.256", Intrinsic::x86_avx2_vpdpbsud_256)
656 .Case("suds.128", Intrinsic::x86_avx2_vpdpbsuds_128)
657 .Case("suds.256", Intrinsic::x86_avx2_vpdpbsuds_256)
658 .Case("uud.128", Intrinsic::x86_avx2_vpdpbuud_128)
659 .Case("uud.256", Intrinsic::x86_avx2_vpdpbuud_256)
660 .Case("uuds.128", Intrinsic::x86_avx2_vpdpbuuds_128)
661 .Case("uuds.256", Intrinsic::x86_avx2_vpdpbuuds_256)
662 .Default(Intrinsic::not_intrinsic);
663 if (ID != Intrinsic::not_intrinsic)
664 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
665 } else if (Name.consume_front("vpdpw")) {
666 // Added in 21.1
667 ID = StringSwitch<Intrinsic::ID>(Name)
668 .Case("sud.128", Intrinsic::x86_avx2_vpdpwsud_128)
669 .Case("sud.256", Intrinsic::x86_avx2_vpdpwsud_256)
670 .Case("suds.128", Intrinsic::x86_avx2_vpdpwsuds_128)
671 .Case("suds.256", Intrinsic::x86_avx2_vpdpwsuds_256)
672 .Case("usd.128", Intrinsic::x86_avx2_vpdpwusd_128)
673 .Case("usd.256", Intrinsic::x86_avx2_vpdpwusd_256)
674 .Case("usds.128", Intrinsic::x86_avx2_vpdpwusds_128)
675 .Case("usds.256", Intrinsic::x86_avx2_vpdpwusds_256)
676 .Case("uud.128", Intrinsic::x86_avx2_vpdpwuud_128)
677 .Case("uud.256", Intrinsic::x86_avx2_vpdpwuud_256)
678 .Case("uuds.128", Intrinsic::x86_avx2_vpdpwuuds_128)
679 .Case("uuds.256", Intrinsic::x86_avx2_vpdpwuuds_256)
680 .Default(Intrinsic::not_intrinsic);
681 if (ID != Intrinsic::not_intrinsic)
682 return upgradeX86MultiplyAddWords(F, ID, NewFn);
683 }
684 return false; // No other 'x86.avx2.*'
685 }
686
687 if (Name.consume_front("avx10.")) {
688 if (Name.consume_front("vpdpb")) {
689 // Added in 21.1
690 ID = StringSwitch<Intrinsic::ID>(Name)
691 .Case("ssd.512", Intrinsic::x86_avx10_vpdpbssd_512)
692 .Case("ssds.512", Intrinsic::x86_avx10_vpdpbssds_512)
693 .Case("sud.512", Intrinsic::x86_avx10_vpdpbsud_512)
694 .Case("suds.512", Intrinsic::x86_avx10_vpdpbsuds_512)
695 .Case("uud.512", Intrinsic::x86_avx10_vpdpbuud_512)
696 .Case("uuds.512", Intrinsic::x86_avx10_vpdpbuuds_512)
697 .Default(Intrinsic::not_intrinsic);
698 if (ID != Intrinsic::not_intrinsic)
699 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
700 } else if (Name.consume_front("vpdpw")) {
701 ID = StringSwitch<Intrinsic::ID>(Name)
702 .Case("sud.512", Intrinsic::x86_avx10_vpdpwsud_512)
703 .Case("suds.512", Intrinsic::x86_avx10_vpdpwsuds_512)
704 .Case("usd.512", Intrinsic::x86_avx10_vpdpwusd_512)
705 .Case("usds.512", Intrinsic::x86_avx10_vpdpwusds_512)
706 .Case("uud.512", Intrinsic::x86_avx10_vpdpwuud_512)
707 .Case("uuds.512", Intrinsic::x86_avx10_vpdpwuuds_512)
708 .Default(Intrinsic::not_intrinsic);
709 if (ID != Intrinsic::not_intrinsic)
710 return upgradeX86MultiplyAddWords(F, ID, NewFn);
711 }
712 return false; // No other 'x86.avx10.*'
713 }
714
715 if (Name.consume_front("avx512bf16.")) {
716 // Added in 9.0
717 ID = StringSwitch<Intrinsic::ID>(Name)
718 .Case("cvtne2ps2bf16.128",
719 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
720 .Case("cvtne2ps2bf16.256",
721 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
722 .Case("cvtne2ps2bf16.512",
723 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
724 .Case("mask.cvtneps2bf16.128",
725 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
726 .Case("cvtneps2bf16.256",
727 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
728 .Case("cvtneps2bf16.512",
729 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
730 .Default(Intrinsic::not_intrinsic);
731 if (ID != Intrinsic::not_intrinsic)
732 return upgradeX86BF16Intrinsic(F, ID, NewFn);
733
734 // Added in 9.0
735 ID = StringSwitch<Intrinsic::ID>(Name)
736 .Case("dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
737 .Case("dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
738 .Case("dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
739 .Default(Intrinsic::not_intrinsic);
740 if (ID != Intrinsic::not_intrinsic)
741 return upgradeX86BF16DPIntrinsic(F, ID, NewFn);
742 return false; // No other 'x86.avx512bf16.*'.
743 }
744
745 if (Name.consume_front("xop.")) {
746 Intrinsic::ID ID = Intrinsic::not_intrinsic;
747 if (Name.starts_with("vpermil2")) { // Added in 3.9
748 // Upgrade any XOP PERMIL2 index operand still using a float/double
749 // vector.
750 auto Idx = F->getFunctionType()->getParamType(2);
751 if (Idx->isFPOrFPVectorTy()) {
752 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
753 unsigned EltSize = Idx->getScalarSizeInBits();
754 if (EltSize == 64 && IdxSize == 128)
755 ID = Intrinsic::x86_xop_vpermil2pd;
756 else if (EltSize == 32 && IdxSize == 128)
757 ID = Intrinsic::x86_xop_vpermil2ps;
758 else if (EltSize == 64 && IdxSize == 256)
759 ID = Intrinsic::x86_xop_vpermil2pd_256;
760 else
761 ID = Intrinsic::x86_xop_vpermil2ps_256;
762 }
763 } else if (F->arg_size() == 2)
764 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
765 ID = StringSwitch<Intrinsic::ID>(Name)
766 .Case("vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
767 .Case("vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
768 .Default(Intrinsic::not_intrinsic);
769
770 if (ID != Intrinsic::not_intrinsic) {
771 rename(F);
772 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
773 return true;
774 }
775 return false; // No other 'x86.xop.*'
776 }
777
778 if (Name == "seh.recoverfp") {
779 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
780 Intrinsic::eh_recoverfp);
781 return true;
782 }
783
784 return false;
785}
786
787// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
788// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
789static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
790 StringRef Name,
791 Function *&NewFn) {
792 if (Name.starts_with("rbit")) {
793 // '(arm|aarch64).rbit'.
794 NewFn = Intrinsic::getOrInsertDeclaration(
795 F->getParent(), Intrinsic::bitreverse, F->arg_begin()->getType());
796 return true;
797 }
798
799 if (Name == "thread.pointer") {
800 // '(arm|aarch64).thread.pointer'.
801 NewFn = Intrinsic::getOrInsertDeclaration(
802 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
803 return true;
804 }
805
806 bool Neon = Name.consume_front("neon.");
807 if (Neon) {
808 // '(arm|aarch64).neon.*'.
809 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
810 // v16i8 respectively.
811 if (Name.consume_front("bfdot.")) {
812 // (arm|aarch64).neon.bfdot.*'.
813 Intrinsic::ID ID =
814 StringSwitch<Intrinsic::ID>(Name)
815 .Cases({"v2f32.v8i8", "v4f32.v16i8"},
816 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
817 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
818 .Default(Intrinsic::not_intrinsic);
819 if (ID != Intrinsic::not_intrinsic) {
820 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
821 assert((OperandWidth == 64 || OperandWidth == 128) &&
822 "Unexpected operand width");
823 LLVMContext &Ctx = F->getParent()->getContext();
824 std::array<Type *, 2> Tys{
825 {F->getReturnType(),
826 FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)}};
827 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
828 return true;
829 }
830 return false; // No other '(arm|aarch64).neon.bfdot.*'.
831 }
832
833 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
834 // anymore and accept v8bf16 instead of v16i8.
835 if (Name.consume_front("bfm")) {
836 // (arm|aarch64).neon.bfm*'.
837 if (Name.consume_back(".v4f32.v16i8")) {
838 // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
839 Intrinsic::ID ID =
840 StringSwitch<Intrinsic::ID>(Name)
841 .Case("mla",
842 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
843 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
844 .Case("lalb",
845 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
846 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
847 .Case("lalt",
848 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
849 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
850 .Default(Intrinsic::not_intrinsic);
851 if (ID != Intrinsic::not_intrinsic) {
852 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
853 return true;
854 }
855 return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
856 }
857 return false; // No other '(arm|aarch64).neon.bfm*.
858 }
859 // Continue on to Aarch64 Neon or Arm Neon.
860 }
861 // Continue on to Arm or Aarch64.
862
863 if (IsArm) {
864 // 'arm.*'.
865 if (Neon) {
866 // 'arm.neon.*'.
867 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
868 .StartsWith("vclz.", Intrinsic::ctlz)
869 .StartsWith("vcnt.", Intrinsic::ctpop)
870 .StartsWith("vqadds.", Intrinsic::sadd_sat)
871 .StartsWith("vqaddu.", Intrinsic::uadd_sat)
872 .StartsWith("vqsubs.", Intrinsic::ssub_sat)
873 .StartsWith("vqsubu.", Intrinsic::usub_sat)
874 .StartsWith("vrinta.", Intrinsic::round)
875 .StartsWith("vrintn.", Intrinsic::roundeven)
876 .StartsWith("vrintm.", Intrinsic::floor)
877 .StartsWith("vrintp.", Intrinsic::ceil)
878 .StartsWith("vrintx.", Intrinsic::rint)
879 .StartsWith("vrintz.", Intrinsic::trunc)
880 .Default(Intrinsic::not_intrinsic);
881 if (ID != Intrinsic::not_intrinsic) {
882 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
883 F->arg_begin()->getType());
884 return true;
885 }
886
887 if (Name.consume_front("vst")) {
888 // 'arm.neon.vst*'.
889 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
890 SmallVector<StringRef, 2> Groups;
891 if (vstRegex.match(Name, &Groups)) {
892 static const Intrinsic::ID StoreInts[] = {
893 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
894 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
895
896 static const Intrinsic::ID StoreLaneInts[] = {
897 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
898 Intrinsic::arm_neon_vst4lane};
899
900 auto fArgs = F->getFunctionType()->params();
901 Type *Tys[] = {fArgs[0], fArgs[1]};
902 if (Groups[1].size() == 1)
903 NewFn = Intrinsic::getOrInsertDeclaration(
904 F->getParent(), StoreInts[fArgs.size() - 3], Tys);
905 else
906 NewFn = Intrinsic::getOrInsertDeclaration(
907 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
908 return true;
909 }
910 return false; // No other 'arm.neon.vst*'.
911 }
912
913 return false; // No other 'arm.neon.*'.
914 }
915
916 if (Name.consume_front("mve.")) {
917 // 'arm.mve.*'.
918 if (Name == "vctp64") {
919 if (cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
920 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
921 // the function and deal with it below in UpgradeIntrinsicCall.
922 rename(F);
923 return true;
924 }
925 return false; // Not 'arm.mve.vctp64'.
926 }
927
928 if (Name.starts_with("vrintn.v")) {
929 NewFn = Intrinsic::getOrInsertDeclaration(
930 F->getParent(), Intrinsic::roundeven, F->arg_begin()->getType());
931 return true;
932 }
933
934 // These too are changed to accept a v2i1 instead of the old v4i1.
935 if (Name.consume_back(".v4i1")) {
936 // 'arm.mve.*.v4i1'.
937 if (Name.consume_back(".predicated.v2i64.v4i32"))
938 // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
939 return Name == "mull.int" || Name == "vqdmull";
940
941 if (Name.consume_back(".v2i64")) {
942 // 'arm.mve.*.v2i64.v4i1'
943 bool IsGather = Name.consume_front("vldr.gather.");
944 if (IsGather || Name.consume_front("vstr.scatter.")) {
945 if (Name.consume_front("base.")) {
946 // Optional 'wb.' prefix.
947 Name.consume_front("wb.");
948 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
949 // predicated.v2i64.v2i64.v4i1'.
950 return Name == "predicated.v2i64";
951 }
952
953 if (Name.consume_front("offset.predicated."))
954 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
955 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
956
957 // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
958 return false;
959 }
960
961 return false; // No other 'arm.mve.*.v2i64.v4i1'.
962 }
963 return false; // No other 'arm.mve.*.v4i1'.
964 }
965 return false; // No other 'arm.mve.*'.
966 }
967
968 if (Name.consume_front("cde.vcx")) {
969 // 'arm.cde.vcx*'.
970 if (Name.consume_back(".predicated.v2i64.v4i1"))
971 // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
972 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
973 Name == "3q" || Name == "3qa";
974
975 return false; // No other 'arm.cde.vcx*'.
976 }
977 } else {
978 // 'aarch64.*'.
979 if (Neon) {
980 // 'aarch64.neon.*'.
981 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
982 .StartsWith("frintn", Intrinsic::roundeven)
983 .StartsWith("rbit", Intrinsic::bitreverse)
984 .Default(Intrinsic::not_intrinsic);
985 if (ID != Intrinsic::not_intrinsic) {
986 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
987 F->arg_begin()->getType());
988 return true;
989 }
990
991 if (Name.starts_with("addp")) {
992 // 'aarch64.neon.addp*'.
993 if (F->arg_size() != 2)
994 return false; // Invalid IR.
995 VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
996 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
997 NewFn = Intrinsic::getOrInsertDeclaration(
998 F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
999 return true;
1000 }
1001 }
1002
1003 // Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
1004 if (Name.starts_with("bfcvt")) {
1005 NewFn = nullptr;
1006 return true;
1007 }
1008
1009 // vcvtfp2hf and vcvthf2fp -> fpext and fptrunc
1010 if (Name == "vcvtfp2hf" || Name == "vcvthf2fp") {
1011 NewFn = nullptr;
1012 return true;
1013 }
1014
1015 return false; // No other 'aarch64.neon.*'.
1016 }
1017 if (Name.consume_front("sve.")) {
1018 // 'aarch64.sve.*'.
1019 if (Name.consume_front("bf")) {
1020 if (Name == "mmla") {
1021 Type *Tys[] = {F->getReturnType(),
1022 std::next(F->arg_begin())->getType()};
1023 NewFn = Intrinsic::getOrInsertDeclaration(
1024 F->getParent(), Intrinsic::aarch64_sve_fmmla, Tys);
1025 return true;
1026 }
1027 if (Name.consume_back(".lane")) {
1028 // 'aarch64.sve.bf*.lane'.
1029 Intrinsic::ID ID =
1030 StringSwitch<Intrinsic::ID>(Name)
1031 .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
1032 .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1033 .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1034 .Default(Intrinsic::not_intrinsic);
1035 if (ID != Intrinsic::not_intrinsic) {
1036 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1037 return true;
1038 }
1039 return false; // No other 'aarch64.sve.bf*.lane'.
1040 }
1041 return false; // No other 'aarch64.sve.bf*'.
1042 }
1043
1044 // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
1045 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
1046 NewFn = nullptr;
1047 return true;
1048 }
1049
1050 if (Name.consume_front("convert.from.svbool")) {
1051 // 'aarch64.sve.convert.from.svbool'
1052 auto *TTy = dyn_cast<TargetExtType>(F->getReturnType());
1053 if (!TTy || TTy->getName() != "aarch64.svcount")
1054 return false;
1055
1056 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_to_svcount;
1057 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1058 return true;
1059 }
1060
1061 if (Name.consume_front("convert.to.svbool")) {
1062 // 'aarch64.sve.convert.to.svbool'
1063 auto *TTy = dyn_cast<TargetExtType>(F->arg_begin()->getType());
1064 if (!TTy || TTy->getName() != "aarch64.svcount")
1065 return false;
1066
1067 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_from_svcount;
1068 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1069 return true;
1070 }
1071
1072 if (Name.consume_front("addqv")) {
1073 // 'aarch64.sve.addqv'.
1074 if (!F->getReturnType()->isFPOrFPVectorTy())
1075 return false;
1076
1077 auto Args = F->getFunctionType()->params();
1078 Type *Tys[] = {F->getReturnType(), Args[1]};
1079 NewFn = Intrinsic::getOrInsertDeclaration(
1080 F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);
1081 return true;
1082 }
1083
1084 if (Name.consume_front("ld")) {
1085 // 'aarch64.sve.ld*'.
1086 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
1087 if (LdRegex.match(Name)) {
1088 Type *ScalarTy =
1089 cast<VectorType>(F->getReturnType())->getElementType();
1090 ElementCount EC =
1091 cast<VectorType>(F->arg_begin()->getType())->getElementCount();
1092 assert(F->arg_size() == 2 &&
1093 "Expected 2 arguments for ld* intrinsic.");
1094 Type *PtrTy = F->getArg(1)->getType();
1095 Type *Ty = VectorType::get(ScalarTy, EC);
1096 static const Intrinsic::ID LoadIDs[] = {
1097 Intrinsic::aarch64_sve_ld2_sret,
1098 Intrinsic::aarch64_sve_ld3_sret,
1099 Intrinsic::aarch64_sve_ld4_sret,
1100 };
1101 NewFn = Intrinsic::getOrInsertDeclaration(
1102 F->getParent(), LoadIDs[Name[0] - '2'], {Ty, PtrTy});
1103 return true;
1104 }
1105 return false; // No other 'aarch64.sve.ld*'.
1106 }
1107
1108 if (Name.consume_front("tuple.")) {
1109 // 'aarch64.sve.tuple.*'.
1110 if (Name.starts_with("get")) {
1111 // 'aarch64.sve.tuple.get*'.
1112 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1113 NewFn = Intrinsic::getOrInsertDeclaration(
1114 F->getParent(), Intrinsic::vector_extract, Tys);
1115 return true;
1116 }
1117
1118 if (Name.starts_with("set")) {
1119 // 'aarch64.sve.tuple.set*'.
1120 auto Args = F->getFunctionType()->params();
1121 Type *Tys[] = {Args[0], Args[2], Args[1]};
1122 NewFn = Intrinsic::getOrInsertDeclaration(
1123 F->getParent(), Intrinsic::vector_insert, Tys);
1124 return true;
1125 }
1126
1127 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
1128 if (CreateTupleRegex.match(Name)) {
1129 // 'aarch64.sve.tuple.create*'.
1130 auto Args = F->getFunctionType()->params();
1131 Type *Tys[] = {F->getReturnType(), Args[1]};
1132 NewFn = Intrinsic::getOrInsertDeclaration(
1133 F->getParent(), Intrinsic::vector_insert, Tys);
1134 return true;
1135 }
1136 return false; // No other 'aarch64.sve.tuple.*'.
1137 }
1138
1139 if (Name.starts_with("rev.nxv")) {
1140 // 'aarch64.sve.rev.<Ty>'
1141 NewFn = Intrinsic::getOrInsertDeclaration(
1142 F->getParent(), Intrinsic::vector_reverse, F->getReturnType());
1143 return true;
1144 }
1145
1146 return false; // No other 'aarch64.sve.*'.
1147 }
1148 if (Name.consume_front("sme.")) {
1149 // 'aarch64.sme.*'.
1150 if (Name.consume_front("ftmopa.")) {
1151 // The FP8 FTMOPA intrinsics were split out from the non-FP8 FTMOPA
1152 // intrinsics to model their FPMR dependency.
1153 Intrinsic::ID ID =
1154 StringSwitch<Intrinsic::ID>(Name)
1155 .Case("za16.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1156 .Case("za32.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1157 .Default(Intrinsic::not_intrinsic);
1158 if (ID != Intrinsic::not_intrinsic) {
1159 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1160 return true;
1161 }
1162 return false; // No other 'aarch64.sme.ftmopa.*'.
1163 }
1164
1165 return false; // No other 'aarch64.sme.*'.
1166 }
1167 }
1168 return false; // No other 'arm.*', 'aarch64.*'.
1169}
1170
1171// The TMA G2S (global-to-shared) tensor copy modes that have legacy
1172// declarations requiring an auto-upgrade. The same set applies to the
1173// cluster (g2s) and CTA (g2s_cta) variants.
1174#define NVVM_TMA_G2S_MODES(M) \
1175 M(tile_1d, "tile.1d") \
1176 M(tile_2d, "tile.2d") \
1177 M(tile_3d, "tile.3d") \
1178 M(tile_4d, "tile.4d") \
1179 M(tile_5d, "tile.5d") \
1180 M(tile_gather4_2d, "tile.gather4.2d") \
1181 M(im2col_3d, "im2col.3d") \
1182 M(im2col_4d, "im2col.4d") \
1183 M(im2col_5d, "im2col.5d") \
1184 M(im2col_w_3d, "im2col.w.3d") \
1185 M(im2col_w_4d, "im2col.w.4d") \
1186 M(im2col_w_5d, "im2col.w.5d") \
1187 M(im2col_w_128_3d, "im2col.w.128.3d") \
1188 M(im2col_w_128_4d, "im2col.w.128.4d") \
1189 M(im2col_w_128_5d, "im2col.w.128.5d")
1190
1191// Two legacy tails are:
1192//
1193// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch
1194// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group
1195//
1196// The current tail appends a trailing i32 %validate_pattern, so both
1197// legacy tails are recognized by an i1 at parameter N-2.
1198static Intrinsic::ID
1199shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name,
1200 SmallVectorImpl<Type *> &OvlTys) {
1201 if (!Name.consume_front("cp.async.bulk.tensor.g2s."))
1202 return Intrinsic::not_intrinsic;
1203
1204#define G2S_ID(ID_SUFFIX, NAME) \
1205 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX)
1206 // clang-format off
1207 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1208 NVVM_TMA_G2S_MODES(G2S_ID)
1209 .Default(Intrinsic::not_intrinsic);
1210#undef G2S_ID
1211 // clang-format on
1212 if (ID == Intrinsic::not_intrinsic)
1213 return ID;
1214
1215 size_t NumParams = F->getFunctionType()->getNumParams();
1216
1217 // Parameter N-2 is i1 for both legacy tails; the current tail ends
1218 // with i32 %cta_group, i32 %validate_pattern, for which N-2 is i32.
1219 if (!F->getFunctionType()->getParamType(NumParams - 2)->isIntegerTy(1))
1220 return Intrinsic::not_intrinsic;
1221
1222 // The multicast mask is the parameter immediately before the i64
1223 // cache-hint: N-4 for the 2-flag tail, N-5 for the 3-flag tail.
1224 ArrayRef<Type *> Params = F->getFunctionType()->params();
1225 size_t MaskIdx =
1226 Params[NumParams - 1]->isIntegerTy(1) ? NumParams - 4 : NumParams - 5;
1227 assert(Params[MaskIdx + 1]->isIntegerTy(64) &&
1228 "expected the i64 cache-hint after the multicast mask");
1229 Type *MaskTy = Params[MaskIdx];
1230 assert(MaskTy->isIntegerTy(16) && "unexpected multicast mask type");
1231 OvlTys.push_back(MaskTy);
1232
1233 return ID;
1234}
1235
1236// The legacy tail is:
1237//
1238// arg1, arg2, .. i64 %ch, i1 %flag_ch
1239//
1240// The current tail appends a trailing i32 %validate_pattern, so the
1241// legacy tail is recognized by an i1 at parameter N-1.
1242static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F,
1243 StringRef Name) {
1244 if (!Name.consume_front("cp.async.bulk.tensor.g2s.cta."))
1245 return Intrinsic::not_intrinsic;
1246
1247#define G2S_CTA_ID(ID_SUFFIX, NAME) \
1248 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX)
1249 // clang-format off
1250 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1251 NVVM_TMA_G2S_MODES(G2S_CTA_ID)
1252 .Default(Intrinsic::not_intrinsic);
1253#undef G2S_CTA_ID
1254 // clang-format on
1255 if (ID == Intrinsic::not_intrinsic)
1256 return ID;
1257
1258 // Parameter N-1 is i1 for the legacy tail; the current tail ends
1259 // with i32 %validate_pattern, for which N-1 is i32.
1260 if (!F->getFunctionType()
1261 ->getParamType(F->getFunctionType()->getNumParams() - 1)
1262 ->isIntegerTy(1))
1263 return Intrinsic::not_intrinsic;
1264
1265 return ID;
1266}
1267// The legacy TMA reduction intrinsics encode the reduction operator in their
1268// name, while the current ones take it as an immediate argument. Map the
1269// operator part of a legacy name to the corresponding immediate value.
1270static std::optional<unsigned> getNVPTXTMAReductionOp(StringRef Name) {
1271 return StringSwitch<std::optional<unsigned>>(Name)
1272 .Case("add", static_cast<unsigned>(nvvm::TMAReductionOp::ADD))
1273 .Case("min", static_cast<unsigned>(nvvm::TMAReductionOp::MIN))
1274 .Case("max", static_cast<unsigned>(nvvm::TMAReductionOp::MAX))
1275 .Case("inc", static_cast<unsigned>(nvvm::TMAReductionOp::INC))
1276 .Case("dec", static_cast<unsigned>(nvvm::TMAReductionOp::DEC))
1277 .Case("and", static_cast<unsigned>(nvvm::TMAReductionOp::AND))
1278 .Case("or", static_cast<unsigned>(nvvm::TMAReductionOp::OR))
1279 .Case("xor", static_cast<unsigned>(nvvm::TMAReductionOp::XOR))
1280 .Default(std::nullopt);
1281}
1282
1283static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name) {
1284 if (!Name.consume_front("cp.async.bulk.tensor.reduce."))
1285 return Intrinsic::not_intrinsic;
1286
1287 auto [RedOpName, ShapeName] = Name.split('.');
1288 if (!getNVPTXTMAReductionOp(RedOpName))
1289 return Intrinsic::not_intrinsic;
1290
1291 return StringSwitch<Intrinsic::ID>(ShapeName)
1292 .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d)
1293 .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d)
1294 .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d)
1295 .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d)
1296 .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d)
1297 .Case("im2col.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d)
1298 .Case("im2col.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d)
1299 .Case("im2col.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d)
1300 .Default(Intrinsic::not_intrinsic);
1301}
1302
1303static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
1304 StringRef Name) {
1305 if (Name.consume_front("mapa.shared.cluster"))
1306 if (F->getReturnType()->getPointerAddressSpace() ==
1307 NVPTXAS::ADDRESS_SPACE_SHARED)
1308 return Intrinsic::nvvm_mapa_shared_cluster;
1309
1310 if (Name.consume_front("cp.async.bulk.")) {
1311 Intrinsic::ID ID =
1312 StringSwitch<Intrinsic::ID>(Name)
1313 .Case("global.to.shared.cluster",
1314 Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster)
1315 .Case("shared.cta.to.cluster",
1316 Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1317 .Default(Intrinsic::not_intrinsic);
1318
1319 if (ID != Intrinsic::not_intrinsic)
1320 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1321 NVPTXAS::ADDRESS_SPACE_SHARED)
1322 return ID;
1323 }
1324
1325 return Intrinsic::not_intrinsic;
1326}
1327
1328static Intrinsic::ID
1329shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name) {
1330 if (!Name.consume_front("tcgen05.commit."))
1331 return Intrinsic::not_intrinsic;
1332
1333 if (Name.consume_front("shared."))
1334 return StringSwitch<Intrinsic::ID>(Name)
1335 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_cg1)
1336 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_cg2)
1337 .Default(Intrinsic::not_intrinsic);
1338
1339 if (Name.consume_front("mc.shared.")) {
1340 // Only upgrade older i16 mc variants.
1341 if (!F->getArg(1)->getType()->isIntegerTy(16))
1342 return Intrinsic::not_intrinsic;
1343
1344 return StringSwitch<Intrinsic::ID>(Name)
1345 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_mc_cg1)
1346 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_mc_cg2)
1347 .Default(Intrinsic::not_intrinsic);
1348 }
1349
1350 return Intrinsic::not_intrinsic;
1351}
1352
1353static Intrinsic::ID
1354shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name) {
1355 if (F->arg_size() != 2)
1356 return Intrinsic::not_intrinsic;
1357
1358 if (Name.consume_front("tcgen05.alloc.shared.") ||
1359 Name.consume_front("tcgen05.alloc."))
1360 return StringSwitch<Intrinsic::ID>(Name)
1361 .Case("cg1", Intrinsic::nvvm_tcgen05_alloc_cg1)
1362 .Case("cg2", Intrinsic::nvvm_tcgen05_alloc_cg2)
1363 .Default(Intrinsic::not_intrinsic);
1364
1365 if (Name.consume_front("tcgen05.dealloc."))
1366 return StringSwitch<Intrinsic::ID>(Name)
1367 .Case("cg1", Intrinsic::nvvm_tcgen05_dealloc_cg1)
1368 .Case("cg2", Intrinsic::nvvm_tcgen05_dealloc_cg2)
1369 .Default(Intrinsic::not_intrinsic);
1370
1371 return Intrinsic::not_intrinsic;
1372}
1373
1374static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
1375 if (Name.consume_front("fma.rn."))
1376 return StringSwitch<Intrinsic::ID>(Name)
1377 .Case("bf16", Intrinsic::nvvm_fma_rn_bf16)
1378 .Case("bf16x2", Intrinsic::nvvm_fma_rn_bf16x2)
1379 .Case("relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16)
1380 .Case("relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2)
1381 .Default(Intrinsic::not_intrinsic);
1382
1383 if (Name.consume_front("fmax."))
1384 return StringSwitch<Intrinsic::ID>(Name)
1385 .Case("bf16", Intrinsic::nvvm_fmax_bf16)
1386 .Case("bf16x2", Intrinsic::nvvm_fmax_bf16x2)
1387 .Case("ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
1388 .Case("ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
1389 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
1390 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1391 .Case("ftz.nan.xorsign.abs.bf16",
1392 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1393 .Case("ftz.nan.xorsign.abs.bf16x2",
1394 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1395 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1396 .Case("ftz.xorsign.abs.bf16x2",
1397 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1398 .Case("nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
1399 .Case("nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
1400 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1401 .Case("nan.xorsign.abs.bf16x2",
1402 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1403 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1404 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1405 .Default(Intrinsic::not_intrinsic);
1406
1407 if (Name.consume_front("fmin."))
1408 return StringSwitch<Intrinsic::ID>(Name)
1409 .Case("bf16", Intrinsic::nvvm_fmin_bf16)
1410 .Case("bf16x2", Intrinsic::nvvm_fmin_bf16x2)
1411 .Case("ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
1412 .Case("ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
1413 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
1414 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1415 .Case("ftz.nan.xorsign.abs.bf16",
1416 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1417 .Case("ftz.nan.xorsign.abs.bf16x2",
1418 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1419 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1420 .Case("ftz.xorsign.abs.bf16x2",
1421 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1422 .Case("nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
1423 .Case("nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
1424 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1425 .Case("nan.xorsign.abs.bf16x2",
1426 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1427 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1428 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1429 .Default(Intrinsic::not_intrinsic);
1430
1431 if (Name.consume_front("neg."))
1432 return StringSwitch<Intrinsic::ID>(Name)
1433 .Case("bf16", Intrinsic::nvvm_neg_bf16)
1434 .Case("bf16x2", Intrinsic::nvvm_neg_bf16x2)
1435 .Default(Intrinsic::not_intrinsic);
1436
1437 return Intrinsic::not_intrinsic;
1438}
1439
1440static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID) {
1441 FunctionType *NewFnTy = Intrinsic::getType(F->getContext(), IID);
1442 FunctionType *OldFnTy = F->getFunctionType();
1443 auto IsOldBF16StorageTy = [](Type *OldTy, Type *NewTy) {
1444 return OldTy->getScalarType()->isIntegerTy() &&
1445 OldTy->getPrimitiveSizeInBits() == NewTy->getPrimitiveSizeInBits();
1446 };
1447
1448 if (!IsOldBF16StorageTy(OldFnTy->getReturnType(), NewFnTy->getReturnType()))
1449 return false;
1450
1451 if (OldFnTy->getNumParams() != NewFnTy->getNumParams())
1452 return false;
1453
1454 for (unsigned I = 0, E = OldFnTy->getNumParams(); I != E; ++I)
1455 if (!IsOldBF16StorageTy(OldFnTy->getParamType(I), NewFnTy->getParamType(I)))
1456 return false;
1457
1458 return true;
1459}
1460
1461static Intrinsic::ID shouldUpgradeNVPTXTcgen05MMAIntrinsic(Function *F,
1462 StringRef Name) {
1463 if (!Name.consume_front("tcgen05.mma."))
1464 return Intrinsic::not_intrinsic;
1465
1466 // tcgen05.mma.ws.* variants do not need collector-b appended.
1467 if (Name.starts_with("ws"))
1468 return Intrinsic::not_intrinsic;
1469
1470 return F->getIntrinsicID();
1471}
1472
1473static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
1474getNVVMFAddUpgrade(StringRef Name) {
1475 auto [Modifiers, Type] = Name.rsplit('.');
1476 if (!is_contained({"f", "d", "f16", "v2f16"}, Type))
1477 return std::nullopt;
1478
1479 std::optional<llvm::RoundingMode> RoundingMode =
1480 StringSwitch<std::optional<llvm::RoundingMode>>(Modifiers.take_front(2))
1481 .Case("rn", llvm::RoundingMode::NearestTiesToEven)
1482 .Case("rz", llvm::RoundingMode::TowardZero)
1483 .Case("rm", llvm::RoundingMode::TowardNegative)
1484 .Case("rp", llvm::RoundingMode::TowardPositive)
1485 .Default(std::nullopt);
1486 if (!RoundingMode)
1487 return std::nullopt;
1488
1489 Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Modifiers.drop_front(2))
1490 .Case("", Intrinsic::nvvm_fadd)
1491 .Case(".ftz", Intrinsic::nvvm_fadd_ftz)
1492 .Case(".sat", Intrinsic::nvvm_fadd_sat)
1493 .Case(".ftz.sat", Intrinsic::nvvm_fadd_ftz_sat)
1494 .Default(Intrinsic::not_intrinsic);
1495 if (IID == Intrinsic::not_intrinsic)
1496 return std::nullopt;
1497
1498 return std::make_pair(IID, *RoundingMode);
1499}
1500
1501static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
1502 return Name.consume_front("local") || Name.consume_front("shared") ||
1503 Name.consume_front("global") || Name.consume_front("constant") ||
1504 Name.consume_front("param");
1505}
1506
1507static unsigned getFunctionalOpcodeForVP(StringRef Name) {
1508 if (!Name.consume_front("vp."))
1509 return 0;
1510 return StringSwitch<unsigned>(Name)
1511 .StartsWith("select", Instruction::Select)
1512 .StartsWith("add", Instruction::Add)
1513 .StartsWith("sub", Instruction::Sub)
1514 .StartsWith("mul", Instruction::Mul)
1515 .StartsWith("ashr", Instruction::AShr)
1516 .StartsWith("lshr", Instruction::LShr)
1517 .StartsWith("shl", Instruction::Shl)
1518 .StartsWith("or", Instruction::Or)
1519 .StartsWith("and", Instruction::And)
1520 .StartsWith("xor", Instruction::Xor)
1521 .StartsWith("fadd", Instruction::FAdd)
1522 .StartsWith("fsub", Instruction::FSub)
1523 .StartsWith("fmuladd", 0)
1524 .StartsWith("fmul", Instruction::FMul)
1525 .StartsWith("fdiv", Instruction::FDiv)
1526 .StartsWith("frem", Instruction::FRem)
1527 .StartsWith("fneg", Instruction::FNeg)
1528 .StartsWith("trunc", Instruction::Trunc)
1529 .StartsWith("zext", Instruction::ZExt)
1530 .StartsWith("sext", Instruction::SExt)
1531 .StartsWith("fptrunc", Instruction::FPTrunc)
1532 .StartsWith("fpext", Instruction::FPExt)
1533 .StartsWith("fptoui", Instruction::FPToUI)
1534 .StartsWith("fptosi", Instruction::FPToSI)
1535 .StartsWith("uitofp", Instruction::UIToFP)
1536 .StartsWith("sitofp", Instruction::SIToFP)
1537 .StartsWith("ptrtoint", Instruction::PtrToInt)
1538 .StartsWith("inttoptr", Instruction::IntToPtr)
1539 .StartsWith("icmp", Instruction::ICmp)
1540 .StartsWith("fcmp", Instruction::FCmp)
1541 .Default(0);
1542}
1543
1544static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name) {
1545 if (!Name.consume_front("vp."))
1546 return 0;
1547 return StringSwitch<Intrinsic::ID>(Name)
1548 .StartsWith("abs", Intrinsic::abs)
1549 .StartsWith("smax", Intrinsic::smax)
1550 .StartsWith("smin", Intrinsic::smin)
1551 .StartsWith("umax", Intrinsic::umax)
1552 .StartsWith("umin", Intrinsic::umin)
1553 .StartsWith("copysign", Intrinsic::copysign)
1554 .StartsWith("minnum", Intrinsic::minnum)
1555 .StartsWith("maxnum", Intrinsic::maxnum)
1556 .StartsWith("minimum", Intrinsic::minimum)
1557 .StartsWith("maximum", Intrinsic::maximum)
1558 .StartsWith("fabs", Intrinsic::fabs)
1559 .StartsWith("sqrt", Intrinsic::sqrt)
1560 .StartsWith("fma", Intrinsic::fma)
1561 .StartsWith("fmuladd", Intrinsic::fmuladd)
1562 .StartsWith("ceil", Intrinsic::ceil)
1563 .StartsWith("floor", Intrinsic::floor)
1564 .StartsWith("rint", Intrinsic::rint)
1565 .StartsWith("nearbyint", Intrinsic::nearbyint)
1566 .StartsWith("roundeven", Intrinsic::roundeven)
1567 .StartsWith("roundtozero", Intrinsic::trunc)
1568 .StartsWith("round", Intrinsic::round)
1569 .StartsWith("lrint", Intrinsic::lrint)
1570 .StartsWith("llrint", Intrinsic::llrint)
1571 .StartsWith("bitreverse", Intrinsic::bitreverse)
1572 .StartsWith("bswap", Intrinsic::bswap)
1573 .StartsWith("ctpop", Intrinsic::ctpop)
1574 .StartsWith("ctlz", Intrinsic::ctlz)
1575 .StartsWith("cttz.elts", 0)
1576 .StartsWith("cttz", Intrinsic::cttz)
1577 .StartsWith("sadd.sat", Intrinsic::sadd_sat)
1578 .StartsWith("uadd.sat", Intrinsic::uadd_sat)
1579 .StartsWith("ssub.sat", Intrinsic::ssub_sat)
1580 .StartsWith("usub.sat", Intrinsic::usub_sat)
1581 .StartsWith("fshl", Intrinsic::fshl)
1582 .StartsWith("fshr", Intrinsic::fshr)
1583 .StartsWith("is.fpclass", Intrinsic::is_fpclass)
1584 .Default(0);
1585}
1586
1590
1591static bool convertIntrinsicValidType(StringRef Name,
1592 const FunctionType *FuncTy) {
1593 Type *HalfTy = Type::getHalfTy(FuncTy->getContext());
1594 if (Name.starts_with("to.fp16")) {
1595 return CastInst::castIsValid(Instruction::FPTrunc, FuncTy->getParamType(0),
1596 HalfTy) &&
1597 CastInst::castIsValid(Instruction::BitCast, HalfTy,
1598 FuncTy->getReturnType());
1599 }
1600
1601 if (Name.starts_with("from.fp16")) {
1602 return CastInst::castIsValid(Instruction::BitCast, FuncTy->getParamType(0),
1603 HalfTy) &&
1604 CastInst::castIsValid(Instruction::FPExt, HalfTy,
1605 FuncTy->getReturnType());
1606 }
1607
1608 return false;
1609}
1610
1611static bool upgradeIntrinsicDeclWithDefaultArgs(Function *F, Function *&NewFn) {
1612 Intrinsic::ID IID = Intrinsic::lookupIntrinsicID(F->getName());
1613 if (IID == Intrinsic::not_intrinsic)
1614 return false;
1615
1616 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
1617 if (Defaults.empty())
1618 return false;
1619
1620 // Overloaded intrinsics are out of scope for the default-arg feature
1621 // and will be supported in a follow-up.
1622 if (Intrinsic::isOverloaded(IID))
1623 return false;
1624
1625 // Get the canonical full declaration for this intrinsic.
1626 Function *FullDecl = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1627
1628 // If the existing declaration already has all args, nothing to upgrade
1629 if (F->arg_size() >= FullDecl->arg_size())
1630 return false;
1631
1632 // Defaults are a contiguous trailing block, so checking the first missing
1633 // argument is enough.
1634 if (F->arg_size() < FirstDefault)
1635 return false;
1636
1637 NewFn = FullDecl;
1638 return true;
1639}
1640
1641static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
1642 bool CanUpgradeDebugIntrinsicsToRecords) {
1643 assert(F && "Illegal to upgrade a non-existent Function.");
1644
1645 StringRef Name = F->getName();
1646
1647 // Quickly eliminate it, if it's not a candidate.
1648 if (!Name.consume_front("llvm.") || Name.empty())
1649 return false;
1650
1651 switch (Name[0]) {
1652 default: break;
1653 case 'a': {
1654 bool IsArm = Name.consume_front("arm.");
1655 if (IsArm || Name.consume_front("aarch64.")) {
1656 if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1657 return true;
1658 break;
1659 }
1660
1661 if (Name.consume_front("amdgcn.")) {
1662 if (Name == "alignbit") {
1663 // Target specific intrinsic became redundant
1664 NewFn = Intrinsic::getOrInsertDeclaration(
1665 F->getParent(), Intrinsic::fshr, {F->getReturnType()});
1666 return true;
1667 }
1668
1669 if (Name.consume_front("atomic.")) {
1670 if (Name.starts_with("inc") || Name.starts_with("dec") ||
1671 Name.starts_with("cond.sub") || Name.starts_with("csub")) {
1672 // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1673 // and usub_sat so there's no new declaration.
1674 NewFn = nullptr;
1675 return true;
1676 }
1677 break; // No other 'amdgcn.atomic.*'
1678 }
1679
1680 if (Name.starts_with("addrspacecast.nonnull")) {
1681 // Replaced with an addrspacecast instruction carrying the nonnull flag,
1682 // so there's no new declaration.
1683 NewFn = nullptr;
1684 return true;
1685 }
1686
1687 switch (F->getIntrinsicID()) {
1688 default:
1689 break;
1690 // Legacy wmma iu intrinsics without the optional clamp operand.
1691 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1692 if (F->arg_size() == 7) {
1693 NewFn = nullptr;
1694 return true;
1695 }
1696 break;
1697 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1698 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1699 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1700 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1701 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1702 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1703 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1704 if (F->arg_size() == 8) {
1705 NewFn = nullptr;
1706 return true;
1707 }
1708 break;
1709 }
1710
1711 if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||
1712 Name.consume_front("flat.atomic.")) {
1713 if (Name.starts_with("fadd") ||
1714 // FIXME: We should also remove fmin.num and fmax.num intrinsics.
1715 (Name.starts_with("fmin") && !Name.starts_with("fmin.num")) ||
1716 (Name.starts_with("fmax") && !Name.starts_with("fmax.num"))) {
1717 // Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1718 // declaration.
1719 NewFn = nullptr;
1720 return true;
1721 }
1722 }
1723
1724 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
1725 NewFn = nullptr;
1726 return true;
1727 }
1728
1729 if (Name.starts_with("ldexp.")) {
1730 // Target specific intrinsic became redundant
1731 NewFn = Intrinsic::getOrInsertDeclaration(
1732 F->getParent(), Intrinsic::ldexp,
1733 {F->getReturnType(), F->getArg(1)->getType()});
1734 return true;
1735 }
1736 break; // No other 'amdgcn.*'
1737 }
1738
1739 break;
1740 }
1741 case 'c': {
1742 if (F->arg_size() == 1) {
1743 if (Name.consume_front("convert.")) {
1744 if (convertIntrinsicValidType(Name, F->getFunctionType())) {
1745 NewFn = nullptr;
1746 return true;
1747 }
1748 }
1749
1750 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1751 .StartsWith("ctlz.", Intrinsic::ctlz)
1752 .StartsWith("cttz.", Intrinsic::cttz)
1753 .Default(Intrinsic::not_intrinsic);
1754 if (ID != Intrinsic::not_intrinsic) {
1755 rename(F);
1756 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1757 F->arg_begin()->getType());
1758 return true;
1759 }
1760 }
1761
1762 Intrinsic::ID CoroEndID = Intrinsic::not_intrinsic;
1763 if (Name == "coro.end" &&
1764 (F->arg_size() == 2 || F->getReturnType()->isIntegerTy(1)))
1765 CoroEndID = Intrinsic::coro_end;
1766 else if (Name == "coro.end.async" && F->getReturnType()->isIntegerTy(1))
1767 CoroEndID = Intrinsic::coro_end_async;
1768
1769 if (CoroEndID != Intrinsic::not_intrinsic) {
1770 rename(F);
1771 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), CoroEndID);
1772 return true;
1773 }
1774
1775 break;
1776 }
1777 case 'd':
1778 if (Name.consume_front("dbg.")) {
1779 // Mark debug intrinsics for upgrade to new debug format.
1780 if (CanUpgradeDebugIntrinsicsToRecords) {
1781 if (Name == "addr" || Name == "value" || Name == "assign" ||
1782 Name == "declare" || Name == "label") {
1783 // There's no function to replace these with.
1784 NewFn = nullptr;
1785 // But we do want these to get upgraded.
1786 return true;
1787 }
1788 }
1789 // Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1790 // converted to DbgVariableRecords later.
1791 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1792 rename(F);
1793 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1794 Intrinsic::dbg_value);
1795 return true;
1796 }
1797 break; // No other 'dbg.*'.
1798 }
1799 break;
1800 case 'e':
1801 if (Name.consume_front("experimental.vector.")) {
1802 Intrinsic::ID ID =
1803 StringSwitch<Intrinsic::ID>(Name)
1804 // Skip over extract.last.active, otherwise it will be 'upgraded'
1805 // to a regular vector extract which is a different operation.
1806 .StartsWith("extract.last.active.", Intrinsic::not_intrinsic)
1807 .StartsWith("extract.", Intrinsic::vector_extract)
1808 .StartsWith("insert.", Intrinsic::vector_insert)
1809 .StartsWith("reverse.", Intrinsic::vector_reverse)
1810 .StartsWith("interleave2.", Intrinsic::vector_interleave2)
1811 .StartsWith("deinterleave2.", Intrinsic::vector_deinterleave2)
1812 .StartsWith("partial.reduce.add",
1813 Intrinsic::vector_partial_reduce_add)
1814 .Default(Intrinsic::not_intrinsic);
1815 if (ID != Intrinsic::not_intrinsic) {
1816 const auto *FT = F->getFunctionType();
1817 SmallVector<Type *, 2> Tys;
1818 if (ID == Intrinsic::vector_extract ||
1819 ID == Intrinsic::vector_interleave2)
1820 // Extracting overloads the return type.
1821 Tys.push_back(FT->getReturnType());
1822 if (ID != Intrinsic::vector_interleave2)
1823 Tys.push_back(FT->getParamType(0));
1824 if (ID == Intrinsic::vector_insert ||
1825 ID == Intrinsic::vector_partial_reduce_add)
1826 // Inserting overloads the inserted type.
1827 Tys.push_back(FT->getParamType(1));
1828 rename(F);
1829 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
1830 return true;
1831 }
1832
1833 if (Name.consume_front("reduce.")) {
1834 SmallVector<StringRef, 2> Groups;
1835 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1836 if (R.match(Name, &Groups))
1837 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1838 .Case("add", Intrinsic::vector_reduce_add)
1839 .Case("mul", Intrinsic::vector_reduce_mul)
1840 .Case("and", Intrinsic::vector_reduce_and)
1841 .Case("or", Intrinsic::vector_reduce_or)
1842 .Case("xor", Intrinsic::vector_reduce_xor)
1843 .Case("smax", Intrinsic::vector_reduce_smax)
1844 .Case("smin", Intrinsic::vector_reduce_smin)
1845 .Case("umax", Intrinsic::vector_reduce_umax)
1846 .Case("umin", Intrinsic::vector_reduce_umin)
1847 .Case("fmax", Intrinsic::vector_reduce_fmax)
1848 .Case("fmin", Intrinsic::vector_reduce_fmin)
1849 .Default(Intrinsic::not_intrinsic);
1850
1851 bool V2 = false;
1852 if (ID == Intrinsic::not_intrinsic) {
1853 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1854 Groups.clear();
1855 V2 = true;
1856 if (R2.match(Name, &Groups))
1857 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1858 .Case("fadd", Intrinsic::vector_reduce_fadd)
1859 .Case("fmul", Intrinsic::vector_reduce_fmul)
1860 .Default(Intrinsic::not_intrinsic);
1861 }
1862 if (ID != Intrinsic::not_intrinsic) {
1863 rename(F);
1864 auto Args = F->getFunctionType()->params();
1865 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1866 {Args[V2 ? 1 : 0]});
1867 return true;
1868 }
1869 break; // No other 'expermental.vector.reduce.*'.
1870 }
1871
1872 if (Name.consume_front("splice"))
1873 return true;
1874 break; // No other 'experimental.vector.*'.
1875 }
1876 if (Name.consume_front("experimental.stepvector.")) {
1877 Intrinsic::ID ID = Intrinsic::stepvector;
1878 rename(F);
1879 NewFn = Intrinsic::getOrInsertDeclaration(
1880 F->getParent(), ID, F->getFunctionType()->getReturnType());
1881 return true;
1882 }
1883 break; // No other 'e*'.
1884 case 'f':
1885 if (Name.starts_with("flt.rounds")) {
1886 rename(F);
1887 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1888 Intrinsic::get_rounding);
1889 return true;
1890 }
1891 break;
1892 case 'i':
1893 if (Name.starts_with("invariant.group.barrier")) {
1894 // Rename invariant.group.barrier to launder.invariant.group
1895 auto Args = F->getFunctionType()->params();
1896 Type* ObjectPtr[1] = {Args[0]};
1897 rename(F);
1898 NewFn = Intrinsic::getOrInsertDeclaration(
1899 F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);
1900 return true;
1901 }
1902 break;
1903 case 'l': {
1904 bool IsLifetimeStart = Name.consume_front("lifetime.start");
1905 bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front("lifetime.end");
1906 if (IsLifetimeStart || IsLifetimeEnd) {
1907 if (F->arg_size() == 2) {
1908 Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
1909 : Intrinsic::lifetime_end;
1910 rename(F);
1911 // Old 2 argument form of these intrinsics have [Size, Ptr] as
1912 // arguments. Use the Ptr argument to create new declaration.
1913 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1914 F->getArg(1)->getType());
1915 return true;
1916 } else if (F->arg_size() == 1 && Name == ".i64") {
1917 // Matches @llvm.lifetime.{start/end}.i64 which used to be created by
1918 // Autoupgrade prior to
1919 // https://github.com/llvm/llvm-project/pull/204601. This is an invalid
1920 // intrinsic with no expected calls. To allow auto-upgrade process to
1921 // delete such invalid intrinsic declaration, set NewFn = nullptr
1922 // and return true here. If there are actual calls to this intrinsic
1923 // (which is not expected), they will be deleted in
1924 // UpgradeIntrinsicCall.
1925 NewFn = nullptr;
1926 return true;
1927 }
1928 }
1929 break;
1930 }
1931 case 'm': {
1932 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
1933 // alignment parameter to embedding the alignment as an attribute of
1934 // the pointer args.
1935 if (unsigned ID = StringSwitch<unsigned>(Name)
1936 .StartsWith("memcpy.", Intrinsic::memcpy)
1937 .StartsWith("memmove.", Intrinsic::memmove)
1938 .Default(0)) {
1939 if (F->arg_size() == 5) {
1940 rename(F);
1941 // Get the types of dest, src, and len
1942 ArrayRef<Type *> ParamTypes =
1943 F->getFunctionType()->params().slice(0, 3);
1944 NewFn =
1945 Intrinsic::getOrInsertDeclaration(F->getParent(), ID, ParamTypes);
1946 return true;
1947 }
1948 }
1949 if (Name.starts_with("memset.") && F->arg_size() == 5) {
1950 rename(F);
1951 // Get the types of dest, and len
1952 const auto *FT = F->getFunctionType();
1953 Type *ParamTypes[2] = {
1954 FT->getParamType(0), // Dest
1955 FT->getParamType(2) // len
1956 };
1957 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1958 Intrinsic::memset, ParamTypes);
1959 return true;
1960 }
1961
1962 unsigned MaskedID =
1963 StringSwitch<unsigned>(Name)
1964 .StartsWith("masked.load", Intrinsic::masked_load)
1965 .StartsWith("masked.gather", Intrinsic::masked_gather)
1966 .StartsWith("masked.store", Intrinsic::masked_store)
1967 .StartsWith("masked.scatter", Intrinsic::masked_scatter)
1968 .Default(0);
1969 if (MaskedID && F->arg_size() == 4) {
1970 rename(F);
1971 if (MaskedID == Intrinsic::masked_load ||
1972 MaskedID == Intrinsic::masked_gather) {
1973 NewFn = Intrinsic::getOrInsertDeclaration(
1974 F->getParent(), MaskedID,
1975 {F->getReturnType(), F->getArg(0)->getType()});
1976 return true;
1977 }
1978 NewFn = Intrinsic::getOrInsertDeclaration(
1979 F->getParent(), MaskedID,
1980 {F->getArg(0)->getType(), F->getArg(1)->getType()});
1981 return true;
1982 }
1983 break;
1984 }
1985 case 'n': {
1986 if (Name.consume_front("nvvm.")) {
1987 // Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
1988 if (F->arg_size() == 1) {
1989 Intrinsic::ID IID =
1990 StringSwitch<Intrinsic::ID>(Name)
1991 .Cases({"brev32", "brev64"}, Intrinsic::bitreverse)
1992 .Case("clz.i", Intrinsic::ctlz)
1993 .Case("popc.i", Intrinsic::ctpop)
1994 .Default(Intrinsic::not_intrinsic);
1995 if (IID != Intrinsic::not_intrinsic) {
1996 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1997 {F->getReturnType()});
1998 return true;
1999 }
2000 } else if (F->arg_size() == 2) {
2001 Intrinsic::ID IID =
2002 StringSwitch<Intrinsic::ID>(Name)
2003 .Cases({"max.s", "max.i", "max.ll"}, Intrinsic::smax)
2004 .Cases({"min.s", "min.i", "min.ll"}, Intrinsic::smin)
2005 .Cases({"max.us", "max.ui", "max.ull"}, Intrinsic::umax)
2006 .Cases({"min.us", "min.ui", "min.ull"}, Intrinsic::umin)
2007 .Default(Intrinsic::not_intrinsic);
2008 if (IID != Intrinsic::not_intrinsic) {
2009 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2010 {F->getReturnType()});
2011 return true;
2012 }
2013 }
2014
2015 // Check for nvvm intrinsics that need a return type adjustment.
2016 {
2017 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
2018 if (IID != Intrinsic::not_intrinsic &&
2019 isLegacyNVPTXBF16IntSignature(F, IID)) {
2020 NewFn = nullptr;
2021 return true;
2022 }
2023 }
2024
2025 // Upgrade Distributed Shared Memory Intrinsics
2026 Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
2027 if (IID != Intrinsic::not_intrinsic) {
2028 rename(F);
2029 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2030 return true;
2031 }
2032
2033 // Upgrade TMA reduction intrinsics
2034 // llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>* =>
2035 // llvm.nvvm.cp.async.bulk.tensor.reduce.<shape>*
2036 IID = shouldUpgradeNVPTXTMAReductionIntrinsics(Name);
2037 if (IID != Intrinsic::not_intrinsic) {
2038 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2039 return true;
2040 }
2041
2042 // Upgrade tcgen05.commit shared variants to anyptr intrinsics.
2043 IID = shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(F, Name);
2044 if (IID != Intrinsic::not_intrinsic) {
2045 rename(F);
2046 NewFn = Intrinsic::getOrInsertDeclaration(
2047 F->getParent(), IID, F->getReturnType(),
2048 F->getFunctionType()->params());
2049 return true;
2050 }
2051
2052 // Upgrade tcgen05.alloc/dealloc with the is_exclusive argument and
2053 // tcgen05.alloc shared variants to anyptr intrinsics.
2054 IID = shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(F, Name);
2055 if (IID != Intrinsic::not_intrinsic) {
2056 rename(F);
2057 if (Intrinsic::isOverloaded(IID))
2058 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2059 {F->getArg(0)->getType()});
2060 else
2061 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2062 return true;
2063 }
2064
2065 // Upgrade TMA copy G2S CTA intrinsics.
2066 IID = shouldUpgradeNVPTXTMAG2SCTAIntrinsics(F, Name);
2067 if (IID != Intrinsic::not_intrinsic) {
2068 rename(F);
2069 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2070 return true;
2071 }
2072
2073 // Upgrade TMA copy G2S (cluster) intrinsics.
2074 SmallVector<Type *, 1> OvlTys;
2075 IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name, OvlTys);
2076 if (IID != Intrinsic::not_intrinsic) {
2077 rename(F);
2078 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID, OvlTys);
2079 return true;
2080 }
2081
2082 // Upgrade tcgen05.mma intrinsics missing collector_usage_b.
2083 IID = shouldUpgradeNVPTXTcgen05MMAIntrinsic(F, Name);
2084 if (IID != Intrinsic::not_intrinsic) {
2085 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2086 return NewFn != F;
2087 }
2088
2089 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
2090 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
2091 //
2092 // TODO: We could add lohi.i2d.
2093 bool Expand = false;
2094 if (Name.consume_front("abs."))
2095 // nvvm.abs.{i,ii}
2096 Expand =
2097 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
2098 else if (Name.consume_front("fabs."))
2099 // nvvm.fabs.{f,ftz.f,d}
2100 Expand = Name == "f" || Name == "ftz.f" || Name == "d";
2101 else if (Name.consume_front("add."))
2102 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
2103 Expand = getNVVMFAddUpgrade(Name).has_value();
2104 else if (Name.consume_front("ex2.approx."))
2105 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
2106 Expand =
2107 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
2108 else if (Name.consume_front("atomic.load."))
2109 // nvvm.atomic.load.add.{f32,f64}.p
2110 // nvvm.atomic.load.{inc,dec}.32.p
2111 Expand = StringSwitch<bool>(Name)
2112 .StartsWith("add.f32.p", true)
2113 .StartsWith("add.f64.p", true)
2114 .StartsWith("inc.32.p", true)
2115 .StartsWith("dec.32.p", true)
2116 .Default(false);
2117 else if (Name.consume_front("atomic."))
2118 // nvvm.atomic.{add,exch,max,min,inc,dec,and,or,xor}.gen.{i,f}.{cta,sys}
2119 // nvvm.atomic.cas.gen.i.{cta,sys}
2120 Expand = StringSwitch<bool>(Name)
2121 .StartsWith("add.gen.", true)
2122 .StartsWith("exch.gen.", true)
2123 .StartsWith("max.gen.", true)
2124 .StartsWith("min.gen.", true)
2125 .StartsWith("inc.gen.", true)
2126 .StartsWith("dec.gen.", true)
2127 .StartsWith("and.gen.", true)
2128 .StartsWith("or.gen.", true)
2129 .StartsWith("xor.gen.", true)
2130 .StartsWith("cas.gen.", true)
2131 .Default(false);
2132 else if (Name.consume_front("bitcast."))
2133 // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
2134 Expand =
2135 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
2136 else if (Name.consume_front("rotate."))
2137 // nvvm.rotate.{b32,b64,right.b64}
2138 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
2139 else if (Name.consume_front("ptr.gen.to."))
2140 // nvvm.ptr.gen.to.{local,shared,global,constant,param}
2141 Expand = consumeNVVMPtrAddrSpace(Name);
2142 else if (Name.consume_front("ptr."))
2143 // nvvm.ptr.{local,shared,global,constant,param}.to.gen
2144 Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(".to.gen");
2145 else if (Name.consume_front("ldg.global."))
2146 // nvvm.ldg.global.{i,p,f}
2147 Expand = (Name.starts_with("i.") || Name.starts_with("f.") ||
2148 Name.starts_with("p."));
2149 else
2150 Expand = StringSwitch<bool>(Name)
2151 .Case("barrier0", true)
2152 .Case("barrier.n", true)
2153 .Case("barrier.sync.cnt", true)
2154 .Case("barrier.sync", true)
2155 .Case("barrier", true)
2156 .Case("bar.sync", true)
2157 .Case("barrier0.popc", true)
2158 .Case("barrier0.and", true)
2159 .Case("barrier0.or", true)
2160 .Case("clz.ll", true)
2161 .Case("popc.ll", true)
2162 .Case("h2f", true)
2163 .Case("swap.lo.hi.b64", true)
2164 .Case("tanh.approx.f32", true)
2165 .Default(false);
2166
2167 if (Expand) {
2168 NewFn = nullptr;
2169 return true;
2170 }
2171 break; // No other 'nvvm.*'.
2172 }
2173 break;
2174 }
2175 case 'o':
2176 if (Name.starts_with("objectsize.")) {
2177 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
2178 if (F->arg_size() == 2 || F->arg_size() == 3) {
2179 rename(F);
2180 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2181 Intrinsic::objectsize, Tys);
2182 return true;
2183 }
2184 }
2185 break;
2186
2187 case 'p':
2188 if (Name.starts_with("ptr.annotation.") && F->arg_size() == 4) {
2189 rename(F);
2190 NewFn = Intrinsic::getOrInsertDeclaration(
2191 F->getParent(), Intrinsic::ptr_annotation,
2192 {F->arg_begin()->getType(), F->getArg(1)->getType()});
2193 return true;
2194 }
2195 break;
2196
2197 case 'r': {
2198 if (Name.consume_front("riscv.")) {
2199 Intrinsic::ID ID;
2200 ID = StringSwitch<Intrinsic::ID>(Name)
2201 .Case("aes32dsi", Intrinsic::riscv_aes32dsi)
2202 .Case("aes32dsmi", Intrinsic::riscv_aes32dsmi)
2203 .Case("aes32esi", Intrinsic::riscv_aes32esi)
2204 .Case("aes32esmi", Intrinsic::riscv_aes32esmi)
2205 .Default(Intrinsic::not_intrinsic);
2206 if (ID != Intrinsic::not_intrinsic) {
2207 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
2208 rename(F);
2209 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2210 return true;
2211 }
2212 break; // No other applicable upgrades.
2213 }
2214
2215 ID = StringSwitch<Intrinsic::ID>(Name)
2216 .StartsWith("sm4ks", Intrinsic::riscv_sm4ks)
2217 .StartsWith("sm4ed", Intrinsic::riscv_sm4ed)
2218 .Default(Intrinsic::not_intrinsic);
2219 if (ID != Intrinsic::not_intrinsic) {
2220 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
2221 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2222 rename(F);
2223 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2224 return true;
2225 }
2226 break; // No other applicable upgrades.
2227 }
2228
2229 ID = StringSwitch<Intrinsic::ID>(Name)
2230 .StartsWith("sha256sig0", Intrinsic::riscv_sha256sig0)
2231 .StartsWith("sha256sig1", Intrinsic::riscv_sha256sig1)
2232 .StartsWith("sha256sum0", Intrinsic::riscv_sha256sum0)
2233 .StartsWith("sha256sum1", Intrinsic::riscv_sha256sum1)
2234 .StartsWith("sm3p0", Intrinsic::riscv_sm3p0)
2235 .StartsWith("sm3p1", Intrinsic::riscv_sm3p1)
2236 .Default(Intrinsic::not_intrinsic);
2237 if (ID != Intrinsic::not_intrinsic) {
2238 if (F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2239 rename(F);
2240 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2241 return true;
2242 }
2243 break; // No other applicable upgrades.
2244 }
2245
2246 // Replace llvm.riscv.clmul with llvm.clmul.
2247 if (Name == "clmul.i32" || Name == "clmul.i64") {
2248 NewFn = Intrinsic::getOrInsertDeclaration(
2249 F->getParent(), Intrinsic::clmul, {F->getReturnType()});
2250 return true;
2251 }
2252
2253 break; // No other 'riscv.*' intrinsics
2254 }
2255 } break;
2256
2257 case 's':
2258 if (Name == "stackprotectorcheck") {
2259 NewFn = nullptr;
2260 return true;
2261 }
2262 break;
2263
2264 case 't':
2265 if (Name == "thread.pointer") {
2266 NewFn = Intrinsic::getOrInsertDeclaration(
2267 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
2268 return true;
2269 }
2270 break;
2271
2272 case 'v': {
2273 if (Name == "var.annotation" && F->arg_size() == 4) {
2274 rename(F);
2275 NewFn = Intrinsic::getOrInsertDeclaration(
2276 F->getParent(), Intrinsic::var_annotation,
2277 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
2278 return true;
2279 }
2280 if (Name.consume_front("vector.splice")) {
2281 if (Name.starts_with(".left") || Name.starts_with(".right"))
2282 break;
2283 return true;
2284 }
2285 if (shouldUpgradeVPIntrinsic(Name))
2286 return true;
2287 break;
2288 }
2289
2290 case 'w':
2291 if (Name.consume_front("wasm.")) {
2292 Intrinsic::ID ID =
2293 StringSwitch<Intrinsic::ID>(Name)
2294 .StartsWith("fma.", Intrinsic::wasm_relaxed_madd)
2295 .StartsWith("fms.", Intrinsic::wasm_relaxed_nmadd)
2296 .StartsWith("laneselect.", Intrinsic::wasm_relaxed_laneselect)
2297 .Default(Intrinsic::not_intrinsic);
2298 if (ID != Intrinsic::not_intrinsic) {
2299 rename(F);
2300 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
2301 F->getReturnType());
2302 return true;
2303 }
2304
2305 if (Name.consume_front("dot.i8x16.i7x16.")) {
2306 ID = StringSwitch<Intrinsic::ID>(Name)
2307 .Case("signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
2308 .Case("add.signed",
2309 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
2310 .Default(Intrinsic::not_intrinsic);
2311 if (ID != Intrinsic::not_intrinsic) {
2312 rename(F);
2313 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2314 return true;
2315 }
2316 break; // No other 'wasm.dot.i8x16.i7x16.*'.
2317 }
2318 break; // No other 'wasm.*'.
2319 }
2320 break;
2321
2322 case 'x':
2323 if (upgradeX86IntrinsicFunction(F, Name, NewFn))
2324 return true;
2325 }
2326
2327 auto *ST = dyn_cast<StructType>(F->getReturnType());
2328 if (ST && (!ST->isLiteral() || ST->isPacked()) &&
2329 F->getIntrinsicID() != Intrinsic::not_intrinsic) {
2330 // Replace return type with literal non-packed struct. Only do this for
2331 // intrinsics declared to return a struct, not for intrinsics with
2332 // overloaded return type, in which case the exact struct type will be
2333 // mangled into the name.
2334 if (Intrinsic::hasStructReturnType(F->getIntrinsicID())) {
2335 FunctionType *FT = F->getFunctionType();
2336 auto *NewST = StructType::get(ST->getContext(), ST->elements());
2337 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
2338 std::string Name = F->getName().str();
2339 rename(F);
2340 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(),
2341 Name, F->getParent());
2342
2343 // The new function may also need remangling.
2344 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn))
2345 NewFn = *Result;
2346 return true;
2347 }
2348 }
2349
2350 // Remangle our intrinsic since we upgrade the mangling
2351 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
2352 if (Result != std::nullopt) {
2353 NewFn = *Result;
2354 return true;
2355 }
2356
2357 // This may not belong here. This function is effectively being overloaded
2358 // to both detect an intrinsic which needs upgrading, and to provide the
2359 // upgraded form of the intrinsic. We should perhaps have two separate
2360 // functions for this.
2361 if (upgradeIntrinsicDeclWithDefaultArgs(F, NewFn))
2362 return true;
2363
2364 return false;
2365}
2366
2367bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
2368 bool CanUpgradeDebugIntrinsicsToRecords) {
2369 NewFn = nullptr;
2370 bool Upgraded =
2371 upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
2372
2373 // Upgrade intrinsic attributes. This does not change the function.
2374 if (NewFn)
2375 F = NewFn;
2376 if (Intrinsic::ID id = F->getIntrinsicID()) {
2377 // Only do this if the intrinsic signature is valid.
2378 SmallVector<Type *> OverloadTys;
2379 if (Intrinsic::isSignatureValid(id, F->getFunctionType(), OverloadTys))
2380 F->setAttributes(
2381 Intrinsic::getAttributes(F->getContext(), id, F->getFunctionType()));
2382 }
2383 return Upgraded;
2384}
2385
2386GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
2387 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
2388 GV->getName() == "llvm.global_dtors")) ||
2389 !GV->hasInitializer())
2390 return nullptr;
2391 ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
2392 if (!ATy)
2393 return nullptr;
2394 StructType *STy = dyn_cast<StructType>(ATy->getElementType());
2395 if (!STy || STy->getNumElements() != 2)
2396 return nullptr;
2397
2398 LLVMContext &C = GV->getContext();
2399 IRBuilder<> IRB(C);
2400 auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
2401 IRB.getPtrTy());
2402 Constant *Init = GV->getInitializer();
2403 unsigned N = Init->getNumOperands();
2404 std::vector<Constant *> NewCtors(N);
2405 for (unsigned i = 0; i != N; ++i) {
2406 auto Ctor = cast<Constant>(Init->getOperand(i));
2407 NewCtors[i] = ConstantStruct::get(EltTy, Ctor->getAggregateElement(0u),
2408 Ctor->getAggregateElement(1),
2409 ConstantPointerNull::get(IRB.getPtrTy()));
2410 }
2411 Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
2412
2413 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
2414 NewInit, GV->getName());
2415}
2416
2417// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
2418// to byte shuffles.
2419static Value *upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2420 unsigned Shift) {
2421 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2422 unsigned NumElts = ResultTy->getNumElements() * 8;
2423
2424 // Bitcast from a 64-bit element type to a byte element type.
2425 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2426 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2427
2428 // We'll be shuffling in zeroes.
2429 Value *Res = Constant::getNullValue(VecTy);
2430
2431 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2432 // we'll just return the zero vector.
2433 if (Shift < 16) {
2434 int Idxs[64];
2435 // 256/512-bit version is split into 2/4 16-byte lanes.
2436 for (unsigned l = 0; l != NumElts; l += 16)
2437 for (unsigned i = 0; i != 16; ++i) {
2438 unsigned Idx = NumElts + i - Shift;
2439 if (Idx < NumElts)
2440 Idx -= NumElts - 16; // end of lane, switch operand.
2441 Idxs[l + i] = Idx + l;
2442 }
2443
2444 Res = Builder.CreateShuffleVector(Res, Op, ArrayRef(Idxs, NumElts));
2445 }
2446
2447 // Bitcast back to a 64-bit element type.
2448 return Builder.CreateBitCast(Res, ResultTy, "cast");
2449}
2450
2451// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
2452// to byte shuffles.
2453static Value *upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2454 unsigned Shift) {
2455 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2456 unsigned NumElts = ResultTy->getNumElements() * 8;
2457
2458 // Bitcast from a 64-bit element type to a byte element type.
2459 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2460 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2461
2462 // We'll be shuffling in zeroes.
2463 Value *Res = Constant::getNullValue(VecTy);
2464
2465 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2466 // we'll just return the zero vector.
2467 if (Shift < 16) {
2468 int Idxs[64];
2469 // 256/512-bit version is split into 2/4 16-byte lanes.
2470 for (unsigned l = 0; l != NumElts; l += 16)
2471 for (unsigned i = 0; i != 16; ++i) {
2472 unsigned Idx = i + Shift;
2473 if (Idx >= 16)
2474 Idx += NumElts - 16; // end of lane, switch operand.
2475 Idxs[l + i] = Idx + l;
2476 }
2477
2478 Res = Builder.CreateShuffleVector(Op, Res, ArrayRef(Idxs, NumElts));
2479 }
2480
2481 // Bitcast back to a 64-bit element type.
2482 return Builder.CreateBitCast(Res, ResultTy, "cast");
2483}
2484
2485static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
2486 unsigned NumElts) {
2487 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2488 llvm::VectorType *MaskTy = FixedVectorType::get(
2489 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
2490 Mask = Builder.CreateBitCast(Mask, MaskTy);
2491
2492 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2493 // i8 and we need to extract down to the right number of elements.
2494 if (NumElts <= 4) {
2495 int Indices[4];
2496 for (unsigned i = 0; i != NumElts; ++i)
2497 Indices[i] = i;
2498 Mask = Builder.CreateShuffleVector(Mask, Mask, ArrayRef(Indices, NumElts),
2499 "extract");
2500 }
2501
2502 return Mask;
2503}
2504
2505static Value *emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2506 Value *Op1) {
2507 // If the mask is all ones just emit the first operation.
2508 if (const auto *C = dyn_cast<Constant>(Mask))
2509 if (C->isAllOnesValue())
2510 return Op0;
2511
2512 Mask = getX86MaskVec(Builder, Mask,
2513 cast<FixedVectorType>(Op0->getType())->getNumElements());
2514 return Builder.CreateSelect(Mask, Op0, Op1);
2515}
2516
2517static Value *emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2518 Value *Op1) {
2519 // If the mask is all ones just emit the first operation.
2520 if (const auto *C = dyn_cast<Constant>(Mask))
2521 if (C->isAllOnesValue())
2522 return Op0;
2523
2524 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
2525 Mask->getType()->getIntegerBitWidth());
2526 Mask = Builder.CreateBitCast(Mask, MaskTy);
2527 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
2528 return Builder.CreateSelect(Mask, Op0, Op1);
2529}
2530
2531// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2532// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2533// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2534static Value *upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
2535 Value *Op1, Value *Shift,
2536 Value *Passthru, Value *Mask,
2537 bool IsVALIGN) {
2538 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
2539
2540 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2541 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
2542 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
2543 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2544
2545 // Mask the immediate for VALIGN.
2546 if (IsVALIGN)
2547 ShiftVal &= (NumElts - 1);
2548
2549 // If palignr is shifting the pair of vectors more than the size of two
2550 // lanes, emit zero.
2551 if (ShiftVal >= 32)
2552 return llvm::Constant::getNullValue(Op0->getType());
2553
2554 // If palignr is shifting the pair of input vectors more than one lane,
2555 // but less than two lanes, convert to shifting in zeroes.
2556 if (ShiftVal > 16) {
2557 ShiftVal -= 16;
2558 Op1 = Op0;
2559 Op0 = llvm::Constant::getNullValue(Op0->getType());
2560 }
2561
2562 int Indices[64];
2563 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2564 for (unsigned l = 0; l < NumElts; l += 16) {
2565 for (unsigned i = 0; i != 16; ++i) {
2566 unsigned Idx = ShiftVal + i;
2567 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
2568 Idx += NumElts - 16; // End of lane, switch operand.
2569 Indices[l + i] = Idx + l;
2570 }
2571 }
2572
2573 Value *Align = Builder.CreateShuffleVector(
2574 Op1, Op0, ArrayRef(Indices, NumElts), "palignr");
2575
2576 return emitX86Select(Builder, Mask, Align, Passthru);
2577}
2578
2579static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
2580 bool ZeroMask, bool IndexForm) {
2581 Type *Ty = CI.getType();
2582 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2583 unsigned EltWidth = Ty->getScalarSizeInBits();
2584 bool IsFloat = Ty->isFPOrFPVectorTy();
2585 Intrinsic::ID IID;
2586 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2587 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2588 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2589 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2590 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2591 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2592 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2593 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2594 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2595 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2596 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2597 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2598 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2599 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2600 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2601 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2602 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2603 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2604 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2605 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2606 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2607 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2608 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2609 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2610 else if (VecWidth == 128 && EltWidth == 16)
2611 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2612 else if (VecWidth == 256 && EltWidth == 16)
2613 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2614 else if (VecWidth == 512 && EltWidth == 16)
2615 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2616 else if (VecWidth == 128 && EltWidth == 8)
2617 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2618 else if (VecWidth == 256 && EltWidth == 8)
2619 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2620 else if (VecWidth == 512 && EltWidth == 8)
2621 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2622 else
2623 llvm_unreachable("Unexpected intrinsic");
2624
2625 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
2626 CI.getArgOperand(2) };
2627
2628 // If this isn't index form we need to swap operand 0 and 1.
2629 if (!IndexForm)
2630 std::swap(Args[0], Args[1]);
2631
2632 Value *V = Builder.CreateIntrinsic(IID, Args);
2633 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2634 : Builder.CreateBitCast(CI.getArgOperand(1),
2635 Ty);
2636 return emitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
2637}
2638
2639static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
2640 Intrinsic::ID IID) {
2641 Type *Ty = CI.getType();
2642 Value *Op0 = CI.getOperand(0);
2643 Value *Op1 = CI.getOperand(1);
2644 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1});
2645
2646 if (CI.arg_size() == 4) { // For masked intrinsics.
2647 Value *VecSrc = CI.getOperand(2);
2648 Value *Mask = CI.getOperand(3);
2649 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2650 }
2651 return Res;
2652}
2653
2654static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
2655 bool IsRotateRight) {
2656 Type *Ty = CI.getType();
2657 Value *Src = CI.getArgOperand(0);
2658 Value *Amt = CI.getArgOperand(1);
2659
2660 // Amount may be scalar immediate, in which case create a splat vector.
2661 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2662 // we only care about the lowest log2 bits anyway.
2663 if (Amt->getType() != Ty) {
2664 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2665 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2666 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2667 }
2668
2669 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2670 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Src, Src, Amt});
2671
2672 if (CI.arg_size() == 4) { // For masked intrinsics.
2673 Value *VecSrc = CI.getOperand(2);
2674 Value *Mask = CI.getOperand(3);
2675 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2676 }
2677 return Res;
2678}
2679
2680static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
2681 bool IsSigned) {
2682 Type *Ty = CI.getType();
2683 Value *LHS = CI.getArgOperand(0);
2684 Value *RHS = CI.getArgOperand(1);
2685
2686 CmpInst::Predicate Pred;
2687 switch (Imm) {
2688 case 0x0:
2689 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
2690 break;
2691 case 0x1:
2692 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2693 break;
2694 case 0x2:
2695 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
2696 break;
2697 case 0x3:
2698 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
2699 break;
2700 case 0x4:
2701 Pred = ICmpInst::ICMP_EQ;
2702 break;
2703 case 0x5:
2704 Pred = ICmpInst::ICMP_NE;
2705 break;
2706 case 0x6:
2707 return Constant::getNullValue(Ty); // FALSE
2708 case 0x7:
2709 return Constant::getAllOnesValue(Ty); // TRUE
2710 default:
2711 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2712 }
2713
2714 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
2715 Value *Ext = Builder.CreateSExt(Cmp, Ty);
2716 return Ext;
2717}
2718
2719static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
2720 bool IsShiftRight, bool ZeroMask) {
2721 Type *Ty = CI.getType();
2722 Value *Op0 = CI.getArgOperand(0);
2723 Value *Op1 = CI.getArgOperand(1);
2724 Value *Amt = CI.getArgOperand(2);
2725
2726 if (IsShiftRight)
2727 std::swap(Op0, Op1);
2728
2729 // Amount may be scalar immediate, in which case create a splat vector.
2730 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2731 // we only care about the lowest log2 bits anyway.
2732 if (Amt->getType() != Ty) {
2733 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2734 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2735 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2736 }
2737
2738 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2739 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1, Amt});
2740
2741 unsigned NumArgs = CI.arg_size();
2742 if (NumArgs >= 4) { // For masked intrinsics.
2743 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
2744 ZeroMask ? ConstantAggregateZero::get(CI.getType()) :
2745 CI.getArgOperand(0);
2746 Value *Mask = CI.getOperand(NumArgs - 1);
2747 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2748 }
2749 return Res;
2750}
2751
2752static Value *upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data,
2753 Value *Mask, bool Aligned) {
2754 const Align Alignment =
2755 Aligned
2756 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2757 : Align(1);
2758
2759 // If the mask is all ones just emit a regular store.
2760 if (const auto *C = dyn_cast<Constant>(Mask))
2761 if (C->isAllOnesValue())
2762 return Builder.CreateAlignedStore(Data, Ptr, Alignment);
2763
2764 // Convert the mask from an integer type to a vector of i1.
2765 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
2766 Mask = getX86MaskVec(Builder, Mask, NumElts);
2767 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
2768}
2769
2770static Value *upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr,
2771 Value *Passthru, Value *Mask, bool Aligned) {
2772 Type *ValTy = Passthru->getType();
2773 const Align Alignment =
2774 Aligned
2775 ? Align(
2776 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
2777 8)
2778 : Align(1);
2779
2780 // If the mask is all ones just emit a regular store.
2781 if (const auto *C = dyn_cast<Constant>(Mask))
2782 if (C->isAllOnesValue())
2783 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
2784
2785 // Convert the mask from an integer type to a vector of i1.
2786 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
2787 Mask = getX86MaskVec(Builder, Mask, NumElts);
2788 return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
2789}
2790
2791static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2792 Type *Ty = CI.getType();
2793 Value *Op0 = CI.getArgOperand(0);
2794 Value *Res = Builder.CreateIntrinsic(Intrinsic::abs, Ty,
2795 {Op0, Builder.getInt1(false)});
2796 if (CI.arg_size() == 3)
2797 Res = emitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
2798 return Res;
2799}
2800
2801static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
2802 Type *Ty = CI.getType();
2803
2804 // Arguments have a vXi32 type so cast to vXi64.
2805 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
2806 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
2807
2808 if (IsSigned) {
2809 // Shift left then arithmetic shift right.
2810 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
2811 LHS = Builder.CreateShl(LHS, ShiftAmt);
2812 LHS = Builder.CreateAShr(LHS, ShiftAmt);
2813 RHS = Builder.CreateShl(RHS, ShiftAmt);
2814 RHS = Builder.CreateAShr(RHS, ShiftAmt);
2815 } else {
2816 // Clear the upper bits.
2817 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
2818 LHS = Builder.CreateAnd(LHS, Mask);
2819 RHS = Builder.CreateAnd(RHS, Mask);
2820 }
2821
2822 Value *Res = Builder.CreateMul(LHS, RHS);
2823
2824 if (CI.arg_size() == 4)
2825 Res = emitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
2826
2827 return Res;
2828}
2829
2830// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2831static Value *applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
2832 Value *Mask) {
2833 unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
2834 if (Mask) {
2835 const auto *C = dyn_cast<Constant>(Mask);
2836 if (!C || !C->isAllOnesValue())
2837 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
2838 }
2839
2840 if (NumElts < 8) {
2841 int Indices[8];
2842 for (unsigned i = 0; i != NumElts; ++i)
2843 Indices[i] = i;
2844 for (unsigned i = NumElts; i != 8; ++i)
2845 Indices[i] = NumElts + i % NumElts;
2846 Vec = Builder.CreateShuffleVector(Vec,
2847 Constant::getNullValue(Vec->getType()),
2848 Indices);
2849 }
2850 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
2851}
2852
2853static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
2854 unsigned CC, bool Signed) {
2855 Value *Op0 = CI.getArgOperand(0);
2856 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2857
2858 Value *Cmp;
2859 if (CC == 3) {
2860 Cmp = Constant::getNullValue(
2861 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2862 } else if (CC == 7) {
2863 Cmp = Constant::getAllOnesValue(
2864 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2865 } else {
2866 ICmpInst::Predicate Pred;
2867 switch (CC) {
2868 default: llvm_unreachable("Unknown condition code");
2869 case 0: Pred = ICmpInst::ICMP_EQ; break;
2870 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
2871 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
2872 case 4: Pred = ICmpInst::ICMP_NE; break;
2873 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
2874 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
2875 }
2876 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
2877 }
2878
2879 Value *Mask = CI.getArgOperand(CI.arg_size() - 1);
2880
2881 return applyX86MaskOn1BitsVec(Builder, Cmp, Mask);
2882}
2883
2884// Replace a masked intrinsic with an older unmasked intrinsic.
2885static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
2886 Intrinsic::ID IID) {
2887 Value *Rep =
2888 Builder.CreateIntrinsic(IID, {CI.getArgOperand(0), CI.getArgOperand(1)});
2889 return emitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
2890}
2891
2892static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
2893 Value* A = CI.getArgOperand(0);
2894 Value* B = CI.getArgOperand(1);
2895 Value* Src = CI.getArgOperand(2);
2896 Value* Mask = CI.getArgOperand(3);
2897
2898 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
2899 Value* Cmp = Builder.CreateIsNotNull(AndNode);
2900 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
2901 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
2902 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
2903 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
2904}
2905
2906static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
2907 Value* Op = CI.getArgOperand(0);
2908 Type* ReturnOp = CI.getType();
2909 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
2910 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
2911 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
2912}
2913
2914// Replace intrinsic with unmasked version and a select.
2915static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
2916 CallBase &CI, Value *&Rep) {
2917 Name = Name.substr(12); // Remove avx512.mask.
2918
2919 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
2920 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
2921 Intrinsic::ID IID;
2922 if (Name.starts_with("max.p")) {
2923 if (VecWidth == 128 && EltWidth == 32)
2924 IID = Intrinsic::x86_sse_max_ps;
2925 else if (VecWidth == 128 && EltWidth == 64)
2926 IID = Intrinsic::x86_sse2_max_pd;
2927 else if (VecWidth == 256 && EltWidth == 32)
2928 IID = Intrinsic::x86_avx_max_ps_256;
2929 else if (VecWidth == 256 && EltWidth == 64)
2930 IID = Intrinsic::x86_avx_max_pd_256;
2931 else
2932 llvm_unreachable("Unexpected intrinsic");
2933 } else if (Name.starts_with("min.p")) {
2934 if (VecWidth == 128 && EltWidth == 32)
2935 IID = Intrinsic::x86_sse_min_ps;
2936 else if (VecWidth == 128 && EltWidth == 64)
2937 IID = Intrinsic::x86_sse2_min_pd;
2938 else if (VecWidth == 256 && EltWidth == 32)
2939 IID = Intrinsic::x86_avx_min_ps_256;
2940 else if (VecWidth == 256 && EltWidth == 64)
2941 IID = Intrinsic::x86_avx_min_pd_256;
2942 else
2943 llvm_unreachable("Unexpected intrinsic");
2944 } else if (Name.starts_with("pshuf.b.")) {
2945 if (VecWidth == 128)
2946 IID = Intrinsic::x86_ssse3_pshuf_b_128;
2947 else if (VecWidth == 256)
2948 IID = Intrinsic::x86_avx2_pshuf_b;
2949 else if (VecWidth == 512)
2950 IID = Intrinsic::x86_avx512_pshuf_b_512;
2951 else
2952 llvm_unreachable("Unexpected intrinsic");
2953 } else if (Name.starts_with("pmul.hr.sw.")) {
2954 if (VecWidth == 128)
2955 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
2956 else if (VecWidth == 256)
2957 IID = Intrinsic::x86_avx2_pmul_hr_sw;
2958 else if (VecWidth == 512)
2959 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
2960 else
2961 llvm_unreachable("Unexpected intrinsic");
2962 } else if (Name.starts_with("pmulh.w.")) {
2963 if (VecWidth == 128)
2964 IID = Intrinsic::x86_sse2_pmulh_w;
2965 else if (VecWidth == 256)
2966 IID = Intrinsic::x86_avx2_pmulh_w;
2967 else if (VecWidth == 512)
2968 IID = Intrinsic::x86_avx512_pmulh_w_512;
2969 else
2970 llvm_unreachable("Unexpected intrinsic");
2971 } else if (Name.starts_with("pmulhu.w.")) {
2972 if (VecWidth == 128)
2973 IID = Intrinsic::x86_sse2_pmulhu_w;
2974 else if (VecWidth == 256)
2975 IID = Intrinsic::x86_avx2_pmulhu_w;
2976 else if (VecWidth == 512)
2977 IID = Intrinsic::x86_avx512_pmulhu_w_512;
2978 else
2979 llvm_unreachable("Unexpected intrinsic");
2980 } else if (Name.starts_with("pmaddw.d.")) {
2981 if (VecWidth == 128)
2982 IID = Intrinsic::x86_sse2_pmadd_wd;
2983 else if (VecWidth == 256)
2984 IID = Intrinsic::x86_avx2_pmadd_wd;
2985 else if (VecWidth == 512)
2986 IID = Intrinsic::x86_avx512_pmaddw_d_512;
2987 else
2988 llvm_unreachable("Unexpected intrinsic");
2989 } else if (Name.starts_with("pmaddubs.w.")) {
2990 if (VecWidth == 128)
2991 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
2992 else if (VecWidth == 256)
2993 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
2994 else if (VecWidth == 512)
2995 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
2996 else
2997 llvm_unreachable("Unexpected intrinsic");
2998 } else if (Name.starts_with("packsswb.")) {
2999 if (VecWidth == 128)
3000 IID = Intrinsic::x86_sse2_packsswb_128;
3001 else if (VecWidth == 256)
3002 IID = Intrinsic::x86_avx2_packsswb;
3003 else if (VecWidth == 512)
3004 IID = Intrinsic::x86_avx512_packsswb_512;
3005 else
3006 llvm_unreachable("Unexpected intrinsic");
3007 } else if (Name.starts_with("packssdw.")) {
3008 if (VecWidth == 128)
3009 IID = Intrinsic::x86_sse2_packssdw_128;
3010 else if (VecWidth == 256)
3011 IID = Intrinsic::x86_avx2_packssdw;
3012 else if (VecWidth == 512)
3013 IID = Intrinsic::x86_avx512_packssdw_512;
3014 else
3015 llvm_unreachable("Unexpected intrinsic");
3016 } else if (Name.starts_with("packuswb.")) {
3017 if (VecWidth == 128)
3018 IID = Intrinsic::x86_sse2_packuswb_128;
3019 else if (VecWidth == 256)
3020 IID = Intrinsic::x86_avx2_packuswb;
3021 else if (VecWidth == 512)
3022 IID = Intrinsic::x86_avx512_packuswb_512;
3023 else
3024 llvm_unreachable("Unexpected intrinsic");
3025 } else if (Name.starts_with("packusdw.")) {
3026 if (VecWidth == 128)
3027 IID = Intrinsic::x86_sse41_packusdw;
3028 else if (VecWidth == 256)
3029 IID = Intrinsic::x86_avx2_packusdw;
3030 else if (VecWidth == 512)
3031 IID = Intrinsic::x86_avx512_packusdw_512;
3032 else
3033 llvm_unreachable("Unexpected intrinsic");
3034 } else if (Name.starts_with("vpermilvar.")) {
3035 if (VecWidth == 128 && EltWidth == 32)
3036 IID = Intrinsic::x86_avx_vpermilvar_ps;
3037 else if (VecWidth == 128 && EltWidth == 64)
3038 IID = Intrinsic::x86_avx_vpermilvar_pd;
3039 else if (VecWidth == 256 && EltWidth == 32)
3040 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
3041 else if (VecWidth == 256 && EltWidth == 64)
3042 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
3043 else if (VecWidth == 512 && EltWidth == 32)
3044 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
3045 else if (VecWidth == 512 && EltWidth == 64)
3046 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
3047 else
3048 llvm_unreachable("Unexpected intrinsic");
3049 } else if (Name == "cvtpd2dq.256") {
3050 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
3051 } else if (Name == "cvtpd2ps.256") {
3052 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
3053 } else if (Name == "cvttpd2dq.256") {
3054 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
3055 } else if (Name == "cvttps2dq.128") {
3056 IID = Intrinsic::x86_sse2_cvttps2dq;
3057 } else if (Name == "cvttps2dq.256") {
3058 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
3059 } else if (Name.starts_with("permvar.")) {
3060 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
3061 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
3062 IID = Intrinsic::x86_avx2_permps;
3063 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
3064 IID = Intrinsic::x86_avx2_permd;
3065 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
3066 IID = Intrinsic::x86_avx512_permvar_df_256;
3067 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
3068 IID = Intrinsic::x86_avx512_permvar_di_256;
3069 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
3070 IID = Intrinsic::x86_avx512_permvar_sf_512;
3071 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
3072 IID = Intrinsic::x86_avx512_permvar_si_512;
3073 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
3074 IID = Intrinsic::x86_avx512_permvar_df_512;
3075 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
3076 IID = Intrinsic::x86_avx512_permvar_di_512;
3077 else if (VecWidth == 128 && EltWidth == 16)
3078 IID = Intrinsic::x86_avx512_permvar_hi_128;
3079 else if (VecWidth == 256 && EltWidth == 16)
3080 IID = Intrinsic::x86_avx512_permvar_hi_256;
3081 else if (VecWidth == 512 && EltWidth == 16)
3082 IID = Intrinsic::x86_avx512_permvar_hi_512;
3083 else if (VecWidth == 128 && EltWidth == 8)
3084 IID = Intrinsic::x86_avx512_permvar_qi_128;
3085 else if (VecWidth == 256 && EltWidth == 8)
3086 IID = Intrinsic::x86_avx512_permvar_qi_256;
3087 else if (VecWidth == 512 && EltWidth == 8)
3088 IID = Intrinsic::x86_avx512_permvar_qi_512;
3089 else
3090 llvm_unreachable("Unexpected intrinsic");
3091 } else if (Name.starts_with("dbpsadbw.")) {
3092 if (VecWidth == 128)
3093 IID = Intrinsic::x86_avx512_dbpsadbw_128;
3094 else if (VecWidth == 256)
3095 IID = Intrinsic::x86_avx512_dbpsadbw_256;
3096 else if (VecWidth == 512)
3097 IID = Intrinsic::x86_avx512_dbpsadbw_512;
3098 else
3099 llvm_unreachable("Unexpected intrinsic");
3100 } else if (Name.starts_with("pmultishift.qb.")) {
3101 if (VecWidth == 128)
3102 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
3103 else if (VecWidth == 256)
3104 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
3105 else if (VecWidth == 512)
3106 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
3107 else
3108 llvm_unreachable("Unexpected intrinsic");
3109 } else if (Name.starts_with("conflict.")) {
3110 if (Name[9] == 'd' && VecWidth == 128)
3111 IID = Intrinsic::x86_avx512_conflict_d_128;
3112 else if (Name[9] == 'd' && VecWidth == 256)
3113 IID = Intrinsic::x86_avx512_conflict_d_256;
3114 else if (Name[9] == 'd' && VecWidth == 512)
3115 IID = Intrinsic::x86_avx512_conflict_d_512;
3116 else if (Name[9] == 'q' && VecWidth == 128)
3117 IID = Intrinsic::x86_avx512_conflict_q_128;
3118 else if (Name[9] == 'q' && VecWidth == 256)
3119 IID = Intrinsic::x86_avx512_conflict_q_256;
3120 else if (Name[9] == 'q' && VecWidth == 512)
3121 IID = Intrinsic::x86_avx512_conflict_q_512;
3122 else
3123 llvm_unreachable("Unexpected intrinsic");
3124 } else if (Name.starts_with("pavg.")) {
3125 if (Name[5] == 'b' && VecWidth == 128)
3126 IID = Intrinsic::x86_sse2_pavg_b;
3127 else if (Name[5] == 'b' && VecWidth == 256)
3128 IID = Intrinsic::x86_avx2_pavg_b;
3129 else if (Name[5] == 'b' && VecWidth == 512)
3130 IID = Intrinsic::x86_avx512_pavg_b_512;
3131 else if (Name[5] == 'w' && VecWidth == 128)
3132 IID = Intrinsic::x86_sse2_pavg_w;
3133 else if (Name[5] == 'w' && VecWidth == 256)
3134 IID = Intrinsic::x86_avx2_pavg_w;
3135 else if (Name[5] == 'w' && VecWidth == 512)
3136 IID = Intrinsic::x86_avx512_pavg_w_512;
3137 else
3138 llvm_unreachable("Unexpected intrinsic");
3139 } else
3140 return false;
3141
3142 SmallVector<Value *, 4> Args(CI.args());
3143 Args.pop_back();
3144 Args.pop_back();
3145 Rep = Builder.CreateIntrinsic(IID, Args);
3146 unsigned NumArgs = CI.arg_size();
3147 Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
3148 CI.getArgOperand(NumArgs - 2));
3149 return true;
3150}
3151
3152/// Upgrade comment in call to inline asm that represents an objc retain release
3153/// marker.
3154void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
3155 size_t Pos;
3156 if (AsmStr->find("mov\tfp") == 0 &&
3157 AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
3158 (Pos = AsmStr->find("# marker")) != std::string::npos) {
3159 AsmStr->replace(Pos, 1, ";");
3160 }
3161}
3162
3163static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
3164 Function *F, IRBuilder<> &Builder) {
3165 Value *Rep = nullptr;
3166
3167 if (Name == "abs.i" || Name == "abs.ll") {
3168 Value *Arg = CI->getArgOperand(0);
3169 Rep = Builder.CreateIntrinsic(Intrinsic::abs, {Arg->getType()},
3170 {Arg, Builder.getTrue()},
3171 /*FMFSource=*/nullptr, "abs");
3172 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
3173 Type *Ty = (Name == "abs.bf16")
3174 ? Builder.getBFloatTy()
3175 : FixedVectorType::get(Builder.getBFloatTy(), 2);
3176 Value *Arg = Builder.CreateBitCast(CI->getArgOperand(0), Ty);
3177 Value *Abs = Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs, Arg);
3178 Rep = Builder.CreateBitCast(Abs, CI->getType());
3179 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
3180 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
3181 : Intrinsic::nvvm_fabs;
3182 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3183 } else if (Name.consume_front("add.")) {
3184 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
3185 auto FAdd = getNVVMFAddUpgrade(Name);
3186 assert(FAdd && "unsupported nvvm.add.* intrinsic");
3187 auto [IID, RoundingMode] = *FAdd;
3188 Value *A = CI->getArgOperand(0);
3189 Rep = Builder.CreateIntrinsic(
3190 A->getType(), IID,
3191 {A, CI->getArgOperand(1),
3192 Builder.getInt32(static_cast<int>(RoundingMode))});
3193 } else if (Name.consume_front("ex2.approx.")) {
3194 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
3195 Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
3196 : Intrinsic::nvvm_ex2_approx;
3197 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3198 } else if (Name.starts_with("atomic.load.add.f32.p") ||
3199 Name.starts_with("atomic.load.add.f64.p")) {
3200 Value *Ptr = CI->getArgOperand(0);
3201 Value *Val = CI->getArgOperand(1);
3202 Rep = Builder.CreateAtomicRMW(
3203 AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3204 CI->getContext().getOrInsertSyncScopeID("device"));
3205 // The default scope for atomic.load.* intrinsics is device
3206 // (= gpu scope in ptx), but the default LLVM atomic scope is
3207 // "system"
3208 } else if (Name.starts_with("atomic.load.inc.32.p") ||
3209 Name.starts_with("atomic.load.dec.32.p")) {
3210 Value *Ptr = CI->getArgOperand(0);
3211 Value *Val = CI->getArgOperand(1);
3212 auto Op = Name.starts_with("atomic.load.inc") ? AtomicRMWInst::UIncWrap
3213 : AtomicRMWInst::UDecWrap;
3214 Rep = Builder.CreateAtomicRMW(
3215 Op, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3216 CI->getContext().getOrInsertSyncScopeID("device"));
3217 // See comment above.
3218 } else if (Name.starts_with("atomic.") && Name.contains(".gen.")) {
3219 // nvvm.atomic.{op}.gen.{i,f}.{cta,sys} -> atomicrmw / cmpxchg.
3220 StringRef Op = Name.substr(StringRef("atomic.").size());
3221 Value *Ptr = CI->getArgOperand(0);
3222 Value *Val = CI->getArgOperand(1);
3223 SyncScope::ID SSID = CI->getContext().getOrInsertSyncScopeID(
3224 Op.contains(".cta.") ? "block" : "");
3225 if (Op.starts_with("cas.")) {
3226 Value *New = CI->getArgOperand(2);
3227 Value *Pair = Builder.CreateAtomicCmpXchg(
3228 Ptr, Val, New, MaybeAlign(), AtomicOrdering::Monotonic,
3229 AtomicOrdering::Monotonic, SSID);
3230 Rep = Builder.CreateExtractValue(Pair, 0);
3231 } else {
3232 // Note we don't upgrade anything to AtomicRMWInst::UMin/UMax. This is
3233 // because we were actually missing those intrinsics!
3234 AtomicRMWInst::BinOp BinOp =
3235 StringSwitch<AtomicRMWInst::BinOp>(Op)
3236 .StartsWith("add.gen.f", AtomicRMWInst::FAdd)
3237 .StartsWith("add.gen.i", AtomicRMWInst::Add)
3238 .StartsWith("exch.", AtomicRMWInst::Xchg)
3239 .StartsWith("max.", AtomicRMWInst::Max)
3240 .StartsWith("min.", AtomicRMWInst::Min)
3241 .StartsWith("inc.", AtomicRMWInst::UIncWrap)
3242 .StartsWith("dec.", AtomicRMWInst::UDecWrap)
3243 .StartsWith("and.", AtomicRMWInst::And)
3244 .StartsWith("or.", AtomicRMWInst::Or)
3245 .StartsWith("xor.", AtomicRMWInst::Xor)
3246 .Default(AtomicRMWInst::BAD_BINOP);
3247 assert(BinOp != AtomicRMWInst::BAD_BINOP &&
3248 "unexpected nvvm scoped atomic intrinsic");
3249 Rep = Builder.CreateAtomicRMW(BinOp, Ptr, Val, MaybeAlign(),
3250 AtomicOrdering::Monotonic, SSID);
3251 }
3252 } else if (Name == "clz.ll") {
3253 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
3254 Value *Arg = CI->getArgOperand(0);
3255 Value *Ctlz = Builder.CreateIntrinsic(Intrinsic::ctlz, {Arg->getType()},
3256 {Arg, Builder.getFalse()},
3257 /*FMFSource=*/nullptr, "ctlz");
3258 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
3259 } else if (Name == "popc.ll") {
3260 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
3261 // i64.
3262 Value *Arg = CI->getArgOperand(0);
3263 Value *Popc = Builder.CreateIntrinsic(Intrinsic::ctpop, {Arg->getType()},
3264 Arg, /*FMFSource=*/nullptr, "ctpop");
3265 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
3266 } else if (Name == "h2f") {
3267 Value *Cast =
3268 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
3269 Rep = Builder.CreateFPExt(Cast, Builder.getFloatTy());
3270 } else if (Name.consume_front("bitcast.") &&
3271 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
3272 Name == "d2ll")) {
3273 Rep = Builder.CreateBitCast(CI->getArgOperand(0), CI->getType());
3274 } else if (Name == "rotate.b32") {
3275 Value *Arg = CI->getOperand(0);
3276 Value *ShiftAmt = CI->getOperand(1);
3277 Rep = Builder.CreateIntrinsic(Builder.getInt32Ty(), Intrinsic::fshl,
3278 {Arg, Arg, ShiftAmt});
3279 } else if (Name == "rotate.b64") {
3280 Type *Int64Ty = Builder.getInt64Ty();
3281 Value *Arg = CI->getOperand(0);
3282 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3283 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3284 {Arg, Arg, ZExtShiftAmt});
3285 } else if (Name == "rotate.right.b64") {
3286 Type *Int64Ty = Builder.getInt64Ty();
3287 Value *Arg = CI->getOperand(0);
3288 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3289 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshr,
3290 {Arg, Arg, ZExtShiftAmt});
3291 } else if (Name == "swap.lo.hi.b64") {
3292 Type *Int64Ty = Builder.getInt64Ty();
3293 Value *Arg = CI->getOperand(0);
3294 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3295 {Arg, Arg, Builder.getInt64(32)});
3296 } else if ((Name.consume_front("ptr.gen.to.") &&
3297 consumeNVVMPtrAddrSpace(Name)) ||
3298 (Name.consume_front("ptr.") && consumeNVVMPtrAddrSpace(Name) &&
3299 Name.starts_with(".to.gen"))) {
3300 Rep = Builder.CreateAddrSpaceCast(CI->getArgOperand(0), CI->getType());
3301 } else if (Name.consume_front("ldg.global")) {
3302 Value *Ptr = CI->getArgOperand(0);
3303 Align PtrAlign = cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue();
3304 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
3305 Value *ASC = Builder.CreateAddrSpaceCast(Ptr, Builder.getPtrTy(1));
3306 Instruction *LD = Builder.CreateAlignedLoad(CI->getType(), ASC, PtrAlign);
3307 MDNode *MD = MDNode::get(Builder.getContext(), {});
3308 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
3309 return LD;
3310 } else if (Name == "tanh.approx.f32") {
3311 // nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
3312 FastMathFlags FMF;
3313 FMF.setApproxFunc();
3314 Rep = Builder.CreateUnaryIntrinsic(Intrinsic::tanh, CI->getArgOperand(0),
3315 FMF);
3316 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {
3317 Value *Arg =
3318 Name.ends_with('0') ? Builder.getInt32(0) : CI->getArgOperand(0);
3319 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all,
3320 {}, {Arg});
3321 } else if (Name == "barrier") {
3322 Rep = Builder.CreateIntrinsic(
3323 Intrinsic::nvvm_barrier_cta_sync_aligned_count, {},
3324 {CI->getArgOperand(0), CI->getArgOperand(1)});
3325 } else if (Name == "barrier.sync") {
3326 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_all, {},
3327 {CI->getArgOperand(0)});
3328 } else if (Name == "barrier.sync.cnt") {
3329 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_count, {},
3330 {CI->getArgOperand(0), CI->getArgOperand(1)});
3331 } else if (Name == "barrier0.popc" || Name == "barrier0.and" ||
3332 Name == "barrier0.or") {
3333 Value *C = CI->getArgOperand(0);
3334 C = Builder.CreateICmpNE(C, Builder.getInt32(0));
3335
3336 Intrinsic::ID IID =
3337 StringSwitch<Intrinsic::ID>(Name)
3338 .Case("barrier0.popc",
3339 Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
3340 .Case("barrier0.and",
3341 Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
3342 .Case("barrier0.or",
3343 Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
3344 Value *Bar = Builder.CreateIntrinsic(IID, {}, {Builder.getInt32(0), C});
3345 Rep = Builder.CreateZExt(Bar, CI->getType());
3346 } else {
3347 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
3348 if (IID != Intrinsic::not_intrinsic &&
3349 isLegacyNVPTXBF16IntSignature(F, IID)) {
3350 rename(F);
3351 Function *NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
3352 SmallVector<Value *, 2> Args;
3353 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
3354 Value *Arg = CI->getArgOperand(I);
3355 Type *OldType = Arg->getType();
3356 Type *NewType = NewFn->getArg(I)->getType();
3357 Args.push_back(
3358 (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
3359 ? Builder.CreateBitCast(Arg, NewType)
3360 : Arg);
3361 }
3362 Rep = Builder.CreateCall(NewFn, Args);
3363 if (F->getReturnType()->isIntegerTy())
3364 Rep = Builder.CreateBitCast(Rep, F->getReturnType());
3365 }
3366 }
3367
3368 return Rep;
3369}
3370
3371static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
3372 IRBuilder<> &Builder) {
3373 LLVMContext &C = F->getContext();
3374 Value *Rep = nullptr;
3375
3376 if (Name.starts_with("sse4a.movnt.")) {
3377 SmallVector<Metadata *, 1> Elts;
3378 Elts.push_back(
3379 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3380 MDNode *Node = MDNode::get(C, Elts);
3381
3382 Value *Arg0 = CI->getArgOperand(0);
3383 Value *Arg1 = CI->getArgOperand(1);
3384
3385 // Nontemporal (unaligned) store of the 0'th element of the float/double
3386 // vector.
3387 Value *Extract =
3388 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
3389
3390 StoreInst *SI = Builder.CreateAlignedStore(Extract, Arg0, Align(1));
3391 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3392 } else if (Name.starts_with("avx.movnt.") ||
3393 Name.starts_with("avx512.storent.")) {
3394 SmallVector<Metadata *, 1> Elts;
3395 Elts.push_back(
3396 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3397 MDNode *Node = MDNode::get(C, Elts);
3398
3399 Value *Arg0 = CI->getArgOperand(0);
3400 Value *Arg1 = CI->getArgOperand(1);
3401
3402 StoreInst *SI = Builder.CreateAlignedStore(
3403 Arg1, Arg0,
3404 Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3405 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3406 } else if (Name == "sse2.storel.dq") {
3407 Value *Arg0 = CI->getArgOperand(0);
3408 Value *Arg1 = CI->getArgOperand(1);
3409
3410 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
3411 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
3412 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
3413 Builder.CreateAlignedStore(Elt, Arg0, Align(1));
3414 } else if (Name.starts_with("sse.storeu.") ||
3415 Name.starts_with("sse2.storeu.") ||
3416 Name.starts_with("avx.storeu.")) {
3417 Value *Arg0 = CI->getArgOperand(0);
3418 Value *Arg1 = CI->getArgOperand(1);
3419 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
3420 } else if (Name == "avx512.mask.store.ss") {
3421 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
3422 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3423 Mask, false);
3424 } else if (Name.starts_with("avx512.mask.store")) {
3425 // "avx512.mask.storeu." or "avx512.mask.store."
3426 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
3427 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3428 CI->getArgOperand(2), Aligned);
3429 } else if (Name.starts_with("sse2.pcmp") || Name.starts_with("avx2.pcmp")) {
3430 // Upgrade packed integer vector compare intrinsics to compare instructions.
3431 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
3432 bool CmpEq = Name[9] == 'e';
3433 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
3434 CI->getArgOperand(0), CI->getArgOperand(1));
3435 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
3436 } else if (Name.starts_with("avx512.broadcastm")) {
3437 Type *ExtTy = Type::getInt32Ty(C);
3438 if (CI->getOperand(0)->getType()->isIntegerTy(8))
3439 ExtTy = Type::getInt64Ty(C);
3440 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
3441 ExtTy->getPrimitiveSizeInBits();
3442 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
3443 Rep = Builder.CreateVectorSplat(NumElts, Rep);
3444 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
3445 Value *Vec = CI->getArgOperand(0);
3446 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
3447 Elt0 = Builder.CreateIntrinsic(Intrinsic::sqrt, Elt0->getType(), Elt0);
3448 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
3449 } else if (Name.starts_with("avx.sqrt.p") ||
3450 Name.starts_with("sse2.sqrt.p") ||
3451 Name.starts_with("sse.sqrt.p")) {
3452 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3453 {CI->getArgOperand(0)});
3454 } else if (Name.starts_with("avx512.mask.sqrt.p")) {
3455 if (CI->arg_size() == 4 &&
3456 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3457 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3458 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
3459 : Intrinsic::x86_avx512_sqrt_pd_512;
3460
3461 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(3)};
3462 Rep = Builder.CreateIntrinsic(IID, Args);
3463 } else {
3464 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3465 {CI->getArgOperand(0)});
3466 }
3467 Rep =
3468 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3469 } else if (Name.starts_with("avx512.ptestm") ||
3470 Name.starts_with("avx512.ptestnm")) {
3471 Value *Op0 = CI->getArgOperand(0);
3472 Value *Op1 = CI->getArgOperand(1);
3473 Value *Mask = CI->getArgOperand(2);
3474 Rep = Builder.CreateAnd(Op0, Op1);
3475 llvm::Type *Ty = Op0->getType();
3476 Value *Zero = llvm::Constant::getNullValue(Ty);
3477 ICmpInst::Predicate Pred = Name.starts_with("avx512.ptestm")
3478 ? ICmpInst::ICMP_NE
3479 : ICmpInst::ICMP_EQ;
3480 Rep = Builder.CreateICmp(Pred, Rep, Zero);
3481 Rep = applyX86MaskOn1BitsVec(Builder, Rep, Mask);
3482 } else if (Name.starts_with("avx512.mask.pbroadcast")) {
3483 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
3484 ->getNumElements();
3485 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
3486 Rep =
3487 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3488 } else if (Name.starts_with("avx512.kunpck")) {
3489 unsigned NumElts = CI->getType()->getScalarSizeInBits();
3490 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
3491 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
3492 int Indices[64];
3493 for (unsigned i = 0; i != NumElts; ++i)
3494 Indices[i] = i;
3495
3496 // First extract half of each vector. This gives better codegen than
3497 // doing it in a single shuffle.
3498 LHS = Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));
3499 RHS = Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));
3500 // Concat the vectors.
3501 // NOTE: Operands have to be swapped to match intrinsic definition.
3502 Rep = Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));
3503 Rep = Builder.CreateBitCast(Rep, CI->getType());
3504 } else if (Name == "avx512.kand.w") {
3505 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3506 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3507 Rep = Builder.CreateAnd(LHS, RHS);
3508 Rep = Builder.CreateBitCast(Rep, CI->getType());
3509 } else if (Name == "avx512.kandn.w") {
3510 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3511 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3512 LHS = Builder.CreateNot(LHS);
3513 Rep = Builder.CreateAnd(LHS, RHS);
3514 Rep = Builder.CreateBitCast(Rep, CI->getType());
3515 } else if (Name == "avx512.kor.w") {
3516 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3517 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3518 Rep = Builder.CreateOr(LHS, RHS);
3519 Rep = Builder.CreateBitCast(Rep, CI->getType());
3520 } else if (Name == "avx512.kxor.w") {
3521 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3522 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3523 Rep = Builder.CreateXor(LHS, RHS);
3524 Rep = Builder.CreateBitCast(Rep, CI->getType());
3525 } else if (Name == "avx512.kxnor.w") {
3526 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3527 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3528 LHS = Builder.CreateNot(LHS);
3529 Rep = Builder.CreateXor(LHS, RHS);
3530 Rep = Builder.CreateBitCast(Rep, CI->getType());
3531 } else if (Name == "avx512.knot.w") {
3532 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3533 Rep = Builder.CreateNot(Rep);
3534 Rep = Builder.CreateBitCast(Rep, CI->getType());
3535 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
3536 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3537 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3538 Rep = Builder.CreateOr(LHS, RHS);
3539 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
3540 Value *C;
3541 if (Name[14] == 'c')
3542 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
3543 else
3544 C = ConstantInt::getNullValue(Builder.getInt16Ty());
3545 Rep = Builder.CreateICmpEQ(Rep, C);
3546 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
3547 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
3548 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
3549 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
3550 Name == "sse.div.ss" || Name == "sse2.div.sd") {
3551 Type *I32Ty = Type::getInt32Ty(C);
3552 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
3553 ConstantInt::get(I32Ty, 0));
3554 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
3555 ConstantInt::get(I32Ty, 0));
3556 Value *EltOp;
3557 if (Name.contains(".add."))
3558 EltOp = Builder.CreateFAdd(Elt0, Elt1);
3559 else if (Name.contains(".sub."))
3560 EltOp = Builder.CreateFSub(Elt0, Elt1);
3561 else if (Name.contains(".mul."))
3562 EltOp = Builder.CreateFMul(Elt0, Elt1);
3563 else
3564 EltOp = Builder.CreateFDiv(Elt0, Elt1);
3565 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
3566 ConstantInt::get(I32Ty, 0));
3567 } else if (Name.starts_with("avx512.mask.pcmp")) {
3568 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3569 bool CmpEq = Name[16] == 'e';
3570 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
3571 } else if (Name.starts_with("avx512.mask.vpshufbitqmb.")) {
3572 Type *OpTy = CI->getArgOperand(0)->getType();
3573 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3574 Intrinsic::ID IID;
3575 switch (VecWidth) {
3576 default:
3577 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3578 break;
3579 case 128:
3580 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3581 break;
3582 case 256:
3583 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3584 break;
3585 case 512:
3586 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3587 break;
3588 }
3589
3590 Rep =
3591 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3592 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3593 } else if (Name.starts_with("avx512.mask.fpclass.p")) {
3594 Type *OpTy = CI->getArgOperand(0)->getType();
3595 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3596 unsigned EltWidth = OpTy->getScalarSizeInBits();
3597 Intrinsic::ID IID;
3598 if (VecWidth == 128 && EltWidth == 32)
3599 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3600 else if (VecWidth == 256 && EltWidth == 32)
3601 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3602 else if (VecWidth == 512 && EltWidth == 32)
3603 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3604 else if (VecWidth == 128 && EltWidth == 64)
3605 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3606 else if (VecWidth == 256 && EltWidth == 64)
3607 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3608 else if (VecWidth == 512 && EltWidth == 64)
3609 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3610 else
3611 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3612
3613 Rep =
3614 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3615 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3616 } else if (Name.starts_with("avx512.cmp.p")) {
3617 SmallVector<Value *, 4> Args(CI->args());
3618 Type *OpTy = Args[0]->getType();
3619 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3620 unsigned EltWidth = OpTy->getScalarSizeInBits();
3621 Intrinsic::ID IID;
3622 if (VecWidth == 128 && EltWidth == 32)
3623 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3624 else if (VecWidth == 256 && EltWidth == 32)
3625 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3626 else if (VecWidth == 512 && EltWidth == 32)
3627 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3628 else if (VecWidth == 128 && EltWidth == 64)
3629 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3630 else if (VecWidth == 256 && EltWidth == 64)
3631 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3632 else if (VecWidth == 512 && EltWidth == 64)
3633 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3634 else
3635 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3636
3637 Value *Mask = Constant::getAllOnesValue(CI->getType());
3638 if (VecWidth == 512)
3639 std::swap(Mask, Args.back());
3640 Args.push_back(Mask);
3641
3642 Rep = Builder.CreateIntrinsic(IID, Args);
3643 } else if (Name.starts_with("avx512.mask.cmp.")) {
3644 // Integer compare intrinsics.
3645 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3646 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
3647 } else if (Name.starts_with("avx512.mask.ucmp.")) {
3648 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3649 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
3650 } else if (Name.starts_with("avx512.cvtb2mask.") ||
3651 Name.starts_with("avx512.cvtw2mask.") ||
3652 Name.starts_with("avx512.cvtd2mask.") ||
3653 Name.starts_with("avx512.cvtq2mask.")) {
3654 Value *Op = CI->getArgOperand(0);
3655 Value *Zero = llvm::Constant::getNullValue(Op->getType());
3656 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
3657 Rep = applyX86MaskOn1BitsVec(Builder, Rep, nullptr);
3658 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
3659 Name == "ssse3.pabs.d.128" || Name.starts_with("avx2.pabs") ||
3660 Name.starts_with("avx512.mask.pabs")) {
3661 Rep = upgradeAbs(Builder, *CI);
3662 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
3663 Name == "sse41.pmaxsd" || Name.starts_with("avx2.pmaxs") ||
3664 Name.starts_with("avx512.mask.pmaxs")) {
3665 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
3666 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
3667 Name == "sse41.pmaxud" || Name.starts_with("avx2.pmaxu") ||
3668 Name.starts_with("avx512.mask.pmaxu")) {
3669 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
3670 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
3671 Name == "sse41.pminsd" || Name.starts_with("avx2.pmins") ||
3672 Name.starts_with("avx512.mask.pmins")) {
3673 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
3674 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
3675 Name == "sse41.pminud" || Name.starts_with("avx2.pminu") ||
3676 Name.starts_with("avx512.mask.pminu")) {
3677 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
3678 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
3679 Name == "avx512.pmulu.dq.512" ||
3680 Name.starts_with("avx512.mask.pmulu.dq.")) {
3681 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ false);
3682 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
3683 Name == "avx512.pmul.dq.512" ||
3684 Name.starts_with("avx512.mask.pmul.dq.")) {
3685 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ true);
3686 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
3687 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
3688 Rep =
3689 Builder.CreateSIToFP(CI->getArgOperand(1),
3690 cast<VectorType>(CI->getType())->getElementType());
3691 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3692 } else if (Name == "avx512.cvtusi2sd") {
3693 Rep =
3694 Builder.CreateUIToFP(CI->getArgOperand(1),
3695 cast<VectorType>(CI->getType())->getElementType());
3696 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3697 } else if (Name == "sse2.cvtss2sd") {
3698 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
3699 Rep = Builder.CreateFPExt(
3700 Rep, cast<VectorType>(CI->getType())->getElementType());
3701 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3702 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
3703 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
3704 Name.starts_with("avx512.mask.cvtdq2pd.") ||
3705 Name.starts_with("avx512.mask.cvtudq2pd.") ||
3706 Name.starts_with("avx512.mask.cvtdq2ps.") ||
3707 Name.starts_with("avx512.mask.cvtudq2ps.") ||
3708 Name.starts_with("avx512.mask.cvtqq2pd.") ||
3709 Name.starts_with("avx512.mask.cvtuqq2pd.") ||
3710 Name == "avx512.mask.cvtqq2ps.256" ||
3711 Name == "avx512.mask.cvtqq2ps.512" ||
3712 Name == "avx512.mask.cvtuqq2ps.256" ||
3713 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
3714 Name == "avx.cvt.ps2.pd.256" ||
3715 Name == "avx512.mask.cvtps2pd.128" ||
3716 Name == "avx512.mask.cvtps2pd.256") {
3717 auto *DstTy = cast<FixedVectorType>(CI->getType());
3718 Rep = CI->getArgOperand(0);
3719 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3720
3721 unsigned NumDstElts = DstTy->getNumElements();
3722 if (NumDstElts < SrcTy->getNumElements()) {
3723 assert(NumDstElts == 2 && "Unexpected vector size");
3724 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
3725 }
3726
3727 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3728 bool IsUnsigned = Name.contains("cvtu");
3729 if (IsPS2PD)
3730 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
3731 else if (CI->arg_size() == 4 &&
3732 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3733 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3734 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3735 : Intrinsic::x86_avx512_sitofp_round;
3736 Rep = Builder.CreateIntrinsic(IID, {DstTy, SrcTy},
3737 {Rep, CI->getArgOperand(3)});
3738 } else {
3739 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
3740 : Builder.CreateSIToFP(Rep, DstTy, "cvt");
3741 }
3742
3743 if (CI->arg_size() >= 3)
3744 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3745 CI->getArgOperand(1));
3746 } else if (Name.starts_with("avx512.mask.vcvtph2ps.") ||
3747 Name.starts_with("vcvtph2ps.")) {
3748 auto *DstTy = cast<FixedVectorType>(CI->getType());
3749 Rep = CI->getArgOperand(0);
3750 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3751 unsigned NumDstElts = DstTy->getNumElements();
3752 if (NumDstElts != SrcTy->getNumElements()) {
3753 assert(NumDstElts == 4 && "Unexpected vector size");
3754 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
3755 }
3756 Rep = Builder.CreateBitCast(
3757 Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
3758 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
3759 if (CI->arg_size() >= 3)
3760 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3761 CI->getArgOperand(1));
3762 } else if (Name.starts_with("avx512.mask.load")) {
3763 // "avx512.mask.loadu." or "avx512.mask.load."
3764 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
3765 Rep = upgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3766 CI->getArgOperand(2), Aligned);
3767 } else if (Name.starts_with("avx512.mask.expand.load.")) {
3768 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3769 auto *PtrTy = CI->getOperand(0)->getType();
3770 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3771 ResultTy->getNumElements());
3772 Rep = Builder.CreateIntrinsic(
3773 Intrinsic::masked_expandload, {ResultTy, PtrTy},
3774 {CI->getOperand(0), MaskVec, CI->getOperand(1)});
3775 } else if (Name.starts_with("avx512.mask.compress.store.")) {
3776 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
3777 auto *PtrTy = CI->getArgOperand(0)->getType();
3778 Value *MaskVec =
3779 getX86MaskVec(Builder, CI->getArgOperand(2),
3780 cast<FixedVectorType>(ResultTy)->getNumElements());
3781 Rep = Builder.CreateIntrinsic(
3782 Intrinsic::masked_compressstore, {ResultTy, PtrTy},
3783 {CI->getArgOperand(1), CI->getArgOperand(0), MaskVec});
3784 } else if (Name.starts_with("avx512.mask.compress.") ||
3785 Name.starts_with("avx512.mask.expand.")) {
3786 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3787
3788 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3789 ResultTy->getNumElements());
3790
3791 bool IsCompress = Name[12] == 'c';
3792 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3793 : Intrinsic::x86_avx512_mask_expand;
3794 Rep = Builder.CreateIntrinsic(
3795 IID, ResultTy, {CI->getOperand(0), CI->getOperand(1), MaskVec});
3796 } else if (Name.starts_with("xop.vpcom")) {
3797 bool IsSigned;
3798 if (Name.ends_with("ub") || Name.ends_with("uw") || Name.ends_with("ud") ||
3799 Name.ends_with("uq"))
3800 IsSigned = false;
3801 else if (Name.ends_with("b") || Name.ends_with("w") ||
3802 Name.ends_with("d") || Name.ends_with("q"))
3803 IsSigned = true;
3804 else
3805 reportFatalUsageErrorWithCI("Intrinsic has unknown suffix", CI);
3806
3807 unsigned Imm;
3808 if (CI->arg_size() == 3) {
3809 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3810 } else {
3811 Name = Name.substr(9); // strip off "xop.vpcom"
3812 if (Name.starts_with("lt"))
3813 Imm = 0;
3814 else if (Name.starts_with("le"))
3815 Imm = 1;
3816 else if (Name.starts_with("gt"))
3817 Imm = 2;
3818 else if (Name.starts_with("ge"))
3819 Imm = 3;
3820 else if (Name.starts_with("eq"))
3821 Imm = 4;
3822 else if (Name.starts_with("ne"))
3823 Imm = 5;
3824 else if (Name.starts_with("false"))
3825 Imm = 6;
3826 else if (Name.starts_with("true"))
3827 Imm = 7;
3828 else
3829 llvm_unreachable("Unknown condition");
3830 }
3831
3832 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
3833 } else if (Name.starts_with("xop.vpcmov")) {
3834 Value *Sel = CI->getArgOperand(2);
3835 Value *NotSel = Builder.CreateNot(Sel);
3836 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
3837 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
3838 Rep = Builder.CreateOr(Sel0, Sel1);
3839 } else if (Name.starts_with("xop.vprot") || Name.starts_with("avx512.prol") ||
3840 Name.starts_with("avx512.mask.prol")) {
3841 Rep = upgradeX86Rotate(Builder, *CI, false);
3842 } else if (Name.starts_with("avx512.pror") ||
3843 Name.starts_with("avx512.mask.pror")) {
3844 Rep = upgradeX86Rotate(Builder, *CI, true);
3845 } else if (Name.starts_with("avx512.vpshld.") ||
3846 Name.starts_with("avx512.mask.vpshld") ||
3847 Name.starts_with("avx512.maskz.vpshld")) {
3848 bool ZeroMask = Name[11] == 'z';
3849 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
3850 } else if (Name.starts_with("avx512.vpshrd.") ||
3851 Name.starts_with("avx512.mask.vpshrd") ||
3852 Name.starts_with("avx512.maskz.vpshrd")) {
3853 bool ZeroMask = Name[11] == 'z';
3854 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
3855 } else if (Name == "sse42.crc32.64.8") {
3856 Value *Trunc0 =
3857 Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
3858 Rep = Builder.CreateIntrinsic(Intrinsic::x86_sse42_crc32_32_8,
3859 {Trunc0, CI->getArgOperand(1)});
3860 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
3861 } else if (Name.starts_with("avx.vbroadcast.s") ||
3862 Name.starts_with("avx512.vbroadcast.s")) {
3863 // Replace broadcasts with a series of insertelements.
3864 auto *VecTy = cast<FixedVectorType>(CI->getType());
3865 Type *EltTy = VecTy->getElementType();
3866 unsigned EltNum = VecTy->getNumElements();
3867 Value *Load = Builder.CreateLoad(EltTy, CI->getArgOperand(0));
3868 Type *I32Ty = Type::getInt32Ty(C);
3869 Rep = PoisonValue::get(VecTy);
3870 for (unsigned I = 0; I < EltNum; ++I)
3871 Rep = Builder.CreateInsertElement(Rep, Load, ConstantInt::get(I32Ty, I));
3872 } else if (Name.starts_with("sse41.pmovsx") ||
3873 Name.starts_with("sse41.pmovzx") ||
3874 Name.starts_with("avx2.pmovsx") ||
3875 Name.starts_with("avx2.pmovzx") ||
3876 Name.starts_with("avx512.mask.pmovsx") ||
3877 Name.starts_with("avx512.mask.pmovzx")) {
3878 auto *DstTy = cast<FixedVectorType>(CI->getType());
3879 unsigned NumDstElts = DstTy->getNumElements();
3880
3881 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
3882 SmallVector<int, 8> ShuffleMask(NumDstElts);
3883 for (unsigned i = 0; i != NumDstElts; ++i)
3884 ShuffleMask[i] = i;
3885
3886 Value *SV = Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
3887
3888 bool DoSext = Name.contains("pmovsx");
3889 Rep =
3890 DoSext ? Builder.CreateSExt(SV, DstTy) : Builder.CreateZExt(SV, DstTy);
3891 // If there are 3 arguments, it's a masked intrinsic so we need a select.
3892 if (CI->arg_size() == 3)
3893 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3894 CI->getArgOperand(1));
3895 } else if (Name == "avx512.mask.pmov.qd.256" ||
3896 Name == "avx512.mask.pmov.qd.512" ||
3897 Name == "avx512.mask.pmov.wb.256" ||
3898 Name == "avx512.mask.pmov.wb.512") {
3899 Type *Ty = CI->getArgOperand(1)->getType();
3900 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
3901 Rep =
3902 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3903 } else if (Name.starts_with("avx.vbroadcastf128") ||
3904 Name == "avx2.vbroadcasti128") {
3905 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
3906 Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
3907 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
3908 auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
3909 Value *Load = Builder.CreateAlignedLoad(VT, CI->getArgOperand(0), Align(1));
3910 if (NumSrcElts == 2)
3911 Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
3912 else
3913 Rep = Builder.CreateShuffleVector(Load,
3914 ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
3915 } else if (Name.starts_with("avx512.mask.shuf.i") ||
3916 Name.starts_with("avx512.mask.shuf.f")) {
3917 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3918 Type *VT = CI->getType();
3919 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
3920 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
3921 unsigned ControlBitsMask = NumLanes - 1;
3922 unsigned NumControlBits = NumLanes / 2;
3923 SmallVector<int, 8> ShuffleMask(0);
3924
3925 for (unsigned l = 0; l != NumLanes; ++l) {
3926 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
3927 // We actually need the other source.
3928 if (l >= NumLanes / 2)
3929 LaneMask += NumLanes;
3930 for (unsigned i = 0; i != NumElementsInLane; ++i)
3931 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
3932 }
3933 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
3934 CI->getArgOperand(1), ShuffleMask);
3935 Rep =
3936 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
3937 } else if (Name.starts_with("avx512.mask.broadcastf") ||
3938 Name.starts_with("avx512.mask.broadcasti")) {
3939 unsigned NumSrcElts = cast<FixedVectorType>(CI->getArgOperand(0)->getType())
3940 ->getNumElements();
3941 unsigned NumDstElts =
3942 cast<FixedVectorType>(CI->getType())->getNumElements();
3943
3944 SmallVector<int, 8> ShuffleMask(NumDstElts);
3945 for (unsigned i = 0; i != NumDstElts; ++i)
3946 ShuffleMask[i] = i % NumSrcElts;
3947
3948 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
3949 CI->getArgOperand(0), ShuffleMask);
3950 Rep =
3951 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3952 } else if (Name.starts_with("avx2.pbroadcast") ||
3953 Name.starts_with("avx2.vbroadcast") ||
3954 Name.starts_with("avx512.pbroadcast") ||
3955 Name.starts_with("avx512.mask.broadcast.s")) {
3956 // Replace vp?broadcasts with a vector shuffle.
3957 Value *Op = CI->getArgOperand(0);
3958 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
3959 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
3960 SmallVector<int, 8> M;
3961 ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
3962 Rep = Builder.CreateShuffleVector(Op, M);
3963
3964 if (CI->arg_size() == 3)
3965 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3966 CI->getArgOperand(1));
3967 } else if (Name.starts_with("sse2.padds.") ||
3968 Name.starts_with("avx2.padds.") ||
3969 Name.starts_with("avx512.padds.") ||
3970 Name.starts_with("avx512.mask.padds.")) {
3971 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
3972 } else if (Name.starts_with("sse2.psubs.") ||
3973 Name.starts_with("avx2.psubs.") ||
3974 Name.starts_with("avx512.psubs.") ||
3975 Name.starts_with("avx512.mask.psubs.")) {
3976 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
3977 } else if (Name.starts_with("sse2.paddus.") ||
3978 Name.starts_with("avx2.paddus.") ||
3979 Name.starts_with("avx512.mask.paddus.")) {
3980 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
3981 } else if (Name.starts_with("sse2.psubus.") ||
3982 Name.starts_with("avx2.psubus.") ||
3983 Name.starts_with("avx512.mask.psubus.")) {
3984 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
3985 } else if (Name.starts_with("avx512.mask.palignr.")) {
3986 Rep = upgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
3987 CI->getArgOperand(1), CI->getArgOperand(2),
3988 CI->getArgOperand(3), CI->getArgOperand(4),
3989 false);
3990 } else if (Name.starts_with("avx512.mask.valign.")) {
3991 Rep = upgradeX86ALIGNIntrinsics(
3992 Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3993 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), true);
3994 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
3995 // 128/256-bit shift left specified in bits.
3996 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3997 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
3998 Shift / 8); // Shift is in bits.
3999 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
4000 // 128/256-bit shift right specified in bits.
4001 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4002 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
4003 Shift / 8); // Shift is in bits.
4004 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
4005 Name == "avx512.psll.dq.512") {
4006 // 128/256/512-bit shift left specified in bytes.
4007 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4008 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
4009 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
4010 Name == "avx512.psrl.dq.512") {
4011 // 128/256/512-bit shift right specified in bytes.
4012 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4013 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
4014 } else if (Name == "sse41.pblendw" || Name.starts_with("sse41.blendp") ||
4015 Name.starts_with("avx.blend.p") || Name == "avx2.pblendw" ||
4016 Name.starts_with("avx2.pblendd.")) {
4017 Value *Op0 = CI->getArgOperand(0);
4018 Value *Op1 = CI->getArgOperand(1);
4019 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4020 auto *VecTy = cast<FixedVectorType>(CI->getType());
4021 unsigned NumElts = VecTy->getNumElements();
4022
4023 SmallVector<int, 16> Idxs(NumElts);
4024 for (unsigned i = 0; i != NumElts; ++i)
4025 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
4026
4027 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4028 } else if (Name.starts_with("avx.vinsertf128.") ||
4029 Name == "avx2.vinserti128" ||
4030 Name.starts_with("avx512.mask.insert")) {
4031 Value *Op0 = CI->getArgOperand(0);
4032 Value *Op1 = CI->getArgOperand(1);
4033 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4034 unsigned DstNumElts =
4035 cast<FixedVectorType>(CI->getType())->getNumElements();
4036 unsigned SrcNumElts =
4037 cast<FixedVectorType>(Op1->getType())->getNumElements();
4038 unsigned Scale = DstNumElts / SrcNumElts;
4039
4040 // Mask off the high bits of the immediate value; hardware ignores those.
4041 Imm = Imm % Scale;
4042
4043 // Extend the second operand into a vector the size of the destination.
4044 SmallVector<int, 8> Idxs(DstNumElts);
4045 for (unsigned i = 0; i != SrcNumElts; ++i)
4046 Idxs[i] = i;
4047 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
4048 Idxs[i] = SrcNumElts;
4049 Rep = Builder.CreateShuffleVector(Op1, Idxs);
4050
4051 // Insert the second operand into the first operand.
4052
4053 // Note that there is no guarantee that instruction lowering will actually
4054 // produce a vinsertf128 instruction for the created shuffles. In
4055 // particular, the 0 immediate case involves no lane changes, so it can
4056 // be handled as a blend.
4057
4058 // Example of shuffle mask for 32-bit elements:
4059 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
4060 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
4061
4062 // First fill with identify mask.
4063 for (unsigned i = 0; i != DstNumElts; ++i)
4064 Idxs[i] = i;
4065 // Then replace the elements where we need to insert.
4066 for (unsigned i = 0; i != SrcNumElts; ++i)
4067 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
4068 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
4069
4070 // If the intrinsic has a mask operand, handle that.
4071 if (CI->arg_size() == 5)
4072 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep,
4073 CI->getArgOperand(3));
4074 } else if (Name.starts_with("avx.vextractf128.") ||
4075 Name == "avx2.vextracti128" ||
4076 Name.starts_with("avx512.mask.vextract")) {
4077 Value *Op0 = CI->getArgOperand(0);
4078 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4079 unsigned DstNumElts =
4080 cast<FixedVectorType>(CI->getType())->getNumElements();
4081 unsigned SrcNumElts =
4082 cast<FixedVectorType>(Op0->getType())->getNumElements();
4083 unsigned Scale = SrcNumElts / DstNumElts;
4084
4085 // Mask off the high bits of the immediate value; hardware ignores those.
4086 Imm = Imm % Scale;
4087
4088 // Get indexes for the subvector of the input vector.
4089 SmallVector<int, 8> Idxs(DstNumElts);
4090 for (unsigned i = 0; i != DstNumElts; ++i) {
4091 Idxs[i] = i + (Imm * DstNumElts);
4092 }
4093 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4094
4095 // If the intrinsic has a mask operand, handle that.
4096 if (CI->arg_size() == 4)
4097 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4098 CI->getArgOperand(2));
4099 } else if (Name.starts_with("avx512.mask.perm.df.") ||
4100 Name.starts_with("avx512.mask.perm.di.")) {
4101 Value *Op0 = CI->getArgOperand(0);
4102 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4103 auto *VecTy = cast<FixedVectorType>(CI->getType());
4104 unsigned NumElts = VecTy->getNumElements();
4105
4106 SmallVector<int, 8> Idxs(NumElts);
4107 for (unsigned i = 0; i != NumElts; ++i)
4108 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
4109
4110 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4111
4112 if (CI->arg_size() == 4)
4113 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4114 CI->getArgOperand(2));
4115 } else if (Name.starts_with("avx.vperm2f128.") || Name == "avx2.vperm2i128") {
4116 // The immediate permute control byte looks like this:
4117 // [1:0] - select 128 bits from sources for low half of destination
4118 // [2] - ignore
4119 // [3] - zero low half of destination
4120 // [5:4] - select 128 bits from sources for high half of destination
4121 // [6] - ignore
4122 // [7] - zero high half of destination
4123
4124 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4125
4126 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4127 unsigned HalfSize = NumElts / 2;
4128 SmallVector<int, 8> ShuffleMask(NumElts);
4129
4130 // Determine which operand(s) are actually in use for this instruction.
4131 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4132 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4133
4134 // If needed, replace operands based on zero mask.
4135 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
4136 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
4137
4138 // Permute low half of result.
4139 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
4140 for (unsigned i = 0; i < HalfSize; ++i)
4141 ShuffleMask[i] = StartIndex + i;
4142
4143 // Permute high half of result.
4144 StartIndex = (Imm & 0x10) ? HalfSize : 0;
4145 for (unsigned i = 0; i < HalfSize; ++i)
4146 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
4147
4148 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
4149
4150 } else if (Name.starts_with("avx.vpermil.") || Name == "sse2.pshuf.d" ||
4151 Name.starts_with("avx512.mask.vpermil.p") ||
4152 Name.starts_with("avx512.mask.pshuf.d.")) {
4153 Value *Op0 = CI->getArgOperand(0);
4154 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4155 auto *VecTy = cast<FixedVectorType>(CI->getType());
4156 unsigned NumElts = VecTy->getNumElements();
4157 // Calculate the size of each index in the immediate.
4158 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
4159 unsigned IdxMask = ((1 << IdxSize) - 1);
4160
4161 SmallVector<int, 8> Idxs(NumElts);
4162 // Lookup the bits for this element, wrapping around the immediate every
4163 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
4164 // to offset by the first index of each group.
4165 for (unsigned i = 0; i != NumElts; ++i)
4166 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
4167
4168 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4169
4170 if (CI->arg_size() == 4)
4171 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4172 CI->getArgOperand(2));
4173 } else if (Name == "sse2.pshufl.w" ||
4174 Name.starts_with("avx512.mask.pshufl.w.")) {
4175 Value *Op0 = CI->getArgOperand(0);
4176 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4177 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4178
4179 if (Name == "sse2.pshufl.w" && NumElts % 8 != 0)
4180 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4181
4182 SmallVector<int, 16> Idxs(NumElts);
4183 for (unsigned l = 0; l != NumElts; l += 8) {
4184 for (unsigned i = 0; i != 4; ++i)
4185 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
4186 for (unsigned i = 4; i != 8; ++i)
4187 Idxs[i + l] = i + l;
4188 }
4189
4190 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4191
4192 if (CI->arg_size() == 4)
4193 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4194 CI->getArgOperand(2));
4195 } else if (Name == "sse2.pshufh.w" ||
4196 Name.starts_with("avx512.mask.pshufh.w.")) {
4197 Value *Op0 = CI->getArgOperand(0);
4198 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4199 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4200
4201 if (Name == "sse2.pshufh.w" && NumElts % 8 != 0)
4202 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4203
4204 SmallVector<int, 16> Idxs(NumElts);
4205 for (unsigned l = 0; l != NumElts; l += 8) {
4206 for (unsigned i = 0; i != 4; ++i)
4207 Idxs[i + l] = i + l;
4208 for (unsigned i = 0; i != 4; ++i)
4209 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
4210 }
4211
4212 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4213
4214 if (CI->arg_size() == 4)
4215 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4216 CI->getArgOperand(2));
4217 } else if (Name.starts_with("avx512.mask.shuf.p")) {
4218 Value *Op0 = CI->getArgOperand(0);
4219 Value *Op1 = CI->getArgOperand(1);
4220 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4221 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4222
4223 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4224 unsigned HalfLaneElts = NumLaneElts / 2;
4225
4226 SmallVector<int, 16> Idxs(NumElts);
4227 for (unsigned i = 0; i != NumElts; ++i) {
4228 // Base index is the starting element of the lane.
4229 Idxs[i] = i - (i % NumLaneElts);
4230 // If we are half way through the lane switch to the other source.
4231 if ((i % NumLaneElts) >= HalfLaneElts)
4232 Idxs[i] += NumElts;
4233 // Now select the specific element. By adding HalfLaneElts bits from
4234 // the immediate. Wrapping around the immediate every 8-bits.
4235 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
4236 }
4237
4238 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4239
4240 Rep =
4241 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
4242 } else if (Name.starts_with("avx512.mask.movddup") ||
4243 Name.starts_with("avx512.mask.movshdup") ||
4244 Name.starts_with("avx512.mask.movsldup")) {
4245 Value *Op0 = CI->getArgOperand(0);
4246 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4247 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4248
4249 unsigned Offset = 0;
4250 if (Name.starts_with("avx512.mask.movshdup."))
4251 Offset = 1;
4252
4253 SmallVector<int, 16> Idxs(NumElts);
4254 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
4255 for (unsigned i = 0; i != NumLaneElts; i += 2) {
4256 Idxs[i + l + 0] = i + l + Offset;
4257 Idxs[i + l + 1] = i + l + Offset;
4258 }
4259
4260 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4261
4262 Rep =
4263 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4264 } else if (Name.starts_with("avx512.mask.punpckl") ||
4265 Name.starts_with("avx512.mask.unpckl.")) {
4266 Value *Op0 = CI->getArgOperand(0);
4267 Value *Op1 = CI->getArgOperand(1);
4268 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4269 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4270
4271 SmallVector<int, 64> Idxs(NumElts);
4272 for (int l = 0; l != NumElts; l += NumLaneElts)
4273 for (int i = 0; i != NumLaneElts; ++i)
4274 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
4275
4276 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4277
4278 Rep =
4279 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4280 } else if (Name.starts_with("avx512.mask.punpckh") ||
4281 Name.starts_with("avx512.mask.unpckh.")) {
4282 Value *Op0 = CI->getArgOperand(0);
4283 Value *Op1 = CI->getArgOperand(1);
4284 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4285 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4286
4287 SmallVector<int, 64> Idxs(NumElts);
4288 for (int l = 0; l != NumElts; l += NumLaneElts)
4289 for (int i = 0; i != NumLaneElts; ++i)
4290 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
4291
4292 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4293
4294 Rep =
4295 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4296 } else if (Name.starts_with("avx512.mask.and.") ||
4297 Name.starts_with("avx512.mask.pand.")) {
4298 VectorType *FTy = cast<VectorType>(CI->getType());
4299 VectorType *ITy = VectorType::getInteger(FTy);
4300 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4301 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4302 Rep = Builder.CreateBitCast(Rep, FTy);
4303 Rep =
4304 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4305 } else if (Name.starts_with("avx512.mask.andn.") ||
4306 Name.starts_with("avx512.mask.pandn.")) {
4307 VectorType *FTy = cast<VectorType>(CI->getType());
4308 VectorType *ITy = VectorType::getInteger(FTy);
4309 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
4310 Rep = Builder.CreateAnd(Rep,
4311 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4312 Rep = Builder.CreateBitCast(Rep, FTy);
4313 Rep =
4314 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4315 } else if (Name.starts_with("avx512.mask.or.") ||
4316 Name.starts_with("avx512.mask.por.")) {
4317 VectorType *FTy = cast<VectorType>(CI->getType());
4318 VectorType *ITy = VectorType::getInteger(FTy);
4319 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4320 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4321 Rep = Builder.CreateBitCast(Rep, FTy);
4322 Rep =
4323 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4324 } else if (Name.starts_with("avx512.mask.xor.") ||
4325 Name.starts_with("avx512.mask.pxor.")) {
4326 VectorType *FTy = cast<VectorType>(CI->getType());
4327 VectorType *ITy = VectorType::getInteger(FTy);
4328 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4329 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4330 Rep = Builder.CreateBitCast(Rep, FTy);
4331 Rep =
4332 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4333 } else if (Name.starts_with("avx512.mask.padd.")) {
4334 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4335 Rep =
4336 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4337 } else if (Name.starts_with("avx512.mask.psub.")) {
4338 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
4339 Rep =
4340 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4341 } else if (Name.starts_with("avx512.mask.pmull.")) {
4342 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
4343 Rep =
4344 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4345 } else if (Name.starts_with("avx512.mask.add.p")) {
4346 if (Name.ends_with(".512")) {
4347 Intrinsic::ID IID;
4348 if (Name[17] == 's')
4349 IID = Intrinsic::x86_avx512_add_ps_512;
4350 else
4351 IID = Intrinsic::x86_avx512_add_pd_512;
4352
4353 Rep = Builder.CreateIntrinsic(
4354 IID,
4355 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4356 } else {
4357 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4358 }
4359 Rep =
4360 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4361 } else if (Name.starts_with("avx512.mask.div.p")) {
4362 if (Name.ends_with(".512")) {
4363 Intrinsic::ID IID;
4364 if (Name[17] == 's')
4365 IID = Intrinsic::x86_avx512_div_ps_512;
4366 else
4367 IID = Intrinsic::x86_avx512_div_pd_512;
4368
4369 Rep = Builder.CreateIntrinsic(
4370 IID,
4371 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4372 } else {
4373 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
4374 }
4375 Rep =
4376 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4377 } else if (Name.starts_with("avx512.mask.mul.p")) {
4378 if (Name.ends_with(".512")) {
4379 Intrinsic::ID IID;
4380 if (Name[17] == 's')
4381 IID = Intrinsic::x86_avx512_mul_ps_512;
4382 else
4383 IID = Intrinsic::x86_avx512_mul_pd_512;
4384
4385 Rep = Builder.CreateIntrinsic(
4386 IID,
4387 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4388 } else {
4389 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
4390 }
4391 Rep =
4392 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4393 } else if (Name.starts_with("avx512.mask.sub.p")) {
4394 if (Name.ends_with(".512")) {
4395 Intrinsic::ID IID;
4396 if (Name[17] == 's')
4397 IID = Intrinsic::x86_avx512_sub_ps_512;
4398 else
4399 IID = Intrinsic::x86_avx512_sub_pd_512;
4400
4401 Rep = Builder.CreateIntrinsic(
4402 IID,
4403 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4404 } else {
4405 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
4406 }
4407 Rep =
4408 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4409 } else if ((Name.starts_with("avx512.mask.max.p") ||
4410 Name.starts_with("avx512.mask.min.p")) &&
4411 Name.drop_front(18) == ".512") {
4412 bool IsDouble = Name[17] == 'd';
4413 bool IsMin = Name[13] == 'i';
4414 static const Intrinsic::ID MinMaxTbl[2][2] = {
4415 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4416 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4417 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
4418
4419 Rep = Builder.CreateIntrinsic(
4420 IID,
4421 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4422 Rep =
4423 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4424 } else if (Name.starts_with("avx512.mask.lzcnt.")) {
4425 Rep =
4426 Builder.CreateIntrinsic(Intrinsic::ctlz, CI->getType(),
4427 {CI->getArgOperand(0), Builder.getInt1(false)});
4428 Rep =
4429 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4430 } else if (Name.starts_with("avx512.mask.psll")) {
4431 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4432 bool IsVariable = Name[16] == 'v';
4433 char Size = Name[16] == '.' ? Name[17]
4434 : Name[17] == '.' ? Name[18]
4435 : Name[18] == '.' ? Name[19]
4436 : Name[20];
4437
4438 Intrinsic::ID IID;
4439 if (IsVariable && Name[17] != '.') {
4440 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
4441 IID = Intrinsic::x86_avx2_psllv_q;
4442 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
4443 IID = Intrinsic::x86_avx2_psllv_q_256;
4444 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
4445 IID = Intrinsic::x86_avx2_psllv_d;
4446 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
4447 IID = Intrinsic::x86_avx2_psllv_d_256;
4448 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
4449 IID = Intrinsic::x86_avx512_psllv_w_128;
4450 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
4451 IID = Intrinsic::x86_avx512_psllv_w_256;
4452 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
4453 IID = Intrinsic::x86_avx512_psllv_w_512;
4454 else
4455 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4456 } else if (Name.ends_with(".128")) {
4457 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
4458 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4459 : Intrinsic::x86_sse2_psll_d;
4460 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
4461 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4462 : Intrinsic::x86_sse2_psll_q;
4463 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
4464 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4465 : Intrinsic::x86_sse2_psll_w;
4466 else
4467 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4468 } else if (Name.ends_with(".256")) {
4469 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
4470 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4471 : Intrinsic::x86_avx2_psll_d;
4472 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
4473 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4474 : Intrinsic::x86_avx2_psll_q;
4475 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
4476 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4477 : Intrinsic::x86_avx2_psll_w;
4478 else
4479 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4480 } else {
4481 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
4482 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4483 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4484 : Intrinsic::x86_avx512_psll_d_512;
4485 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
4486 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4487 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4488 : Intrinsic::x86_avx512_psll_q_512;
4489 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
4490 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4491 : Intrinsic::x86_avx512_psll_w_512;
4492 else
4493 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4494 }
4495
4496 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4497 } else if (Name.starts_with("avx512.mask.psrl")) {
4498 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4499 bool IsVariable = Name[16] == 'v';
4500 char Size = Name[16] == '.' ? Name[17]
4501 : Name[17] == '.' ? Name[18]
4502 : Name[18] == '.' ? Name[19]
4503 : Name[20];
4504
4505 Intrinsic::ID IID;
4506 if (IsVariable && Name[17] != '.') {
4507 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
4508 IID = Intrinsic::x86_avx2_psrlv_q;
4509 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
4510 IID = Intrinsic::x86_avx2_psrlv_q_256;
4511 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
4512 IID = Intrinsic::x86_avx2_psrlv_d;
4513 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
4514 IID = Intrinsic::x86_avx2_psrlv_d_256;
4515 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
4516 IID = Intrinsic::x86_avx512_psrlv_w_128;
4517 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
4518 IID = Intrinsic::x86_avx512_psrlv_w_256;
4519 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
4520 IID = Intrinsic::x86_avx512_psrlv_w_512;
4521 else
4522 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4523 } else if (Name.ends_with(".128")) {
4524 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4525 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4526 : Intrinsic::x86_sse2_psrl_d;
4527 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4528 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4529 : Intrinsic::x86_sse2_psrl_q;
4530 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4531 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4532 : Intrinsic::x86_sse2_psrl_w;
4533 else
4534 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4535 } else if (Name.ends_with(".256")) {
4536 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4537 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4538 : Intrinsic::x86_avx2_psrl_d;
4539 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4540 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4541 : Intrinsic::x86_avx2_psrl_q;
4542 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4543 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4544 : Intrinsic::x86_avx2_psrl_w;
4545 else
4546 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4547 } else {
4548 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4549 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4550 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4551 : Intrinsic::x86_avx512_psrl_d_512;
4552 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4553 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4554 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4555 : Intrinsic::x86_avx512_psrl_q_512;
4556 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
4557 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4558 : Intrinsic::x86_avx512_psrl_w_512;
4559 else
4560 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4561 }
4562
4563 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4564 } else if (Name.starts_with("avx512.mask.psra")) {
4565 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4566 bool IsVariable = Name[16] == 'v';
4567 char Size = Name[16] == '.' ? Name[17]
4568 : Name[17] == '.' ? Name[18]
4569 : Name[18] == '.' ? Name[19]
4570 : Name[20];
4571
4572 Intrinsic::ID IID;
4573 if (IsVariable && Name[17] != '.') {
4574 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
4575 IID = Intrinsic::x86_avx2_psrav_d;
4576 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
4577 IID = Intrinsic::x86_avx2_psrav_d_256;
4578 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
4579 IID = Intrinsic::x86_avx512_psrav_w_128;
4580 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
4581 IID = Intrinsic::x86_avx512_psrav_w_256;
4582 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
4583 IID = Intrinsic::x86_avx512_psrav_w_512;
4584 else
4585 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4586 } else if (Name.ends_with(".128")) {
4587 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4588 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4589 : Intrinsic::x86_sse2_psra_d;
4590 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4591 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4592 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4593 : Intrinsic::x86_avx512_psra_q_128;
4594 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4595 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4596 : Intrinsic::x86_sse2_psra_w;
4597 else
4598 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4599 } else if (Name.ends_with(".256")) {
4600 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4601 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4602 : Intrinsic::x86_avx2_psra_d;
4603 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4604 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4605 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4606 : Intrinsic::x86_avx512_psra_q_256;
4607 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4608 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4609 : Intrinsic::x86_avx2_psra_w;
4610 else
4611 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4612 } else {
4613 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
4614 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4615 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4616 : Intrinsic::x86_avx512_psra_d_512;
4617 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
4618 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4619 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4620 : Intrinsic::x86_avx512_psra_q_512;
4621 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
4622 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4623 : Intrinsic::x86_avx512_psra_w_512;
4624 else
4625 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4626 }
4627
4628 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4629 } else if (Name.starts_with("avx512.mask.move.s")) {
4630 Rep = upgradeMaskedMove(Builder, *CI);
4631 } else if (Name.starts_with("avx512.cvtmask2")) {
4632 Rep = upgradeMaskToInt(Builder, *CI);
4633 } else if (Name.ends_with(".movntdqa")) {
4634 MDNode *Node = MDNode::get(
4635 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
4636
4637 LoadInst *LI = Builder.CreateAlignedLoad(
4638 CI->getType(), CI->getArgOperand(0),
4639 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
4640 LI->setMetadata(LLVMContext::MD_nontemporal, Node);
4641 Rep = LI;
4642 } else if (Name.starts_with("fma.vfmadd.") ||
4643 Name.starts_with("fma.vfmsub.") ||
4644 Name.starts_with("fma.vfnmadd.") ||
4645 Name.starts_with("fma.vfnmsub.")) {
4646 bool NegMul = Name[6] == 'n';
4647 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
4648 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
4649
4650 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4651 CI->getArgOperand(2)};
4652
4653 if (IsScalar) {
4654 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4655 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4656 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4657 }
4658
4659 if (NegMul && !IsScalar)
4660 Ops[0] = Builder.CreateFNeg(Ops[0]);
4661 if (NegMul && IsScalar)
4662 Ops[1] = Builder.CreateFNeg(Ops[1]);
4663 if (NegAcc)
4664 Ops[2] = Builder.CreateFNeg(Ops[2]);
4665
4666 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4667
4668 if (IsScalar)
4669 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
4670 } else if (Name.starts_with("fma4.vfmadd.s")) {
4671 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4672 CI->getArgOperand(2)};
4673
4674 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4675 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4676 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4677
4678 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4679
4680 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
4681 Rep, (uint64_t)0);
4682 } else if (Name.starts_with("avx512.mask.vfmadd.s") ||
4683 Name.starts_with("avx512.maskz.vfmadd.s") ||
4684 Name.starts_with("avx512.mask3.vfmadd.s") ||
4685 Name.starts_with("avx512.mask3.vfmsub.s") ||
4686 Name.starts_with("avx512.mask3.vfnmsub.s")) {
4687 bool IsMask3 = Name[11] == '3';
4688 bool IsMaskZ = Name[11] == 'z';
4689 // Drop the "avx512.mask." to make it easier.
4690 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4691 bool NegMul = Name[2] == 'n';
4692 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4693
4694 Value *A = CI->getArgOperand(0);
4695 Value *B = CI->getArgOperand(1);
4696 Value *C = CI->getArgOperand(2);
4697
4698 if (NegMul && (IsMask3 || IsMaskZ))
4699 A = Builder.CreateFNeg(A);
4700 if (NegMul && !(IsMask3 || IsMaskZ))
4701 B = Builder.CreateFNeg(B);
4702 if (NegAcc)
4703 C = Builder.CreateFNeg(C);
4704
4705 A = Builder.CreateExtractElement(A, (uint64_t)0);
4706 B = Builder.CreateExtractElement(B, (uint64_t)0);
4707 C = Builder.CreateExtractElement(C, (uint64_t)0);
4708
4709 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4710 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
4711 Value *Ops[] = {A, B, C, CI->getArgOperand(4)};
4712
4713 Intrinsic::ID IID;
4714 if (Name.back() == 'd')
4715 IID = Intrinsic::x86_avx512_vfmadd_f64;
4716 else
4717 IID = Intrinsic::x86_avx512_vfmadd_f32;
4718 Rep = Builder.CreateIntrinsic(IID, Ops);
4719 } else {
4720 Rep = Builder.CreateFMA(A, B, C);
4721 }
4722
4723 Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType())
4724 : IsMask3 ? C
4725 : A;
4726
4727 // For Mask3 with NegAcc, we need to create a new extractelement that
4728 // avoids the negation above.
4729 if (NegAcc && IsMask3)
4730 PassThru =
4731 Builder.CreateExtractElement(CI->getArgOperand(2), (uint64_t)0);
4732
4733 Rep = emitX86ScalarSelect(Builder, CI->getArgOperand(3), Rep, PassThru);
4734 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), Rep,
4735 (uint64_t)0);
4736 } else if (Name.starts_with("avx512.mask.vfmadd.p") ||
4737 Name.starts_with("avx512.mask.vfnmadd.p") ||
4738 Name.starts_with("avx512.mask.vfnmsub.p") ||
4739 Name.starts_with("avx512.mask3.vfmadd.p") ||
4740 Name.starts_with("avx512.mask3.vfmsub.p") ||
4741 Name.starts_with("avx512.mask3.vfnmsub.p") ||
4742 Name.starts_with("avx512.maskz.vfmadd.p")) {
4743 bool IsMask3 = Name[11] == '3';
4744 bool IsMaskZ = Name[11] == 'z';
4745 // Drop the "avx512.mask." to make it easier.
4746 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4747 bool NegMul = Name[2] == 'n';
4748 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4749
4750 Value *A = CI->getArgOperand(0);
4751 Value *B = CI->getArgOperand(1);
4752 Value *C = CI->getArgOperand(2);
4753
4754 if (NegMul && (IsMask3 || IsMaskZ))
4755 A = Builder.CreateFNeg(A);
4756 if (NegMul && !(IsMask3 || IsMaskZ))
4757 B = Builder.CreateFNeg(B);
4758 if (NegAcc)
4759 C = Builder.CreateFNeg(C);
4760
4761 if (CI->arg_size() == 5 &&
4762 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4763 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
4764 Intrinsic::ID IID;
4765 // Check the character before ".512" in string.
4766 if (Name[Name.size() - 5] == 's')
4767 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4768 else
4769 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4770
4771 Rep = Builder.CreateIntrinsic(IID, {A, B, C, CI->getArgOperand(4)});
4772 } else {
4773 Rep = Builder.CreateFMA(A, B, C);
4774 }
4775
4776 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4777 : IsMask3 ? CI->getArgOperand(2)
4778 : CI->getArgOperand(0);
4779
4780 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4781 } else if (Name.starts_with("fma.vfmsubadd.p")) {
4782 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4783 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4784 Intrinsic::ID IID;
4785 if (VecWidth == 128 && EltWidth == 32)
4786 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4787 else if (VecWidth == 256 && EltWidth == 32)
4788 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4789 else if (VecWidth == 128 && EltWidth == 64)
4790 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4791 else if (VecWidth == 256 && EltWidth == 64)
4792 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4793 else
4794 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4795
4796 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4797 CI->getArgOperand(2)};
4798 Ops[2] = Builder.CreateFNeg(Ops[2]);
4799 Rep = Builder.CreateIntrinsic(IID, Ops);
4800 } else if (Name.starts_with("avx512.mask.vfmaddsub.p") ||
4801 Name.starts_with("avx512.mask3.vfmaddsub.p") ||
4802 Name.starts_with("avx512.maskz.vfmaddsub.p") ||
4803 Name.starts_with("avx512.mask3.vfmsubadd.p")) {
4804 bool IsMask3 = Name[11] == '3';
4805 bool IsMaskZ = Name[11] == 'z';
4806 // Drop the "avx512.mask." to make it easier.
4807 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4808 bool IsSubAdd = Name[3] == 's';
4809 if (CI->arg_size() == 5) {
4810 Intrinsic::ID IID;
4811 // Check the character before ".512" in string.
4812 if (Name[Name.size() - 5] == 's')
4813 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
4814 else
4815 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
4816
4817 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4818 CI->getArgOperand(2), CI->getArgOperand(4)};
4819 if (IsSubAdd)
4820 Ops[2] = Builder.CreateFNeg(Ops[2]);
4821
4822 Rep = Builder.CreateIntrinsic(IID, Ops);
4823 } else {
4824 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4825
4826 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4827 CI->getArgOperand(2)};
4828
4829 Function *FMA = Intrinsic::getOrInsertDeclaration(
4830 CI->getModule(), Intrinsic::fma, Ops[0]->getType());
4831 Value *Odd = Builder.CreateCall(FMA, Ops);
4832 Ops[2] = Builder.CreateFNeg(Ops[2]);
4833 Value *Even = Builder.CreateCall(FMA, Ops);
4834
4835 if (IsSubAdd)
4836 std::swap(Even, Odd);
4837
4838 SmallVector<int, 32> Idxs(NumElts);
4839 for (int i = 0; i != NumElts; ++i)
4840 Idxs[i] = i + (i % 2) * NumElts;
4841
4842 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
4843 }
4844
4845 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4846 : IsMask3 ? CI->getArgOperand(2)
4847 : CI->getArgOperand(0);
4848
4849 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4850 } else if (Name.starts_with("avx512.mask.pternlog.") ||
4851 Name.starts_with("avx512.maskz.pternlog.")) {
4852 bool ZeroMask = Name[11] == 'z';
4853 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4854 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4855 Intrinsic::ID IID;
4856 if (VecWidth == 128 && EltWidth == 32)
4857 IID = Intrinsic::x86_avx512_pternlog_d_128;
4858 else if (VecWidth == 256 && EltWidth == 32)
4859 IID = Intrinsic::x86_avx512_pternlog_d_256;
4860 else if (VecWidth == 512 && EltWidth == 32)
4861 IID = Intrinsic::x86_avx512_pternlog_d_512;
4862 else if (VecWidth == 128 && EltWidth == 64)
4863 IID = Intrinsic::x86_avx512_pternlog_q_128;
4864 else if (VecWidth == 256 && EltWidth == 64)
4865 IID = Intrinsic::x86_avx512_pternlog_q_256;
4866 else if (VecWidth == 512 && EltWidth == 64)
4867 IID = Intrinsic::x86_avx512_pternlog_q_512;
4868 else
4869 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4870
4871 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4872 CI->getArgOperand(2), CI->getArgOperand(3)};
4873 Rep = Builder.CreateIntrinsic(IID, Args);
4874 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4875 : CI->getArgOperand(0);
4876 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
4877 } else if (Name.starts_with("avx512.mask.vpmadd52") ||
4878 Name.starts_with("avx512.maskz.vpmadd52")) {
4879 bool ZeroMask = Name[11] == 'z';
4880 bool High = Name[20] == 'h' || Name[21] == 'h';
4881 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4882 Intrinsic::ID IID;
4883 if (VecWidth == 128 && !High)
4884 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
4885 else if (VecWidth == 256 && !High)
4886 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
4887 else if (VecWidth == 512 && !High)
4888 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
4889 else if (VecWidth == 128 && High)
4890 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
4891 else if (VecWidth == 256 && High)
4892 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
4893 else if (VecWidth == 512 && High)
4894 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
4895 else
4896 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4897
4898 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4899 CI->getArgOperand(2)};
4900 Rep = Builder.CreateIntrinsic(IID, Args);
4901 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4902 : CI->getArgOperand(0);
4903 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4904 } else if (Name.starts_with("avx512.mask.vpermi2var.") ||
4905 Name.starts_with("avx512.mask.vpermt2var.") ||
4906 Name.starts_with("avx512.maskz.vpermt2var.")) {
4907 bool ZeroMask = Name[11] == 'z';
4908 bool IndexForm = Name[17] == 'i';
4909 Rep = upgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
4910 } else if (Name.starts_with("avx512.mask.vpdpbusd.") ||
4911 Name.starts_with("avx512.maskz.vpdpbusd.") ||
4912 Name.starts_with("avx512.mask.vpdpbusds.") ||
4913 Name.starts_with("avx512.maskz.vpdpbusds.")) {
4914 bool ZeroMask = Name[11] == 'z';
4915 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4916 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4917 Intrinsic::ID IID;
4918 if (VecWidth == 128 && !IsSaturating)
4919 IID = Intrinsic::x86_avx512_vpdpbusd_128;
4920 else if (VecWidth == 256 && !IsSaturating)
4921 IID = Intrinsic::x86_avx512_vpdpbusd_256;
4922 else if (VecWidth == 512 && !IsSaturating)
4923 IID = Intrinsic::x86_avx512_vpdpbusd_512;
4924 else if (VecWidth == 128 && IsSaturating)
4925 IID = Intrinsic::x86_avx512_vpdpbusds_128;
4926 else if (VecWidth == 256 && IsSaturating)
4927 IID = Intrinsic::x86_avx512_vpdpbusds_256;
4928 else if (VecWidth == 512 && IsSaturating)
4929 IID = Intrinsic::x86_avx512_vpdpbusds_512;
4930 else
4931 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4932
4933 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4934 CI->getArgOperand(2)};
4935
4936 // Input arguments types were incorrectly set to vectors of i32 before but
4937 // they should be vectors of i8. Insert bit cast when encountering the old
4938 // types
4939 if (Args[1]->getType()->isVectorTy() &&
4940 cast<VectorType>(Args[1]->getType())
4941 ->getElementType()
4942 ->isIntegerTy(32) &&
4943 Args[2]->getType()->isVectorTy() &&
4944 cast<VectorType>(Args[2]->getType())
4945 ->getElementType()
4946 ->isIntegerTy(32)) {
4947 Type *NewArgType = nullptr;
4948 if (VecWidth == 128)
4949 NewArgType = VectorType::get(Builder.getInt8Ty(), 16, false);
4950 else if (VecWidth == 256)
4951 NewArgType = VectorType::get(Builder.getInt8Ty(), 32, false);
4952 else if (VecWidth == 512)
4953 NewArgType = VectorType::get(Builder.getInt8Ty(), 64, false);
4954 else
4955 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
4956 CI);
4957
4958 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
4959 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
4960 }
4961
4962 Rep = Builder.CreateIntrinsic(IID, Args);
4963 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4964 : CI->getArgOperand(0);
4965 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4966 } else if (Name.starts_with("avx512.mask.vpdpwssd.") ||
4967 Name.starts_with("avx512.maskz.vpdpwssd.") ||
4968 Name.starts_with("avx512.mask.vpdpwssds.") ||
4969 Name.starts_with("avx512.maskz.vpdpwssds.")) {
4970 bool ZeroMask = Name[11] == 'z';
4971 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4972 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4973 Intrinsic::ID IID;
4974 if (VecWidth == 128 && !IsSaturating)
4975 IID = Intrinsic::x86_avx512_vpdpwssd_128;
4976 else if (VecWidth == 256 && !IsSaturating)
4977 IID = Intrinsic::x86_avx512_vpdpwssd_256;
4978 else if (VecWidth == 512 && !IsSaturating)
4979 IID = Intrinsic::x86_avx512_vpdpwssd_512;
4980 else if (VecWidth == 128 && IsSaturating)
4981 IID = Intrinsic::x86_avx512_vpdpwssds_128;
4982 else if (VecWidth == 256 && IsSaturating)
4983 IID = Intrinsic::x86_avx512_vpdpwssds_256;
4984 else if (VecWidth == 512 && IsSaturating)
4985 IID = Intrinsic::x86_avx512_vpdpwssds_512;
4986 else
4987 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4988
4989 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4990 CI->getArgOperand(2)};
4991
4992 // Input arguments types were incorrectly set to vectors of i32 before but
4993 // they should be vectors of i16. Insert bit cast when encountering the old
4994 // types
4995 if (Args[1]->getType()->isVectorTy() &&
4996 cast<VectorType>(Args[1]->getType())
4997 ->getElementType()
4998 ->isIntegerTy(32) &&
4999 Args[2]->getType()->isVectorTy() &&
5000 cast<VectorType>(Args[2]->getType())
5001 ->getElementType()
5002 ->isIntegerTy(32)) {
5003 Type *NewArgType = nullptr;
5004 if (VecWidth == 128)
5005 NewArgType = VectorType::get(Builder.getInt16Ty(), 8, false);
5006 else if (VecWidth == 256)
5007 NewArgType = VectorType::get(Builder.getInt16Ty(), 16, false);
5008 else if (VecWidth == 512)
5009 NewArgType = VectorType::get(Builder.getInt16Ty(), 32, false);
5010 else
5011 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
5012 CI);
5013
5014 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5015 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5016 }
5017
5018 Rep = Builder.CreateIntrinsic(IID, Args);
5019 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5020 : CI->getArgOperand(0);
5021 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5022 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
5023 Name == "addcarry.u32" || Name == "addcarry.u64" ||
5024 Name == "subborrow.u32" || Name == "subborrow.u64") {
5025 Intrinsic::ID IID;
5026 if (Name[0] == 'a' && Name.back() == '2')
5027 IID = Intrinsic::x86_addcarry_32;
5028 else if (Name[0] == 'a' && Name.back() == '4')
5029 IID = Intrinsic::x86_addcarry_64;
5030 else if (Name[0] == 's' && Name.back() == '2')
5031 IID = Intrinsic::x86_subborrow_32;
5032 else if (Name[0] == 's' && Name.back() == '4')
5033 IID = Intrinsic::x86_subborrow_64;
5034 else
5035 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5036
5037 // Make a call with 3 operands.
5038 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5039 CI->getArgOperand(2)};
5040 Value *NewCall = Builder.CreateIntrinsic(IID, Args);
5041
5042 // Extract the second result and store it.
5043 Value *Data = Builder.CreateExtractValue(NewCall, 1);
5044 Builder.CreateAlignedStore(Data, CI->getArgOperand(3), Align(1));
5045 // Replace the original call result with the first result of the new call.
5046 Value *CF = Builder.CreateExtractValue(NewCall, 0);
5047
5048 CI->replaceAllUsesWith(CF);
5049 Rep = nullptr;
5050 } else if (Name.starts_with("avx512.mask.") &&
5051 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
5052 // Rep will be updated by the call in the condition.
5053 } else if (Name.starts_with("bmi.pdep.")) {
5054 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pdep);
5055 } else if (Name.starts_with("bmi.pext.")) {
5056 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pext);
5057 } else
5058 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5059
5060 return Rep;
5061}
5062
5063static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
5064 Function *F, IRBuilder<> &Builder) {
5065 if (Name.starts_with("neon.bfcvt")) {
5066 if (Name.starts_with("neon.bfcvtn2")) {
5067 SmallVector<int, 32> LoMask(4);
5068 std::iota(LoMask.begin(), LoMask.end(), 0);
5069 SmallVector<int, 32> ConcatMask(8);
5070 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
5071 Value *Inactive = Builder.CreateShuffleVector(CI->getOperand(0), LoMask);
5072 Value *Trunc =
5073 Builder.CreateFPTrunc(CI->getOperand(1), Inactive->getType());
5074 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
5075 } else if (Name.starts_with("neon.bfcvtn")) {
5076 SmallVector<int, 32> ConcatMask(8);
5077 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
5078 Type *V4BF16 =
5079 FixedVectorType::get(Type::getBFloatTy(F->getContext()), 4);
5080 Value *Trunc = Builder.CreateFPTrunc(CI->getOperand(0), V4BF16);
5081 dbgs() << "Trunc: " << *Trunc << "\n";
5082 return Builder.CreateShuffleVector(
5083 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
5084 } else {
5085 return Builder.CreateFPTrunc(CI->getOperand(0),
5086 Type::getBFloatTy(F->getContext()));
5087 }
5088 } else if (Name.starts_with("sve.fcvt")) {
5089 Intrinsic::ID NewID =
5090 StringSwitch<Intrinsic::ID>(Name)
5091 .Case("sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
5092 .Case("sve.fcvtnt.bf16f32",
5093 Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
5094 .Default(Intrinsic::not_intrinsic);
5095 if (NewID == Intrinsic::not_intrinsic)
5096 llvm_unreachable("Unhandled Intrinsic!");
5097
5098 SmallVector<Value *, 3> Args(CI->args());
5099
5100 // The original intrinsics incorrectly used a predicate based on the
5101 // smallest element type rather than the largest.
5102 Type *BadPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 8);
5103 Type *GoodPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 4);
5104
5105 if (Args[1]->getType() != BadPredTy)
5106 llvm_unreachable("Unexpected predicate type!");
5107
5108 Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
5109 BadPredTy, Args[1]);
5110 Args[1] = Builder.CreateIntrinsic(
5111 Intrinsic::aarch64_sve_convert_from_svbool, GoodPredTy, Args[1]);
5112
5113 return Builder.CreateIntrinsic(NewID, Args, /*FMFSource=*/nullptr,
5114 CI->getName());
5115 }
5116
5117 if (Name == "neon.vcvtfp2hf")
5118 return Builder.CreateBitCast(
5119 Builder.CreateFPTrunc(
5120 CI->getOperand(0),
5121 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5122 FixedVectorType::get(Type::getInt16Ty(F->getContext()), 4));
5123 if (Name == "neon.vcvthf2fp")
5124 return Builder.CreateFPExt(
5125 Builder.CreateBitCast(
5126 CI->getOperand(0),
5127 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5128 FixedVectorType::get(Type::getFloatTy(F->getContext()), 4));
5129
5130 llvm_unreachable("Unhandled Intrinsic!");
5131}
5132
5133static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
5134 IRBuilder<> &Builder) {
5135 if (Name == "mve.vctp64.old") {
5136 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
5137 // correct type.
5138 Value *VCTP = Builder.CreateIntrinsic(Intrinsic::arm_mve_vctp64, {},
5139 CI->getArgOperand(0),
5140 /*FMFSource=*/nullptr, CI->getName());
5141 Value *C1 = Builder.CreateIntrinsic(
5142 Intrinsic::arm_mve_pred_v2i,
5143 {VectorType::get(Builder.getInt1Ty(), 2, false)}, VCTP);
5144 return Builder.CreateIntrinsic(
5145 Intrinsic::arm_mve_pred_i2v,
5146 {VectorType::get(Builder.getInt1Ty(), 4, false)}, C1);
5147 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
5148 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
5149 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
5150 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
5151 Name ==
5152 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
5153 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
5154 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
5155 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
5156 Name ==
5157 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
5158 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
5159 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
5160 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
5161 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
5162 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
5163 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
5164 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
5165 std::vector<Type *> Tys;
5166 unsigned ID = CI->getIntrinsicID();
5167 Type *V2I1Ty = FixedVectorType::get(Builder.getInt1Ty(), 2);
5168 switch (ID) {
5169 case Intrinsic::arm_mve_mull_int_predicated:
5170 case Intrinsic::arm_mve_vqdmull_predicated:
5171 case Intrinsic::arm_mve_vldr_gather_base_predicated:
5172 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty};
5173 break;
5174 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
5175 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
5176 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
5177 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(),
5178 V2I1Ty};
5179 break;
5180 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
5181 Tys = {CI->getType(), CI->getOperand(0)->getType(),
5182 CI->getOperand(1)->getType(), V2I1Ty};
5183 break;
5184 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
5185 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(),
5186 CI->getOperand(2)->getType(), V2I1Ty};
5187 break;
5188 case Intrinsic::arm_cde_vcx1q_predicated:
5189 case Intrinsic::arm_cde_vcx1qa_predicated:
5190 case Intrinsic::arm_cde_vcx2q_predicated:
5191 case Intrinsic::arm_cde_vcx2qa_predicated:
5192 case Intrinsic::arm_cde_vcx3q_predicated:
5193 case Intrinsic::arm_cde_vcx3qa_predicated:
5194 Tys = {CI->getOperand(1)->getType(), V2I1Ty};
5195 break;
5196 default:
5197 llvm_unreachable("Unhandled Intrinsic!");
5198 }
5199
5200 std::vector<Value *> Ops;
5201 for (Value *Op : CI->args()) {
5202 Type *Ty = Op->getType();
5203 if (Ty->getScalarSizeInBits() == 1) {
5204 Value *C1 = Builder.CreateIntrinsic(
5205 Intrinsic::arm_mve_pred_v2i,
5206 {VectorType::get(Builder.getInt1Ty(), 4, false)}, Op);
5207 Op = Builder.CreateIntrinsic(Intrinsic::arm_mve_pred_i2v, {V2I1Ty}, C1);
5208 }
5209 Ops.push_back(Op);
5210 }
5211
5212 return Builder.CreateIntrinsic(ID, Tys, Ops, /*FMFSource=*/nullptr,
5213 CI->getName());
5214 }
5215 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
5216}
5217
5218// These are expected to have the arguments:
5219// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
5220//
5221// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
5222//
5223static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
5224 Function *F, IRBuilder<> &Builder) {
5225 // Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
5226 // for compatibility.
5227 auto UpgradeLegacyWMMAIUIntrinsicCall =
5228 [](Function *F, CallBase *CI, IRBuilder<> &Builder,
5229 ArrayRef<Type *> OverloadTys) -> Value * {
5230 // Prepare arguments, append clamp=0 for compatibility
5231 SmallVector<Value *, 10> Args(CI->args().begin(), CI->args().end());
5232 Args.push_back(Builder.getFalse());
5233
5234 // Insert the declaration for the right overload types
5235 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5236 F->getParent(), F->getIntrinsicID(), OverloadTys);
5237
5238 // Copy operand bundles if any
5239 SmallVector<OperandBundleDef, 1> Bundles;
5240 CI->getOperandBundlesAsDefs(Bundles);
5241
5242 // Create the new call and copy calling properties
5243 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5244 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5245 NewCall->setCallingConv(CI->getCallingConv());
5246 NewCall->setAttributes(CI->getAttributes());
5247 NewCall->copyMetadata(*CI);
5248 return NewCall;
5249 };
5250
5251 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
5252 assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
5253 "intrinsic should have 7 arguments");
5254 Type *T1 = CI->getArgOperand(4)->getType();
5255 Type *T2 = CI->getArgOperand(1)->getType();
5256 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2});
5257 }
5258 if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
5259 assert(CI->arg_size() == 8 && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
5260 "intrinsic should have 8 arguments");
5261 Type *T1 = CI->getArgOperand(4)->getType();
5262 Type *T2 = CI->getArgOperand(1)->getType();
5263 Type *T3 = CI->getArgOperand(3)->getType();
5264 Type *T4 = CI->getArgOperand(5)->getType();
5265 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
5266 }
5267
5268 switch (F->getIntrinsicID()) {
5269 default:
5270 break;
5271 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
5272 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
5273 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
5274 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
5275 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
5276 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
5277 // Drop src0 and src1 modifiers.
5278 const Value *Op0 = CI->getArgOperand(0);
5279 const Value *Op2 = CI->getArgOperand(2);
5280 assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
5281 const ConstantInt *ModA = dyn_cast<ConstantInt>(Op0);
5282 const ConstantInt *ModB = dyn_cast<ConstantInt>(Op2);
5283 if (!ModA->isZero() || !ModB->isZero())
5284 reportFatalUsageError(Name + " matrix A and B modifiers shall be zero");
5285
5286 SmallVector<Value *, 8> Args{CI->getArgOperand(1), CI->getArgOperand(3)};
5287 for (int I = 4, E = CI->arg_size(); I < E; ++I)
5288 Args.push_back(CI->getArgOperand(I));
5289
5290 SmallVector<Type *, 3> Overloads{F->getReturnType(), Args[0]->getType()};
5291 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
5292 Overloads.push_back(Args[3]->getType());
5293 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5294 F->getParent(), F->getIntrinsicID(), Overloads);
5295
5296 SmallVector<OperandBundleDef, 1> Bundles;
5297 CI->getOperandBundlesAsDefs(Bundles);
5298
5299 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5300 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5301 NewCall->setCallingConv(CI->getCallingConv());
5302 NewCall->setAttributes(CI->getAttributes());
5303 NewCall->copyMetadata(*CI);
5304 NewCall->takeName(CI);
5305 return NewCall;
5306 }
5307 }
5308
5309 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
5310 Value *LHS = CI->getArgOperand(0);
5311 Value *RHS = CI->getArgOperand(1);
5312 CmpInst::Predicate Pred = static_cast<CmpInst::Predicate>(
5313 cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue());
5314 Value *Cmp = Builder.CreateCmp(Pred, LHS, RHS);
5315 CallInst *NewCall = Builder.CreateIntrinsicWithoutFolding(
5316 CI->getType(), Intrinsic::amdgcn_ballot, Cmp);
5317 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5318 NewCall->setCallingConv(CI->getCallingConv());
5319 NewCall->copyMetadata(*CI);
5320 NewCall->takeName(CI);
5321 return NewCall;
5322 }
5323
5324 if (Name.starts_with("addrspacecast.nonnull")) {
5325 if (CI->getNumOperands() < 2) // Malformed bitcode.
5326 return nullptr;
5327 Value *ASC = Builder.CreateAddrSpaceCast(
5328 CI->getArgOperand(0), CI->getType(), "", /*IsNonNull=*/true);
5329 ASC->takeName(CI);
5330 return ASC;
5331 }
5332
5333 AtomicRMWInst::BinOp RMWOp =
5334 StringSwitch<AtomicRMWInst::BinOp>(Name)
5335 .StartsWith("ds.fadd", AtomicRMWInst::FAdd)
5336 .StartsWith("ds.fmin", AtomicRMWInst::FMin)
5337 .StartsWith("ds.fmax", AtomicRMWInst::FMax)
5338 .StartsWith("atomic.inc.", AtomicRMWInst::UIncWrap)
5339 .StartsWith("atomic.dec.", AtomicRMWInst::UDecWrap)
5340 .StartsWith("global.atomic.fadd", AtomicRMWInst::FAdd)
5341 .StartsWith("flat.atomic.fadd", AtomicRMWInst::FAdd)
5342 .StartsWith("global.atomic.fmin", AtomicRMWInst::FMin)
5343 .StartsWith("flat.atomic.fmin", AtomicRMWInst::FMin)
5344 .StartsWith("global.atomic.fmax", AtomicRMWInst::FMax)
5345 .StartsWith("flat.atomic.fmax", AtomicRMWInst::FMax)
5346 .StartsWith("atomic.cond.sub", AtomicRMWInst::USubCond)
5347 .StartsWith("atomic.csub", AtomicRMWInst::USubSat);
5348
5349 unsigned NumOperands = CI->getNumOperands();
5350 if (NumOperands < 3) // Malformed bitcode.
5351 return nullptr;
5352
5353 Value *Ptr = CI->getArgOperand(0);
5354 PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
5355 if (!PtrTy) // Malformed.
5356 return nullptr;
5357
5358 Value *Val = CI->getArgOperand(1);
5359 if (Val->getType() != CI->getType()) // Malformed.
5360 return nullptr;
5361
5362 ConstantInt *OrderArg = nullptr;
5363 bool IsVolatile = false;
5364
5365 // These should have 5 arguments (plus the callee). A separate version of the
5366 // ds_fadd intrinsic was defined for bf16 which was missing arguments.
5367 if (NumOperands > 3)
5368 OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5369
5370 // Ignore scope argument at 3
5371
5372 if (NumOperands > 5) {
5373 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4));
5374 IsVolatile = !VolatileArg || !VolatileArg->isZero();
5375 }
5376
5377 AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
5378 if (OrderArg && isValidAtomicOrdering(OrderArg->getZExtValue()))
5379 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
5380 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
5381 Order = AtomicOrdering::SequentiallyConsistent;
5382
5383 LLVMContext &Ctx = F->getContext();
5384
5385 // Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
5386 Type *RetTy = CI->getType();
5387 if (VectorType *VT = dyn_cast<VectorType>(RetTy)) {
5388 if (VT->getElementType()->isIntegerTy(16)) {
5389 VectorType *AsBF16 =
5390 VectorType::get(Type::getBFloatTy(Ctx), VT->getElementCount());
5391 Val = Builder.CreateBitCast(Val, AsBF16);
5392 }
5393 }
5394
5395 // The scope argument never really worked correctly. Use agent as the most
5396 // conservative option which should still always produce the instruction.
5397 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID("agent");
5398 AtomicRMWInst *RMW =
5399 Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);
5400
5401 unsigned AddrSpace = PtrTy->getAddressSpace();
5402 if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
5403 MDNode *EmptyMD = MDNode::get(F->getContext(), {});
5404 RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);
5405 if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
5406 RMW->setMetadata("amdgpu.ignore.denormal.mode", EmptyMD);
5407 }
5408
5409 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
5410 MDBuilder MDB(F->getContext());
5411 MDNode *RangeNotPrivate =
5412 MDB.createRange(APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
5413 APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
5414 RMW->setMetadata(LLVMContext::MD_noalias_addrspace, RangeNotPrivate);
5415 }
5416
5417 if (IsVolatile)
5418 RMW->setVolatile(true);
5419
5420 return Builder.CreateBitCast(RMW, RetTy);
5421}
5422
5423/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
5424/// plain MDNode, as it's the verifier's job to check these are the correct
5425/// types later.
5426static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
5427 if (Op < CI->arg_size()) {
5428 if (MetadataAsValue *MAV =
5429 dyn_cast<MetadataAsValue>(CI->getArgOperand(Op))) {
5430 Metadata *MD = MAV->getMetadata();
5431 return dyn_cast_if_present<MDNode>(MD);
5432 }
5433 }
5434 return nullptr;
5435}
5436
5437/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
5438static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
5439 if (Op < CI->arg_size())
5440 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(CI->getArgOperand(Op)))
5441 return MAV->getMetadata();
5442 return nullptr;
5443}
5444
5445/// Convert debug intrinsic calls to non-instruction debug records.
5446/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
5447/// \p CI - The debug intrinsic call.
5448static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
5449 DbgRecord *DR = nullptr;
5450 if (Name == "label") {
5451 DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(unwrapMAVOp(CI, 0));
5452 } else if (Name == "assign") {
5453 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5454 DbgVariableRecord::LocationType::Assign, unwrapMAVMetadataOp(CI, 0),
5455 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), unwrapMAVOp(CI, 3),
5456 unwrapMAVMetadataOp(CI, 4),
5457 /*The address is a Value ref, it will be stored as a Metadata */
5458 unwrapMAVOp(CI, 5));
5459 } else if (Name == "declare") {
5460 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5461 DbgVariableRecord::LocationType::Declare, unwrapMAVMetadataOp(CI, 0),
5462 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), nullptr, nullptr, nullptr);
5463 } else if (Name == "addr") {
5464 // Upgrade dbg.addr to dbg.value with DW_OP_deref.
5465 MDNode *ExprNode = unwrapMAVOp(CI, 2);
5466 // Don't try to add something to the expression if it's not an expression.
5467 // Instead, allow the verifier to fail later.
5468 if (DIExpression *Expr = dyn_cast<DIExpression>(ExprNode)) {
5469 ExprNode = DIExpression::append(Expr, dwarf::DW_OP_deref);
5470 }
5471 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5472 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5473 unwrapMAVOp(CI, 1), ExprNode, nullptr, nullptr, nullptr);
5474 } else if (Name == "value") {
5475 // An old version of dbg.value had an extra offset argument.
5476 unsigned VarOp = 1;
5477 unsigned ExprOp = 2;
5478 if (CI->arg_size() == 4) {
5479 auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1));
5480 // Nonzero offset dbg.values get dropped without a replacement.
5481 if (!Offset || !Offset->isNullValue())
5482 return;
5483 VarOp = 2;
5484 ExprOp = 3;
5485 }
5486 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5487 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5488 unwrapMAVOp(CI, VarOp), unwrapMAVOp(CI, ExprOp), nullptr, nullptr,
5489 nullptr);
5490 }
5491 DR->setDebugLoc(CI->getDebugLoc());
5492 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
5493 CI->getParent()->insertDbgRecordBefore(DR, CI->getIterator());
5494}
5495
5496static Value *upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder) {
5497 auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5498 if (!Offset)
5499 reportFatalUsageError("Invalid llvm.vector.splice offset argument");
5500 int64_t OffsetVal = Offset->getSExtValue();
5501 return Builder.CreateIntrinsic(OffsetVal >= 0
5502 ? Intrinsic::vector_splice_left
5503 : Intrinsic::vector_splice_right,
5504 CI->getType(),
5505 {CI->getArgOperand(0), CI->getArgOperand(1),
5506 Builder.getInt32(std::abs(OffsetVal))});
5507}
5508
5509static Value *upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI,
5510 Function *F, IRBuilder<> &Builder) {
5511 if (Name.starts_with("to.fp16")) {
5512 Value *Cast =
5513 Builder.CreateFPTrunc(CI->getArgOperand(0), Builder.getHalfTy());
5514 return Builder.CreateBitCast(Cast, CI->getType());
5515 }
5516
5517 if (Name.starts_with("from.fp16")) {
5518 Value *Cast =
5519 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
5520 return Builder.CreateFPExt(Cast, CI->getType());
5521 }
5522
5523 return nullptr;
5524}
5525
5526static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op) {
5527 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5528 if (!MD || !isa<MDString>(MD))
5529 return ICmpInst::BAD_ICMP_PREDICATE;
5530 return StringSwitch<ICmpInst::Predicate>(cast<MDString>(MD)->getString())
5531 .Case("eq", ICmpInst::ICMP_EQ)
5532 .Case("ne", ICmpInst::ICMP_NE)
5533 .Case("ugt", ICmpInst::ICMP_UGT)
5534 .Case("uge", ICmpInst::ICMP_UGE)
5535 .Case("ult", ICmpInst::ICMP_ULT)
5536 .Case("ule", ICmpInst::ICMP_ULE)
5537 .Case("sgt", ICmpInst::ICMP_SGT)
5538 .Case("sge", ICmpInst::ICMP_SGE)
5539 .Case("slt", ICmpInst::ICMP_SLT)
5540 .Case("sle", ICmpInst::ICMP_SLE)
5541 .Default(ICmpInst::BAD_ICMP_PREDICATE);
5542}
5543
5544static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op) {
5545 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5546 if (!MD || !isa<MDString>(MD))
5547 return FCmpInst::BAD_FCMP_PREDICATE;
5548 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(MD)->getString())
5549 .Case("oeq", FCmpInst::FCMP_OEQ)
5550 .Case("ogt", FCmpInst::FCMP_OGT)
5551 .Case("oge", FCmpInst::FCMP_OGE)
5552 .Case("olt", FCmpInst::FCMP_OLT)
5553 .Case("ole", FCmpInst::FCMP_OLE)
5554 .Case("one", FCmpInst::FCMP_ONE)
5555 .Case("ord", FCmpInst::FCMP_ORD)
5556 .Case("uno", FCmpInst::FCMP_UNO)
5557 .Case("ueq", FCmpInst::FCMP_UEQ)
5558 .Case("ugt", FCmpInst::FCMP_UGT)
5559 .Case("uge", FCmpInst::FCMP_UGE)
5560 .Case("ult", FCmpInst::FCMP_ULT)
5561 .Case("ule", FCmpInst::FCMP_ULE)
5562 .Case("une", FCmpInst::FCMP_UNE)
5563 .Default(FCmpInst::BAD_FCMP_PREDICATE);
5564}
5565
5566static Value *upgradeVPIntrinsicCall(StringRef Name, CallBase *CI,
5567 IRBuilder<> &Builder) {
5568 Value *Rep;
5569 unsigned Opcode = getFunctionalOpcodeForVP(Name);
5570 if (Opcode && Instruction::isUnaryOp(Opcode))
5571 Rep =
5572 Builder.CreateUnOp((Instruction::UnaryOps)Opcode, CI->getArgOperand(0));
5573 else if (Opcode && Instruction::isBinaryOp(Opcode))
5574 Rep = Builder.CreateBinOp((Instruction::BinaryOps)Opcode,
5575 CI->getArgOperand(0), CI->getArgOperand(1));
5576 else if (Opcode && Instruction::isCast(Opcode))
5577 Rep = Builder.CreateCast((Instruction::CastOps)Opcode, CI->getArgOperand(0),
5578 CI->getType());
5579 else if (Opcode == Instruction::ICmp)
5580 Rep = Builder.CreateICmp(getVPIntPredicateFromMD(CI->getArgOperand(2)),
5581 CI->getArgOperand(0), CI->getArgOperand(1));
5582 else if (Opcode == Instruction::FCmp)
5583 Rep = Builder.CreateFCmp(getVPFPPredicateFromMD(CI->getArgOperand(2)),
5584 CI->getArgOperand(0), CI->getArgOperand(1));
5585 else if (Opcode == Instruction::Select)
5586 Rep = Builder.CreateSelect(CI->getArgOperand(0), CI->getArgOperand(1),
5587 CI->getArgOperand(2));
5588 else if (auto IntrinsicID = getFunctionalIntrinsicIDForVP(Name)) {
5589 SmallVector<Value *, 2> Args(drop_end(CI->args(), 2));
5590 Rep = Builder.CreateIntrinsic(CI->getType(), IntrinsicID, Args, {});
5591 } else
5592 llvm_unreachable("Unexpected vp intrinsic");
5593 Rep->takeName(CI);
5594 return Rep;
5595}
5596
5597static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn,
5598 IRBuilder<> &Builder) {
5599 Intrinsic::ID IID = NewFn->getIntrinsicID();
5600
5601 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
5602 if (Defaults.empty())
5603 return false;
5604
5605 unsigned OldArgCount = CI->arg_size();
5606 unsigned NewArgCount = NewFn->arg_size();
5607
5608 // If the caller already supplied all arguments (or more), nothing to do.
5609 // This mirrors C++ semantics: an explicitly-passed value is never overridden.
5610 if (OldArgCount >= NewArgCount)
5611 return false;
5612
5613 // Start with the existing arguments from the old call.
5614 SmallVector<Value *, 8> NewArgs(CI->args());
5615
5616 // Defaults are a contiguous trailing block, so checking the first missing
5617 // argument is enough.
5618 if (OldArgCount < FirstDefault)
5619 return false;
5620
5621 // Fill in each missing trailing argument from the table.
5622 FunctionType *NewFT = NewFn->getFunctionType();
5623 for (unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5624 assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5625 "missing argument outside the default range");
5626 Type *ParamTy = NewFT->getParamType(Idx);
5627
5628 // Only integer types are supported (i1, i8, i16, i32, i64).
5629 if (!ParamTy->isIntegerTy())
5630 return false;
5631 NewArgs.push_back(ConstantInt::get(ParamTy, Defaults[Idx - FirstDefault]));
5632 }
5633
5634 // Preserve operand bundles by creating the call with them.
5635 SmallVector<OperandBundleDef, 1> OpBundles;
5636 CI->getOperandBundlesAsDefs(OpBundles);
5637 CallInst *NewCall = Builder.CreateCall(NewFn, NewArgs, OpBundles);
5638
5639 NewCall->takeName(CI);
5640 NewCall->setCallingConv(CI->getCallingConv());
5641 NewCall->copyMetadata(*CI);
5642 if (auto *OldCI = dyn_cast<CallInst>(CI))
5643 NewCall->setTailCallKind(OldCI->getTailCallKind());
5644
5645 CI->replaceAllUsesWith(NewCall);
5646 CI->eraseFromParent();
5647 return true;
5648}
5649
5650/// Upgrade a call to an old intrinsic. All argument and return casting must be
5651/// provided to seamlessly integrate with existing context.
5652void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
5653 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
5654 // checks the callee's function type matches. It's likely we need to handle
5655 // type changes here.
5656 Function *F = dyn_cast<Function>(CI->getCalledOperand());
5657 if (!F)
5658 return;
5659
5660 LLVMContext &C = CI->getContext();
5661 IRBuilder<> Builder(C);
5662 if (isa<FPMathOperator>(CI))
5663 Builder.setFastMathFlags(CI->getFastMathFlags());
5664 Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
5665
5666 if (!NewFn) {
5667 // Get the Function's name.
5668 StringRef Name = F->getName();
5669 if (!Name.consume_front("llvm."))
5670 llvm_unreachable("intrinsic doesn't start with 'llvm.'");
5671
5672 bool IsX86 = Name.consume_front("x86.");
5673 bool IsNVVM = Name.consume_front("nvvm.");
5674 bool IsAArch64 = Name.consume_front("aarch64.");
5675 bool IsARM = Name.consume_front("arm.");
5676 bool IsAMDGCN = Name.consume_front("amdgcn.");
5677 bool IsDbg = Name.consume_front("dbg.");
5678 bool IsOldSplice =
5679 (Name.consume_front("experimental.vector.splice") ||
5680 Name.consume_front("vector.splice")) &&
5681 !(Name.starts_with(".left") || Name.starts_with(".right"));
5682 Value *Rep = nullptr;
5683
5684 if (!IsX86 && Name == "stackprotectorcheck") {
5685 Rep = nullptr;
5686 } else if (IsNVVM) {
5687 Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5688 } else if (IsX86) {
5689 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5690 } else if (IsAArch64) {
5691 Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5692 } else if (IsARM) {
5693 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5694 } else if (IsAMDGCN) {
5695 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5696 } else if (IsDbg) {
5697 upgradeDbgIntrinsicToDbgRecord(Name, CI);
5698 } else if (IsOldSplice) {
5699 Rep = upgradeVectorSplice(CI, Builder);
5700 } else if (Name.consume_front("convert.")) {
5701 Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5702 } else if (Name == "lifetime.start.i64" || Name == "lifetime.end.i64") {
5703 // Delete calls to invalid @llvm.lifetime.{start,end}.i64 intrinsics.
5704 Rep = nullptr;
5705 } else if (shouldUpgradeVPIntrinsic(Name)) {
5706 Rep = upgradeVPIntrinsicCall(Name, CI, Builder);
5707 } else {
5708 llvm_unreachable("Unknown function for CallBase upgrade.");
5709 }
5710
5711 if (Rep)
5712 CI->replaceAllUsesWith(Rep);
5713 CI->eraseFromParent();
5714 return;
5715 }
5716
5717 const auto &DefaultCase = [&]() -> void {
5718 if (F == NewFn)
5719 return;
5720
5721 if (CI->getFunctionType() == NewFn->getFunctionType()) {
5722 // Handle generic mangling change.
5723 assert(
5724 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
5725 "Unknown function for CallBase upgrade and isn't just a name change");
5726 CI->setCalledFunction(NewFn);
5727 return;
5728 }
5729
5730 // This must be an upgrade from a named to a literal struct.
5731 if (auto *OldST = dyn_cast<StructType>(CI->getType())) {
5732 assert(OldST != NewFn->getReturnType() &&
5733 "Return type must have changed");
5734 assert(OldST->getNumElements() ==
5735 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5736 "Must have same number of elements");
5737
5738 SmallVector<Value *> Args(CI->args());
5739 CallInst *NewCI = Builder.CreateCall(NewFn, Args);
5740 NewCI->setAttributes(CI->getAttributes());
5741 Value *Res = PoisonValue::get(OldST);
5742 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5743 Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
5744 Res = Builder.CreateInsertValue(Res, Elem, Idx);
5745 }
5746 CI->replaceAllUsesWith(Res);
5747 CI->eraseFromParent();
5748 return;
5749 }
5750
5751 // We're probably about to produce something invalid. Let the verifier catch
5752 // it instead of dying here.
5753 CI->setCalledOperand(
5754 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
5755 return;
5756 };
5757 CallInst *NewCall = nullptr;
5758 switch (NewFn->getIntrinsicID()) {
5759 default: {
5760 // Last resort: try the data-driven default-arg upgrade.
5761 // Handles any intrinsic annotated with ImmArg<..., DefaultValue<...>>
5762 // in its .td definition, without needing a dedicated case.
5763 if (upgradeIntrinsicCallWithDefaultArgs(CI, NewFn, Builder))
5764 return;
5765 DefaultCase();
5766 return;
5767 }
5768 case Intrinsic::arm_neon_vst1:
5769 case Intrinsic::arm_neon_vst2:
5770 case Intrinsic::arm_neon_vst3:
5771 case Intrinsic::arm_neon_vst4:
5772 case Intrinsic::arm_neon_vst2lane:
5773 case Intrinsic::arm_neon_vst3lane:
5774 case Intrinsic::arm_neon_vst4lane: {
5775 SmallVector<Value *, 4> Args(CI->args());
5776 NewCall = Builder.CreateCall(NewFn, Args);
5777 break;
5778 }
5779 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5780 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5781 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5782 LLVMContext &Ctx = F->getParent()->getContext();
5783 SmallVector<Value *, 4> Args(CI->args());
5784 Args[3] = ConstantInt::get(Type::getInt32Ty(Ctx),
5785 cast<ConstantInt>(Args[3])->getZExtValue());
5786 NewCall = Builder.CreateCall(NewFn, Args);
5787 break;
5788 }
5789 case Intrinsic::aarch64_sve_ld3_sret:
5790 case Intrinsic::aarch64_sve_ld4_sret:
5791 case Intrinsic::aarch64_sve_ld2_sret: {
5792 // Is this a trivial remangle of the name to support ptr address spaces?
5793 if (isa<StructType>(F->getReturnType())) {
5794 DefaultCase();
5795 return;
5796 }
5797
5798 StringRef Name = F->getName();
5799 Name = Name.substr(5);
5800 unsigned N = StringSwitch<unsigned>(Name)
5801 .StartsWith("aarch64.sve.ld2", 2)
5802 .StartsWith("aarch64.sve.ld3", 3)
5803 .StartsWith("aarch64.sve.ld4", 4)
5804 .Default(0);
5805 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5806 unsigned MinElts = RetTy->getMinNumElements() / N;
5807 SmallVector<Value *, 2> Args(CI->args());
5808 Value *NewLdCall = Builder.CreateCall(NewFn, Args);
5809 Value *Ret = llvm::PoisonValue::get(RetTy);
5810 for (unsigned I = 0; I < N; I++) {
5811 Value *SRet = Builder.CreateExtractValue(NewLdCall, I);
5812 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, I * MinElts);
5813 }
5814 NewCall = dyn_cast<CallInst>(Ret);
5815 break;
5816 }
5817
5818 case Intrinsic::coro_end_async:
5819 case Intrinsic::coro_end: {
5820 SmallVector<Value *, 3> Args(CI->args());
5821 if (NewFn->getIntrinsicID() == Intrinsic::coro_end && Args.size() == 2)
5822 Args.push_back(ConstantTokenNone::get(CI->getContext()));
5823 NewCall = Builder.CreateCall(NewFn, Args);
5824
5825 if (!CI->getType()->isVoidTy()) {
5826 if (!CI->use_empty()) {
5827 Function *IsInRamp = Intrinsic::getOrInsertDeclaration(
5828 CI->getModule(), Intrinsic::coro_is_in_ramp);
5829 Value *InRamp = Builder.CreateCall(IsInRamp);
5830 CI->replaceAllUsesWith(Builder.CreateNot(InRamp));
5831 }
5832 CI->eraseFromParent();
5833 return;
5834 }
5835
5836 break;
5837 }
5838
5839 case Intrinsic::vector_extract: {
5840 StringRef Name = F->getName();
5841 Name = Name.substr(5); // Strip llvm
5842 if (!Name.starts_with("aarch64.sve.tuple.get")) {
5843 DefaultCase();
5844 return;
5845 }
5846 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5847 unsigned MinElts = RetTy->getMinNumElements();
5848 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5849 Value *NewIdx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
5850 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});
5851 break;
5852 }
5853
5854 case Intrinsic::vector_insert: {
5855 StringRef Name = F->getName();
5856 Name = Name.substr(5);
5857 if (!Name.starts_with("aarch64.sve.tuple")) {
5858 DefaultCase();
5859 return;
5860 }
5861 if (Name.starts_with("aarch64.sve.tuple.set")) {
5862 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5863 auto *Ty = cast<ScalableVectorType>(CI->getArgOperand(2)->getType());
5864 Value *NewIdx =
5865 ConstantInt::get(Type::getInt64Ty(C), I * Ty->getMinNumElements());
5866 NewCall = Builder.CreateCall(
5867 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx});
5868 break;
5869 }
5870 if (Name.starts_with("aarch64.sve.tuple.create")) {
5871 unsigned N = StringSwitch<unsigned>(Name)
5872 .StartsWith("aarch64.sve.tuple.create2", 2)
5873 .StartsWith("aarch64.sve.tuple.create3", 3)
5874 .StartsWith("aarch64.sve.tuple.create4", 4)
5875 .Default(0);
5876 assert(N > 1 && "Create is expected to be between 2-4");
5877 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5878 Value *Ret = llvm::PoisonValue::get(RetTy);
5879 unsigned MinElts = RetTy->getMinNumElements() / N;
5880 for (unsigned I = 0; I < N; I++) {
5881 Value *V = CI->getArgOperand(I);
5882 Ret = Builder.CreateInsertVector(RetTy, Ret, V, I * MinElts);
5883 }
5884 NewCall = dyn_cast<CallInst>(Ret);
5885 }
5886 break;
5887 }
5888
5889 case Intrinsic::arm_neon_bfdot:
5890 case Intrinsic::arm_neon_bfmmla:
5891 case Intrinsic::arm_neon_bfmlalb:
5892 case Intrinsic::arm_neon_bfmlalt:
5893 case Intrinsic::aarch64_neon_bfdot:
5894 case Intrinsic::aarch64_neon_bfmmla:
5895 case Intrinsic::aarch64_neon_bfmlalb:
5896 case Intrinsic::aarch64_neon_bfmlalt: {
5897 SmallVector<Value *, 3> Args;
5898 assert(CI->arg_size() == 3 &&
5899 "Mismatch between function args and call args");
5900 size_t OperandWidth =
5901 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
5902 assert((OperandWidth == 64 || OperandWidth == 128) &&
5903 "Unexpected operand width");
5904 Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
5905 auto Iter = CI->args().begin();
5906 Args.push_back(*Iter++);
5907 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
5908 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
5909 NewCall = Builder.CreateCall(NewFn, Args);
5910 break;
5911 }
5912
5913 case Intrinsic::bitreverse:
5914 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
5915 break;
5916
5917 case Intrinsic::ctlz:
5918 case Intrinsic::cttz: {
5919 if (CI->arg_size() != 1) {
5920 DefaultCase();
5921 return;
5922 }
5923
5924 NewCall =
5925 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
5926 break;
5927 }
5928
5929 case Intrinsic::objectsize: {
5930 Value *NullIsUnknownSize =
5931 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2);
5932 Value *Dynamic =
5933 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
5934 NewCall = Builder.CreateCall(
5935 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
5936 break;
5937 }
5938
5939 case Intrinsic::ctpop:
5940 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
5941 break;
5942 case Intrinsic::dbg_value: {
5943 StringRef Name = F->getName();
5944 Name = Name.substr(5); // Strip llvm.
5945 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
5946 if (Name.starts_with("dbg.addr")) {
5947 DIExpression *Expr = cast<DIExpression>(
5948 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata());
5949 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
5950 NewCall =
5951 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
5952 MetadataAsValue::get(C, Expr)});
5953 break;
5954 }
5955
5956 // Upgrade from the old version that had an extra offset argument.
5957 assert(CI->arg_size() == 4);
5958 // Drop nonzero offsets instead of attempting to upgrade them.
5959 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
5960 if (Offset->isNullValue()) {
5961 NewCall = Builder.CreateCall(
5962 NewFn,
5963 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
5964 break;
5965 }
5966 CI->eraseFromParent();
5967 return;
5968 }
5969
5970 case Intrinsic::ptr_annotation:
5971 // Upgrade from versions that lacked the annotation attribute argument.
5972 if (CI->arg_size() != 4) {
5973 DefaultCase();
5974 return;
5975 }
5976
5977 // Create a new call with an added null annotation attribute argument.
5978 NewCall = Builder.CreateCall(
5979 NewFn,
5980 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
5981 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
5982 NewCall->takeName(CI);
5983 CI->replaceAllUsesWith(NewCall);
5984 CI->eraseFromParent();
5985 return;
5986
5987 case Intrinsic::var_annotation:
5988 // Upgrade from versions that lacked the annotation attribute argument.
5989 if (CI->arg_size() != 4) {
5990 DefaultCase();
5991 return;
5992 }
5993 // Create a new call with an added null annotation attribute argument.
5994 NewCall = Builder.CreateCall(
5995 NewFn,
5996 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
5997 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
5998 NewCall->takeName(CI);
5999 CI->replaceAllUsesWith(NewCall);
6000 CI->eraseFromParent();
6001 return;
6002
6003 case Intrinsic::riscv_aes32dsi:
6004 case Intrinsic::riscv_aes32dsmi:
6005 case Intrinsic::riscv_aes32esi:
6006 case Intrinsic::riscv_aes32esmi:
6007 case Intrinsic::riscv_sm4ks:
6008 case Intrinsic::riscv_sm4ed: {
6009 // The last argument to these intrinsics used to be i8 and changed to i32.
6010 // The type overload for sm4ks and sm4ed was removed.
6011 Value *Arg2 = CI->getArgOperand(2);
6012 if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64))
6013 return;
6014
6015 Value *Arg0 = CI->getArgOperand(0);
6016 Value *Arg1 = CI->getArgOperand(1);
6017 if (CI->getType()->isIntegerTy(64)) {
6018 Arg0 = Builder.CreateTrunc(Arg0, Builder.getInt32Ty());
6019 Arg1 = Builder.CreateTrunc(Arg1, Builder.getInt32Ty());
6020 }
6021
6022 Arg2 = ConstantInt::get(Type::getInt32Ty(C),
6023 cast<ConstantInt>(Arg2)->getZExtValue());
6024
6025 NewCall = Builder.CreateCall(NewFn, {Arg0, Arg1, Arg2});
6026 Value *Res = NewCall;
6027 if (Res->getType() != CI->getType())
6028 Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6029 NewCall->takeName(CI);
6030 CI->replaceAllUsesWith(Res);
6031 CI->eraseFromParent();
6032 return;
6033 }
6034 case Intrinsic::nvvm_mapa_shared_cluster: {
6035 // Create a new call with the correct address space.
6036 NewCall =
6037 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1)});
6038 Value *Res = NewCall;
6039 Res = Builder.CreateAddrSpaceCast(
6040 Res, Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED));
6041 NewCall->takeName(CI);
6042 CI->replaceAllUsesWith(Res);
6043 CI->eraseFromParent();
6044 return;
6045 }
6046 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster:
6047 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
6048 // Create a new call with the correct address space.
6049 SmallVector<Value *, 4> Args(CI->args());
6050 Args[0] = Builder.CreateAddrSpaceCast(
6051 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6052
6053 NewCall = Builder.CreateCall(NewFn, Args);
6054 NewCall->takeName(CI);
6055 CI->replaceAllUsesWith(NewCall);
6056 CI->eraseFromParent();
6057 return;
6058 }
6059 // clang-format off
6060#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME) \
6061 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX:
6062 NVVM_TMA_G2S_MODES(G2S_CLUSTER_CASE)
6063#undef G2S_CLUSTER_CASE
6064 {
6065 SmallVector<Value *, 16> Args(CI->args());
6066 unsigned AS = CI->getArgOperand(0)->getType()->getPointerAddressSpace();
6067 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6068 Args[0] = Builder.CreateAddrSpaceCast(
6069 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6070
6071 // Append the missing trailing arguments with default values (cta_group,
6072 // validate_pattern).
6073 while (Args.size() < NewFn->getFunctionType()->getNumParams())
6074 Args.push_back(Builder.getInt32(0));
6075
6076 NewCall = Builder.CreateCall(NewFn, Args);
6077 NewCall->takeName(CI);
6078 CI->replaceAllUsesWith(NewCall);
6079 CI->eraseFromParent();
6080 return;
6081 }
6082
6083#define G2S_CTA_CASE(ID_SUFFIX, NAME) \
6084 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX:
6085 NVVM_TMA_G2S_MODES(G2S_CTA_CASE)
6086#undef G2S_CTA_CASE
6087 {
6088 SmallVector<Value *, 16> Args(CI->args());
6089 // Append the missing trailing validate_pattern argument with default
6090 // value 0.
6091 assert(Args.size() + 1 == NewFn->getFunctionType()->getNumParams() &&
6092 "expected only the trailing validate_pattern to be missing");
6093 Args.push_back(Builder.getInt32(0));
6094
6095 NewCall = Builder.CreateCall(NewFn, Args);
6096 NewCall->takeName(CI);
6097 CI->replaceAllUsesWith(NewCall);
6098 CI->eraseFromParent();
6099 return;
6100 }
6101#undef NVVM_TMA_G2S_MODES
6102 // clang-format on
6103
6104 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d:
6105 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d:
6106 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d:
6107 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d:
6108 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d:
6109 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d:
6110 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d:
6111 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d: {
6112 StringRef Name = F->getName();
6113 Name.consume_front("llvm.nvvm.cp.async.bulk.tensor.reduce.");
6114 auto RedOp = getNVPTXTMAReductionOp(Name.split('.').first);
6115
6116 SmallVector<Value *, 16> Args(CI->args());
6117 Args.insert(Args.end() - 1, Builder.getInt32(*RedOp));
6118 NewCall = Builder.CreateCall(NewFn, Args);
6119 break;
6120 }
6121 case Intrinsic::nvvm_tcgen05_mma_shared:
6122 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:
6123 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:
6124 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4_block_scale:
6125 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4_block_scale_block32:
6126 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4nvf4_block_scale_block16:
6127 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4nvf4_block_scale_block32:
6128 case Intrinsic::nvvm_tcgen05_mma_shared_mxf8f6f4_block_scale:
6129 case Intrinsic::nvvm_tcgen05_mma_shared_mxf8f6f4_block_scale_block32:
6130 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d:
6131 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:
6132 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:
6133 case Intrinsic::nvvm_tcgen05_mma_sp_shared:
6134 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:
6135 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:
6136 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4_block_scale:
6137 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4_block_scale_block32:
6138 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4nvf4_block_scale_block16:
6139 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4nvf4_block_scale_block32:
6140 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf8f6f4_block_scale:
6141 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf8f6f4_block_scale_block32:
6142 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d:
6143 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:
6144 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:
6145 case Intrinsic::nvvm_tcgen05_mma_sp_tensor:
6146 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_ashift:
6147 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:
6148 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:
6149 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:
6150 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:
6151 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4_block_scale:
6152 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4_block_scale_block32:
6153 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4nvf4_block_scale_block16:
6154 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4nvf4_block_scale_block32:
6155 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf8f6f4_block_scale:
6156 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf8f6f4_block_scale_block32:
6157 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d:
6158 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_ashift:
6159 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:
6160 case Intrinsic::
6161 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift:
6162 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:
6163 case Intrinsic::
6164 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift:
6165 case Intrinsic::nvvm_tcgen05_mma_tensor:
6166 case Intrinsic::nvvm_tcgen05_mma_tensor_ashift:
6167 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:
6168 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:
6169 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:
6170 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:
6171 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4_block_scale:
6172 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4_block_scale_block32:
6173 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4nvf4_block_scale_block16:
6174 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4nvf4_block_scale_block32:
6175 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf8f6f4_block_scale:
6176 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf8f6f4_block_scale_block32:
6177 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d:
6178 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_ashift:
6179 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:
6180 case Intrinsic::
6181 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:
6182 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:
6183 case Intrinsic::
6184 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift: {
6185 SmallVector<Value *, 12> Args(CI->args());
6186 Args.push_back(Builder.getInt32(0)); // collector_usage_b = discard(0)
6187 NewCall = Builder.CreateCall(NewFn, Args);
6188 break;
6189 }
6190 case Intrinsic::nvvm_tcgen05_alloc_cg1:
6191 case Intrinsic::nvvm_tcgen05_alloc_cg2:
6192 case Intrinsic::nvvm_tcgen05_dealloc_cg1:
6193 case Intrinsic::nvvm_tcgen05_dealloc_cg2:
6194 NewCall =
6195 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
6196 Builder.getFalse()});
6197 break;
6198 case Intrinsic::riscv_sha256sig0:
6199 case Intrinsic::riscv_sha256sig1:
6200 case Intrinsic::riscv_sha256sum0:
6201 case Intrinsic::riscv_sha256sum1:
6202 case Intrinsic::riscv_sm3p0:
6203 case Intrinsic::riscv_sm3p1: {
6204 // The last argument to these intrinsics used to be i8 and changed to i32.
6205 // The type overload for sm4ks and sm4ed was removed.
6206 if (!CI->getType()->isIntegerTy(64))
6207 return;
6208
6209 Value *Arg =
6210 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty());
6211
6212 NewCall = Builder.CreateCall(NewFn, Arg);
6213 Value *Res =
6214 Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6215 NewCall->takeName(CI);
6216 CI->replaceAllUsesWith(Res);
6217 CI->eraseFromParent();
6218 return;
6219 }
6220
6221 case Intrinsic::x86_xop_vfrcz_ss:
6222 case Intrinsic::x86_xop_vfrcz_sd:
6223 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
6224 break;
6225
6226 case Intrinsic::x86_xop_vpermil2pd:
6227 case Intrinsic::x86_xop_vpermil2ps:
6228 case Intrinsic::x86_xop_vpermil2pd_256:
6229 case Intrinsic::x86_xop_vpermil2ps_256: {
6230 SmallVector<Value *, 4> Args(CI->args());
6231 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
6232 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
6233 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
6234 NewCall = Builder.CreateCall(NewFn, Args);
6235 break;
6236 }
6237
6238 case Intrinsic::x86_sse41_ptestc:
6239 case Intrinsic::x86_sse41_ptestz:
6240 case Intrinsic::x86_sse41_ptestnzc: {
6241 // The arguments for these intrinsics used to be v4f32, and changed
6242 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
6243 // So, the only thing required is a bitcast for both arguments.
6244 // First, check the arguments have the old type.
6245 Value *Arg0 = CI->getArgOperand(0);
6246 if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
6247 return;
6248
6249 // Old intrinsic, add bitcasts
6250 Value *Arg1 = CI->getArgOperand(1);
6251
6252 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
6253
6254 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
6255 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
6256
6257 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
6258 break;
6259 }
6260
6261 case Intrinsic::x86_rdtscp: {
6262 // This used to take 1 arguments. If we have no arguments, it is already
6263 // upgraded.
6264 if (CI->getNumOperands() == 0)
6265 return;
6266
6267 NewCall = Builder.CreateCall(NewFn);
6268 // Extract the second result and store it.
6269 Value *Data = Builder.CreateExtractValue(NewCall, 1);
6270 Builder.CreateAlignedStore(Data, CI->getArgOperand(0), Align(1));
6271 // Replace the original call result with the first result of the new call.
6272 Value *TSC = Builder.CreateExtractValue(NewCall, 0);
6273
6274 NewCall->takeName(CI);
6275 CI->replaceAllUsesWith(TSC);
6276 CI->eraseFromParent();
6277 return;
6278 }
6279
6280 case Intrinsic::x86_sse41_insertps:
6281 case Intrinsic::x86_sse41_dppd:
6282 case Intrinsic::x86_sse41_dpps:
6283 case Intrinsic::x86_sse41_mpsadbw:
6284 case Intrinsic::x86_avx_dp_ps_256:
6285 case Intrinsic::x86_avx2_mpsadbw: {
6286 // Need to truncate the last argument from i32 to i8 -- this argument models
6287 // an inherently 8-bit immediate operand to these x86 instructions.
6288 SmallVector<Value *, 4> Args(CI->args());
6289
6290 // Replace the last argument with a trunc.
6291 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
6292 NewCall = Builder.CreateCall(NewFn, Args);
6293 break;
6294 }
6295
6296 case Intrinsic::x86_avx512_mask_cmp_pd_128:
6297 case Intrinsic::x86_avx512_mask_cmp_pd_256:
6298 case Intrinsic::x86_avx512_mask_cmp_pd_512:
6299 case Intrinsic::x86_avx512_mask_cmp_ps_128:
6300 case Intrinsic::x86_avx512_mask_cmp_ps_256:
6301 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
6302 SmallVector<Value *, 4> Args(CI->args());
6303 unsigned NumElts =
6304 cast<FixedVectorType>(Args[0]->getType())->getNumElements();
6305 Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
6306
6307 NewCall = Builder.CreateCall(NewFn, Args);
6308 Value *Res = applyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
6309
6310 NewCall->takeName(CI);
6311 CI->replaceAllUsesWith(Res);
6312 CI->eraseFromParent();
6313 return;
6314 }
6315
6316 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
6317 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
6318 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
6319 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
6320 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
6321 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
6322 SmallVector<Value *, 4> Args(CI->args());
6323 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
6324 if (NewFn->getIntrinsicID() ==
6325 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
6326 Args[1] = Builder.CreateBitCast(
6327 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6328
6329 NewCall = Builder.CreateCall(NewFn, Args);
6330 Value *Res = Builder.CreateBitCast(
6331 NewCall, FixedVectorType::get(Builder.getInt16Ty(), NumElts));
6332
6333 NewCall->takeName(CI);
6334 CI->replaceAllUsesWith(Res);
6335 CI->eraseFromParent();
6336 return;
6337 }
6338 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6339 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6340 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
6341 SmallVector<Value *, 4> Args(CI->args());
6342 unsigned NumElts =
6343 cast<FixedVectorType>(CI->getType())->getNumElements() * 2;
6344 Args[1] = Builder.CreateBitCast(
6345 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6346 Args[2] = Builder.CreateBitCast(
6347 Args[2], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6348
6349 NewCall = Builder.CreateCall(NewFn, Args);
6350 break;
6351 }
6352
6353 case Intrinsic::thread_pointer: {
6354 NewCall = Builder.CreateCall(NewFn, {});
6355 break;
6356 }
6357
6358 case Intrinsic::memcpy:
6359 case Intrinsic::memmove:
6360 case Intrinsic::memset: {
6361 // We have to make sure that the call signature is what we're expecting.
6362 // We only want to change the old signatures by removing the alignment arg:
6363 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
6364 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
6365 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
6366 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
6367 // Note: i8*'s in the above can be any pointer type
6368 if (CI->arg_size() != 5) {
6369 DefaultCase();
6370 return;
6371 }
6372 // Remove alignment argument (3), and add alignment attributes to the
6373 // dest/src pointers.
6374 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
6375 CI->getArgOperand(2), CI->getArgOperand(4)};
6376 NewCall = Builder.CreateCall(NewFn, Args);
6377 AttributeList OldAttrs = CI->getAttributes();
6378 AttributeList NewAttrs = AttributeList::get(
6379 C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),
6380 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
6381 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
6382 NewCall->setAttributes(NewAttrs);
6383 auto *MemCI = cast<MemIntrinsic>(NewCall);
6384 // All mem intrinsics support dest alignment.
6385 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
6386 MemCI->setDestAlignment(Align->getMaybeAlignValue());
6387 // Memcpy/Memmove also support source alignment.
6388 if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
6389 MTI->setSourceAlignment(Align->getMaybeAlignValue());
6390 break;
6391 }
6392
6393 case Intrinsic::masked_load:
6394 case Intrinsic::masked_gather:
6395 case Intrinsic::masked_store:
6396 case Intrinsic::masked_scatter: {
6397 if (CI->arg_size() != 4) {
6398 DefaultCase();
6399 return;
6400 }
6401
6402 auto GetMaybeAlign = [](Value *Op) {
6403 if (auto *CI = dyn_cast<ConstantInt>(Op)) {
6404 uint64_t Val = CI->getZExtValue();
6405 if (Val == 0)
6406 return MaybeAlign();
6407 if (isPowerOf2_64(Val))
6408 return MaybeAlign(Val);
6409 }
6410 reportFatalUsageError("Invalid alignment argument");
6411 };
6412 auto GetAlign = [&](Value *Op) {
6413 MaybeAlign Align = GetMaybeAlign(Op);
6414 if (Align)
6415 return *Align;
6416 reportFatalUsageError("Invalid zero alignment argument");
6417 };
6418
6419 const DataLayout &DL = CI->getDataLayout();
6420 switch (NewFn->getIntrinsicID()) {
6421 case Intrinsic::masked_load:
6422 NewCall = Builder.CreateMaskedLoad(
6423 CI->getType(), CI->getArgOperand(0), GetAlign(CI->getArgOperand(1)),
6424 CI->getArgOperand(2), CI->getArgOperand(3));
6425 break;
6426 case Intrinsic::masked_gather:
6427 NewCall = Builder.CreateMaskedGather(
6428 CI->getType(), CI->getArgOperand(0),
6429 DL.getValueOrABITypeAlignment(GetMaybeAlign(CI->getArgOperand(1)),
6430 CI->getType()->getScalarType()),
6431 CI->getArgOperand(2), CI->getArgOperand(3));
6432 break;
6433 case Intrinsic::masked_store:
6434 NewCall = Builder.CreateMaskedStore(
6435 CI->getArgOperand(0), CI->getArgOperand(1),
6436 GetAlign(CI->getArgOperand(2)), CI->getArgOperand(3));
6437 break;
6438 case Intrinsic::masked_scatter:
6439 NewCall = Builder.CreateMaskedScatter(
6440 CI->getArgOperand(0), CI->getArgOperand(1),
6441 DL.getValueOrABITypeAlignment(
6442 GetMaybeAlign(CI->getArgOperand(2)),
6443 CI->getArgOperand(0)->getType()->getScalarType()),
6444 CI->getArgOperand(3));
6445 break;
6446 default:
6447 llvm_unreachable("Unexpected intrinsic ID");
6448 }
6449 // Previous metadata is still valid.
6450 NewCall->copyMetadata(*CI);
6451 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6452 break;
6453 }
6454
6455 case Intrinsic::lifetime_start:
6456 case Intrinsic::lifetime_end: {
6457 if (CI->arg_size() != 2) {
6458 DefaultCase();
6459 return;
6460 }
6461
6462 Value *Ptr = CI->getArgOperand(1);
6463 // Try to strip pointer casts, such that the lifetime works on an alloca.
6464 Ptr = Ptr->stripPointerCasts();
6465 if (isa<AllocaInst>(Ptr)) {
6466 // Don't use NewFn, as we might have looked through an addrspacecast.
6467 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
6468 NewCall = Builder.CreateLifetimeStart(Ptr);
6469 else
6470 NewCall = Builder.CreateLifetimeEnd(Ptr);
6471 break;
6472 }
6473
6474 // Otherwise remove the lifetime marker.
6475 CI->eraseFromParent();
6476 return;
6477 }
6478
6479 case Intrinsic::x86_avx512_vpdpbusd_128:
6480 case Intrinsic::x86_avx512_vpdpbusd_256:
6481 case Intrinsic::x86_avx512_vpdpbusd_512:
6482 case Intrinsic::x86_avx512_vpdpbusds_128:
6483 case Intrinsic::x86_avx512_vpdpbusds_256:
6484 case Intrinsic::x86_avx512_vpdpbusds_512:
6485 case Intrinsic::x86_avx2_vpdpbssd_128:
6486 case Intrinsic::x86_avx2_vpdpbssd_256:
6487 case Intrinsic::x86_avx10_vpdpbssd_512:
6488 case Intrinsic::x86_avx2_vpdpbssds_128:
6489 case Intrinsic::x86_avx2_vpdpbssds_256:
6490 case Intrinsic::x86_avx10_vpdpbssds_512:
6491 case Intrinsic::x86_avx2_vpdpbsud_128:
6492 case Intrinsic::x86_avx2_vpdpbsud_256:
6493 case Intrinsic::x86_avx10_vpdpbsud_512:
6494 case Intrinsic::x86_avx2_vpdpbsuds_128:
6495 case Intrinsic::x86_avx2_vpdpbsuds_256:
6496 case Intrinsic::x86_avx10_vpdpbsuds_512:
6497 case Intrinsic::x86_avx2_vpdpbuud_128:
6498 case Intrinsic::x86_avx2_vpdpbuud_256:
6499 case Intrinsic::x86_avx10_vpdpbuud_512:
6500 case Intrinsic::x86_avx2_vpdpbuuds_128:
6501 case Intrinsic::x86_avx2_vpdpbuuds_256:
6502 case Intrinsic::x86_avx10_vpdpbuuds_512: {
6503 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8;
6504 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6505 CI->getArgOperand(2)};
6506 Type *NewArgType = VectorType::get(Builder.getInt8Ty(), NumElts, false);
6507 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6508 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6509
6510 NewCall = Builder.CreateCall(NewFn, Args);
6511 break;
6512 }
6513 case Intrinsic::x86_avx512_vpdpwssd_128:
6514 case Intrinsic::x86_avx512_vpdpwssd_256:
6515 case Intrinsic::x86_avx512_vpdpwssd_512:
6516 case Intrinsic::x86_avx512_vpdpwssds_128:
6517 case Intrinsic::x86_avx512_vpdpwssds_256:
6518 case Intrinsic::x86_avx512_vpdpwssds_512:
6519 case Intrinsic::x86_avx2_vpdpwsud_128:
6520 case Intrinsic::x86_avx2_vpdpwsud_256:
6521 case Intrinsic::x86_avx10_vpdpwsud_512:
6522 case Intrinsic::x86_avx2_vpdpwsuds_128:
6523 case Intrinsic::x86_avx2_vpdpwsuds_256:
6524 case Intrinsic::x86_avx10_vpdpwsuds_512:
6525 case Intrinsic::x86_avx2_vpdpwusd_128:
6526 case Intrinsic::x86_avx2_vpdpwusd_256:
6527 case Intrinsic::x86_avx10_vpdpwusd_512:
6528 case Intrinsic::x86_avx2_vpdpwusds_128:
6529 case Intrinsic::x86_avx2_vpdpwusds_256:
6530 case Intrinsic::x86_avx10_vpdpwusds_512:
6531 case Intrinsic::x86_avx2_vpdpwuud_128:
6532 case Intrinsic::x86_avx2_vpdpwuud_256:
6533 case Intrinsic::x86_avx10_vpdpwuud_512:
6534 case Intrinsic::x86_avx2_vpdpwuuds_128:
6535 case Intrinsic::x86_avx2_vpdpwuuds_256:
6536 case Intrinsic::x86_avx10_vpdpwuuds_512:
6537 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16;
6538 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6539 CI->getArgOperand(2)};
6540 Type *NewArgType = VectorType::get(Builder.getInt16Ty(), NumElts, false);
6541 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6542 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6543
6544 NewCall = Builder.CreateCall(NewFn, Args);
6545 break;
6546 }
6547 assert(NewCall && "Should have either set this variable or returned through "
6548 "the default case");
6549 NewCall->takeName(CI);
6550 CI->replaceAllUsesWith(NewCall);
6551 CI->eraseFromParent();
6552}
6553
6554void llvm::UpgradeCallsToIntrinsic(Function *F) {
6555 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
6556
6557 // Check if this function should be upgraded and get the replacement function
6558 // if there is one.
6559 Function *NewFn;
6560 if (UpgradeIntrinsicFunction(F, NewFn)) {
6561 // Replace all users of the old function with the new function or new
6562 // instructions. This is not a range loop because the call is deleted.
6563 for (User *U : make_early_inc_range(F->users()))
6564 if (CallBase *CB = dyn_cast<CallBase>(U))
6565 UpgradeIntrinsicCall(CB, NewFn);
6566
6567 // Remove old function, no longer used, from the module.
6568 if (F != NewFn)
6569 F->eraseFromParent();
6570 }
6571}
6572
6573MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
6574 const unsigned NumOperands = MD.getNumOperands();
6575 if (NumOperands == 0)
6576 return &MD; // Invalid, punt to a verifier error.
6577
6578 // Check if the tag uses struct-path aware TBAA format.
6579 if (isa<MDNode>(MD.getOperand(0)) && NumOperands >= 3)
6580 return &MD;
6581
6582 auto &Context = MD.getContext();
6583 if (NumOperands == 3) {
6584 Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
6585 MDNode *ScalarType = MDNode::get(Context, Elts);
6586 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
6587 Metadata *Elts2[] = {ScalarType, ScalarType,
6588 ConstantAsMetadata::get(
6589 Constant::getNullValue(Type::getInt64Ty(Context))),
6590 MD.getOperand(2)};
6591 return MDNode::get(Context, Elts2);
6592 }
6593 // Create a MDNode <MD, MD, offset 0>
6594 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
6595 Type::getInt64Ty(Context)))};
6596 return MDNode::get(Context, Elts);
6597}
6598
6599Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
6600 Instruction *&Temp) {
6601 if (Opc != Instruction::BitCast)
6602 return nullptr;
6603
6604 Temp = nullptr;
6605 Type *SrcTy = V->getType();
6606 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6607 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6608 LLVMContext &Context = V->getContext();
6609
6610 // We have no information about target data layout, so we assume that
6611 // the maximum pointer size is 64bit.
6612 Type *MidTy = Type::getInt64Ty(Context);
6613 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
6614
6615 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
6616 }
6617
6618 return nullptr;
6619}
6620
6621Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
6622 if (Opc != Instruction::BitCast)
6623 return nullptr;
6624
6625 Type *SrcTy = C->getType();
6626 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6627 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6628 LLVMContext &Context = C->getContext();
6629
6630 // We have no information about target data layout, so we assume that
6631 // the maximum pointer size is 64bit.
6632 Type *MidTy = Type::getInt64Ty(Context);
6633
6634 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
6635 DestTy);
6636 }
6637
6638 return nullptr;
6639}
6640
6641static std::optional<StringRef> getModuleFlagNameSafely(const MDNode &Flag) {
6642 if (Flag.getNumOperands() < 3)
6643 return std::nullopt;
6644 if (MDString *Name = dyn_cast_or_null<MDString>(Flag.getOperand(1)))
6645 return Name->getString();
6646 return std::nullopt;
6647}
6648
6649/// Check the debug info version number, if it is out-dated, drop the debug
6650/// info. Return true if module is modified.
6651bool llvm::UpgradeDebugInfo(Module &M) {
6652 if (DisableAutoUpgradeDebugInfo)
6653 return false;
6654
6655 llvm::TimeTraceScope timeScope("Upgrade debug info");
6656 // We need to get metadata before the module is verified (i.e., getModuleFlag
6657 // makes assumptions that we haven't verified yet). Carefully extract the flag
6658 // from the metadata.
6659 unsigned Version = 0;
6660 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6661 auto OpIt = find_if(ModFlags->operands(), [](const MDNode *Flag) {
6662 if (auto Name = getModuleFlagNameSafely(*Flag))
6663 return *Name == "Debug Info Version";
6664 return false;
6665 });
6666 if (OpIt != ModFlags->op_end()) {
6667 const MDOperand &ValOp = (*OpIt)->getOperand(2);
6668 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(ValOp))
6669 Version = CI->getZExtValue();
6670 }
6671 }
6672
6673 if (Version == DEBUG_METADATA_VERSION) {
6674 bool BrokenDebugInfo = false;
6675 if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
6676 report_fatal_error("Broken module found, compilation aborted!");
6677 if (!BrokenDebugInfo)
6678 // Everything is ok.
6679 return false;
6680 else {
6681 // Diagnose malformed debug info.
6682 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
6683 M.getContext().diagnose(Diag);
6684 }
6685 }
6686 bool Modified = StripDebugInfo(M);
6687 if (Modified && Version != DEBUG_METADATA_VERSION) {
6688 // Diagnose a version mismatch.
6689 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
6690 M.getContext().diagnose(DiagVersion);
6691 }
6692 return Modified;
6693}
6694
6695static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
6696 GlobalValue *GV, const Metadata *V) {
6697 Function *F = cast<Function>(GV);
6698
6699 constexpr StringLiteral DefaultValue = "1";
6700 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
6701 unsigned Length = 0;
6702
6703 if (F->hasFnAttribute(Attr)) {
6704 // We expect the existing attribute to have the form "x[,y[,z]]". Here we
6705 // parse these elements placing them into Vect3
6706 StringRef S = F->getFnAttribute(Attr).getValueAsString();
6707 for (; Length < 3 && !S.empty(); Length++) {
6708 auto [Part, Rest] = S.split(',');
6709 Vect3[Length] = Part.trim();
6710 S = Rest;
6711 }
6712 }
6713
6714 const unsigned Dim = DimC - 'x';
6715 assert(Dim < 3 && "Unexpected dim char");
6716
6717 const uint64_t VInt = mdconst::extract<ConstantInt>(V)->getZExtValue();
6718
6719 // local variable required for StringRef in Vect3 to point to.
6720 const std::string VStr = llvm::utostr(VInt);
6721 Vect3[Dim] = VStr;
6722 Length = std::max(Length, Dim + 1);
6723
6724 const std::string NewAttr = llvm::join(ArrayRef(Vect3, Length), ",");
6725 F->addFnAttr(Attr, NewAttr);
6726}
6727
6728static inline bool isXYZ(StringRef S) {
6729 return S == "x" || S == "y" || S == "z";
6730}
6731
6732bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
6733 const Metadata *V) {
6734 if (K == "kernel") {
6735 if (!mdconst::extract<ConstantInt>(V)->isZero())
6736 cast<Function>(GV)->setCallingConv(CallingConv::PTX_Kernel);
6737 return true;
6738 }
6739 if (K == "align") {
6740 // V is a bitfeild specifying two 16-bit values. The alignment value is
6741 // specfied in low 16-bits, The index is specified in the high bits. For the
6742 // index, 0 indicates the return value while higher values correspond to
6743 // each parameter (idx = param + 1).
6744 const uint64_t AlignIdxValuePair =
6745 mdconst::extract<ConstantInt>(V)->getZExtValue();
6746 const unsigned Idx = (AlignIdxValuePair >> 16);
6747 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
6748 cast<Function>(GV)->addAttributeAtIndex(
6749 Idx, Attribute::getWithStackAlignment(GV->getContext(), StackAlign));
6750 return true;
6751 }
6752 if (K == "maxclusterrank" || K == "cluster_max_blocks") {
6753 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6754 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxClusterRank, llvm::utostr(CV));
6755 return true;
6756 }
6757 if (K == "minctasm") {
6758 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6759 cast<Function>(GV)->addFnAttr(NVVMAttr::MinCTASm, llvm::utostr(CV));
6760 return true;
6761 }
6762 if (K == "maxnreg") {
6763 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6764 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxNReg, llvm::utostr(CV));
6765 return true;
6766 }
6767 if (K.consume_front("maxntid") && isXYZ(K)) {
6768 upgradeNVVMFnVectorAttr(NVVMAttr::MaxNTID, K[0], GV, V);
6769 return true;
6770 }
6771 if (K.consume_front("reqntid") && isXYZ(K)) {
6772 upgradeNVVMFnVectorAttr(NVVMAttr::ReqNTID, K[0], GV, V);
6773 return true;
6774 }
6775 if (K.consume_front("cluster_dim_") && isXYZ(K)) {
6776 upgradeNVVMFnVectorAttr(NVVMAttr::ClusterDim, K[0], GV, V);
6777 return true;
6778 }
6779 if (K == "grid_constant") {
6780 const auto Attr = Attribute::get(GV->getContext(), NVVMAttr::GridConstant);
6781 for (const auto &Op : cast<MDNode>(V)->operands()) {
6782 // For some reason, the index is 1-based in the metadata. Good thing we're
6783 // able to auto-upgrade it!
6784 const auto Index = mdconst::extract<ConstantInt>(Op)->getZExtValue() - 1;
6785 cast<Function>(GV)->addParamAttr(Index, Attr);
6786 }
6787 return true;
6788 }
6789
6790 return false;
6791}
6792
6793void llvm::UpgradeNVVMAnnotations(Module &M) {
6794 NamedMDNode *NamedMD = M.getNamedMetadata("nvvm.annotations");
6795 if (!NamedMD)
6796 return;
6797
6798 SmallVector<MDNode *, 8> NewNodes;
6799 SmallPtrSet<const MDNode *, 8> SeenNodes;
6800 for (MDNode *MD : NamedMD->operands()) {
6801 if (!SeenNodes.insert(MD).second)
6802 continue;
6803
6804 auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD->getOperand(0));
6805 if (!GV)
6806 continue;
6807
6808 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
6809
6810 SmallVector<Metadata *, 8> NewOperands{MD->getOperand(0)};
6811 // Each nvvm.annotations metadata entry will be of the following form:
6812 // !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
6813 // start index = 1, to skip the global variable key
6814 // increment = 2, to skip the value for each property-value pairs
6815 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
6816 MDString *K = cast<MDString>(MD->getOperand(j));
6817 const MDOperand &V = MD->getOperand(j + 1);
6818 bool Upgraded = upgradeSingleNVVMAnnotation(GV, K->getString(), V);
6819 if (!Upgraded)
6820 NewOperands.append({K, V});
6821 }
6822
6823 if (NewOperands.size() > 1)
6824 NewNodes.push_back(MDNode::get(M.getContext(), NewOperands));
6825 }
6826
6827 NamedMD->clearOperands();
6828 for (MDNode *N : NewNodes)
6829 NamedMD->addOperand(N);
6830}
6831
6832/// This checks for objc retain release marker which should be upgraded. It
6833/// returns true if module is modified.
6834static bool upgradeRetainReleaseMarker(Module &M) {
6835 bool Changed = false;
6836 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
6837 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
6838 if (ModRetainReleaseMarker) {
6839 MDNode *Op = ModRetainReleaseMarker->getOperand(0);
6840 if (Op) {
6841 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
6842 if (ID) {
6843 SmallVector<StringRef, 4> ValueComp;
6844 ID->getString().split(ValueComp, "#");
6845 if (ValueComp.size() == 2) {
6846 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
6847 ID = MDString::get(M.getContext(), NewValue);
6848 }
6849 M.addModuleFlag(Module::Error, MarkerKey, ID);
6850 M.eraseNamedMetadata(ModRetainReleaseMarker);
6851 Changed = true;
6852 }
6853 }
6854 }
6855 return Changed;
6856}
6857
6858void llvm::UpgradeARCRuntime(Module &M) {
6859 // This lambda converts normal function calls to ARC runtime functions to
6860 // intrinsic calls.
6861 auto UpgradeToIntrinsic = [&](const char *OldFunc,
6862 llvm::Intrinsic::ID IntrinsicFunc) {
6863 Function *Fn = M.getFunction(OldFunc);
6864
6865 if (!Fn)
6866 return;
6867
6868 Function *NewFn =
6869 llvm::Intrinsic::getOrInsertDeclaration(&M, IntrinsicFunc);
6870
6871 for (User *U : make_early_inc_range(Fn->users())) {
6872 CallInst *CI = dyn_cast<CallInst>(U);
6873 if (!CI || CI->getCalledFunction() != Fn)
6874 continue;
6875
6876 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
6877 FunctionType *NewFuncTy = NewFn->getFunctionType();
6878 SmallVector<Value *, 2> Args;
6879
6880 // Don't upgrade the intrinsic if it's not valid to bitcast the return
6881 // value to the return type of the old function.
6882 if (NewFuncTy->getReturnType() != CI->getType() &&
6883 !CastInst::castIsValid(Instruction::BitCast, CI,
6884 NewFuncTy->getReturnType()))
6885 continue;
6886
6887 bool InvalidCast = false;
6888
6889 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
6890 Value *Arg = CI->getArgOperand(I);
6891
6892 // Bitcast argument to the parameter type of the new function if it's
6893 // not a variadic argument.
6894 if (I < NewFuncTy->getNumParams()) {
6895 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
6896 // to the parameter type of the new function.
6897 if (!CastInst::castIsValid(Instruction::BitCast, Arg,
6898 NewFuncTy->getParamType(I))) {
6899 InvalidCast = true;
6900 break;
6901 }
6902 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
6903 }
6904 Args.push_back(Arg);
6905 }
6906
6907 if (InvalidCast)
6908 continue;
6909
6910 // Create a call instruction that calls the new function.
6911 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
6912 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6913 NewCall->takeName(CI);
6914
6915 // Bitcast the return value back to the type of the old call.
6916 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
6917
6918 if (!CI->use_empty())
6919 CI->replaceAllUsesWith(NewRetVal);
6920 CI->eraseFromParent();
6921 }
6922
6923 if (Fn->use_empty())
6924 Fn->eraseFromParent();
6925 };
6926
6927 // Unconditionally convert a call to "clang.arc.use" to a call to
6928 // "llvm.objc.clang.arc.use".
6929 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
6930
6931 // Upgrade the retain release marker. If there is no need to upgrade
6932 // the marker, that means either the module is already new enough to contain
6933 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
6934 if (!upgradeRetainReleaseMarker(M))
6935 return;
6936
6937 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
6938 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
6939 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
6940 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
6941 {"objc_autoreleaseReturnValue",
6942 llvm::Intrinsic::objc_autoreleaseReturnValue},
6943 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
6944 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
6945 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
6946 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
6947 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
6948 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
6949 {"objc_release", llvm::Intrinsic::objc_release},
6950 {"objc_retain", llvm::Intrinsic::objc_retain},
6951 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
6952 {"objc_retainAutoreleaseReturnValue",
6953 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
6954 {"objc_retainAutoreleasedReturnValue",
6955 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
6956 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
6957 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
6958 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
6959 {"objc_unsafeClaimAutoreleasedReturnValue",
6960 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
6961 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
6962 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
6963 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
6964 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
6965 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
6966 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
6967 {"objc_arc_annotation_topdown_bbstart",
6968 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
6969 {"objc_arc_annotation_topdown_bbend",
6970 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
6971 {"objc_arc_annotation_bottomup_bbstart",
6972 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
6973 {"objc_arc_annotation_bottomup_bbend",
6974 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
6975
6976 for (auto &I : RuntimeFuncs)
6977 UpgradeToIntrinsic(I.first, I.second);
6978}
6979
6980// Upgrade the way signing of pointers to init/fini functions is described.
6981//
6982// Originally, the `@llvm.global_(ctors|dtors)` arrays contained `ptrauth`
6983// constants, if signing was requested. After the upgrade, these arrays contain
6984// plain function pointers and the desired signing schema is described via a
6985// pair of module flags.
6986//
6987// Note that the upgrade is only performed if all elements of *both* arrays
6988// agree on a common signing schema.
6989static bool upgradePtrauthInitFiniArrays(Module &M) {
6990 // As we cannot always decide whether the particular module should have
6991 // ptrauth-init-fini flags, we have to treat absent flags as having zero
6992 // values for compatibility reasons. Thus, upgradePtrauthInitFiniArrays
6993 // returns as soon as it spots any non-signed init/fini pointer: either we
6994 // should request non-signed pointers (safe to omit both flags) or there is
6995 // no common schema (and thus we do not modify anything).
6996 //
6997 // UseAddressDisc's value either represents "not decided yet" state (nullopt)
6998 // or whether we should request address diversity in addition to the basic
6999 // constant diversity. There is no value representing "decided not to sign"
7000 // for the reasons explained above.
7001 std::optional<bool> UseAddressDisc;
7002
7003 // Do not attempt upgrading if the new module flags already exist.
7004 if (const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
7005 for (const MDNode *Flag : ModFlags->operands()) {
7006 std::optional<StringRef> Name = getModuleFlagNameSafely(*Flag);
7007 if (Name && (*Name == "ptrauth-init-fini" ||
7008 *Name == "ptrauth-init-fini-address-discrimination"))
7009 return false;
7010 }
7011 }
7012
7013 auto UpgradeSinglePointer = [&UseAddressDisc](Constant *CV) -> Constant * {
7014 constexpr unsigned ExpectedConstDisc = 0xD9D4;
7015 constexpr unsigned ExpectedAddressMarker = 1;
7016
7017 auto *CPA = dyn_cast<ConstantPtrAuth>(CV);
7018 if (!CPA || !CPA->getDiscriminator()->equalsInt(ExpectedConstDisc))
7019 return nullptr; // Nothing to upgrade or unknown pattern found.
7020
7021 bool HasAddressDisc;
7022 if (!CPA->hasAddressDiscriminator())
7023 HasAddressDisc = false;
7024 else if (CPA->hasSpecialAddressDiscriminator(ExpectedAddressMarker))
7025 HasAddressDisc = true;
7026 else
7027 return nullptr; // Unknown pattern.
7028
7029 if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
7030 return nullptr; // Disagreement with the decided mode.
7031
7032 UseAddressDisc = HasAddressDisc;
7033 return CPA->getPointer();
7034 };
7035
7036 // Do not apply any changes until we know the upgrade is non-ambiguous.
7037 using PendingUpgrade = std::pair<GlobalVariable *, Constant *>;
7038 SmallVector<PendingUpgrade, 2> GlobalArraysToUpgrade;
7039
7040 for (const char *Name : {"llvm.global_ctors", "llvm.global_dtors"}) {
7041 auto *GV = dyn_cast_if_present<GlobalVariable>(M.getNamedValue(Name));
7042 if (!GV || !GV->hasInitializer())
7043 continue; // Skip, but it is okay to upgrade the other variable.
7044
7045 auto *OldStructorsArray = dyn_cast<ConstantArray>(GV->getInitializer());
7046 if (!OldStructorsArray || OldStructorsArray->getNumOperands() == 0)
7047 return false;
7048
7049 std::vector<Constant *> NewStructors;
7050 NewStructors.reserve(OldStructorsArray->getNumOperands());
7051
7052 for (Use &U : OldStructorsArray->operands()) {
7053 ConstantStruct *Structor = dyn_cast<ConstantStruct>(U.get());
7054 if (!Structor || Structor->getNumOperands() != 3)
7055 return false;
7056
7057 Constant *Prio = Structor->getOperand(0);
7058 Constant *Func = Structor->getOperand(1);
7059 Constant *Arg = Structor->getOperand(2);
7060
7061 Func = UpgradeSinglePointer(Func);
7062 if (!Func)
7063 return false;
7064
7065 NewStructors.push_back(
7066 ConstantStruct::get(Structor->getType(), {Prio, Func, Arg}));
7067 }
7068
7069 Constant *NewInit =
7070 ConstantArray::get(OldStructorsArray->getType(), NewStructors);
7071 GlobalArraysToUpgrade.emplace_back(GV, NewInit);
7072 }
7073
7074 if (GlobalArraysToUpgrade.empty())
7075 return false;
7076 assert(UseAddressDisc.has_value());
7077
7078 for (auto [GV, NewInit] : GlobalArraysToUpgrade)
7079 GV->setInitializer(NewInit);
7080
7081 M.addModuleFlag(Module::Error, "ptrauth-init-fini", 1);
7082 M.addModuleFlag(Module::Error, "ptrauth-init-fini-address-discrimination",
7083 *UseAddressDisc);
7084
7085 return true;
7086}
7087
7088bool llvm::UpgradeModuleFlags(Module &M) {
7089 bool Changed = false;
7090 Changed |= upgradePtrauthInitFiniArrays(M);
7091
7092 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7093 if (!ModFlags)
7094 return Changed;
7095
7096 bool HasObjCFlag = false, HasClassProperties = false;
7097 bool HasSwiftVersionFlag = false;
7098 uint8_t SwiftMajorVersion, SwiftMinorVersion;
7099 uint32_t SwiftABIVersion;
7100 auto Int8Ty = Type::getInt8Ty(M.getContext());
7101 auto Int32Ty = Type::getInt32Ty(M.getContext());
7102
7103 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7104 MDNode *Op = ModFlags->getOperand(I);
7105 if (Op->getNumOperands() != 3)
7106 continue;
7107 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7108 if (!ID)
7109 continue;
7110 auto SetBehavior = [&](Module::ModFlagBehavior B) {
7111 Metadata *Ops[3] = {ConstantAsMetadata::get(ConstantInt::get(
7112 Type::getInt32Ty(M.getContext()), B)),
7113 MDString::get(M.getContext(), ID->getString()),
7114 Op->getOperand(2)};
7115 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7116 Changed = true;
7117 };
7118
7119 if (ID->getString() == "Objective-C Image Info Version")
7120 HasObjCFlag = true;
7121 if (ID->getString() == "Objective-C Class Properties")
7122 HasClassProperties = true;
7123 // Upgrade PIC from Error/Max to Min.
7124 if (ID->getString() == "PIC Level") {
7125 if (auto *Behavior =
7126 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
7127 uint64_t V = Behavior->getLimitedValue();
7128 if (V == Module::Error || V == Module::Max)
7129 SetBehavior(Module::Min);
7130 }
7131 }
7132 // Upgrade "PIE Level" from Error to Max.
7133 if (ID->getString() == "PIE Level")
7134 if (auto *Behavior =
7135 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0)))
7136 if (Behavior->getLimitedValue() == Module::Error)
7137 SetBehavior(Module::Max);
7138
7139 // Upgrade branch protection and return address signing module flags. The
7140 // module flag behavior for these fields were Error and now they are Min.
7141 if (ID->getString() == "branch-target-enforcement" ||
7142 ID->getString().starts_with("sign-return-address")) {
7143 if (auto *Behavior =
7144 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
7145 if (Behavior->getLimitedValue() == Module::Error) {
7146 Type *Int32Ty = Type::getInt32Ty(M.getContext());
7147 Metadata *Ops[3] = {
7148 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Min)),
7149 Op->getOperand(1), Op->getOperand(2)};
7150 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7151 Changed = true;
7152 }
7153 }
7154 }
7155
7156 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
7157 // section name so that llvm-lto will not complain about mismatching
7158 // module flags that is functionally the same.
7159 if (ID->getString() == "Objective-C Image Info Section") {
7160 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
7161 SmallVector<StringRef, 4> ValueComp;
7162 Value->getString().split(ValueComp, " ");
7163 if (ValueComp.size() != 1) {
7164 std::string NewValue;
7165 for (auto &S : ValueComp)
7166 NewValue += S.str();
7167 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
7168 MDString::get(M.getContext(), NewValue)};
7169 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7170 Changed = true;
7171 }
7172 }
7173 }
7174
7175 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
7176 // If the higher bits are set, it adds new module flag for swift info.
7177 if (ID->getString() == "Objective-C Garbage Collection") {
7178 auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
7179 if (Md) {
7180 assert(Md->getValue() && "Expected non-empty metadata");
7181 auto Type = Md->getValue()->getType();
7182 if (Type == Int8Ty)
7183 continue;
7184 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
7185 if ((Val & 0xff) != Val) {
7186 HasSwiftVersionFlag = true;
7187 SwiftABIVersion = (Val & 0xff00) >> 8;
7188 SwiftMajorVersion = (Val & 0xff000000) >> 24;
7189 SwiftMinorVersion = (Val & 0xff0000) >> 16;
7190 }
7191 Metadata *Ops[3] = {
7192 ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
7193 Op->getOperand(1),
7194 ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
7195 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7196 Changed = true;
7197 }
7198 }
7199
7200 if (ID->getString() == "amdgpu_code_object_version") {
7201 Metadata *Ops[3] = {
7202 Op->getOperand(0),
7203 MDString::get(M.getContext(), "amdhsa_code_object_version"),
7204 Op->getOperand(2)};
7205 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7206 Changed = true;
7207 }
7208
7209 // clang/PowerPC used to use "float-abi" to describe the long double format;
7210 // it has been renamed to "long-double-type", with its values changed to the
7211 // corresponding IR floating-point type names.
7212 if (M.getTargetTriple().isPPC() && ID->getString() == "float-abi") {
7213 StringRef Format;
7214 if (auto *S = dyn_cast_or_null<MDString>(Op->getOperand(2)))
7215 Format = S->getString();
7216
7217 // The "float-abi" key is now reserved for the target-independent
7218 // soft/hard ABI flag, so leave a valid value alone. Map any other value
7219 // (including unrecognized ones, which were never valid) to the default.
7220 if (!FloatABI::parseABIType(Format)) {
7221 LongDoubleFormat NewFormat =
7222 StringSwitch<LongDoubleFormat>(Format)
7223 .Case("ieeequad", LongDoubleFormat::IEEEquad)
7224 .Case("ieeedouble", LongDoubleFormat::IEEEdouble)
7225 .Default(LongDoubleFormat::PPCDoubleDouble);
7226 Metadata *Ops[3] = {
7227 Op->getOperand(0),
7228 MDString::get(M.getContext(), "long-double-type"),
7229 MDString::get(M.getContext(), getLongDoubleFormatName(NewFormat))};
7230 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7231 Changed = true;
7232 }
7233 }
7234 }
7235
7236 // "Objective-C Class Properties" is recently added for Objective-C. We
7237 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
7238 // flag of value 0, so we can correclty downgrade this flag when trying to
7239 // link an ObjC bitcode without this module flag with an ObjC bitcode with
7240 // this module flag.
7241 if (HasObjCFlag && !HasClassProperties) {
7242 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
7243 (uint32_t)0);
7244 Changed = true;
7245 }
7246
7247 if (HasSwiftVersionFlag) {
7248 M.addModuleFlag(Module::Error, "Swift ABI Version",
7249 SwiftABIVersion);
7250 M.addModuleFlag(Module::Error, "Swift Major Version",
7251 ConstantInt::get(Int8Ty, SwiftMajorVersion));
7252 M.addModuleFlag(Module::Error, "Swift Minor Version",
7253 ConstantInt::get(Int8Ty, SwiftMinorVersion));
7254 Changed = true;
7255 }
7256
7257 return Changed;
7258}
7259
7260bool llvm::UpgradeCFIFunctionsMetadata(Module &M) {
7261 NamedMDNode *CFIConsts = M.getNamedMetadata("cfi.functions");
7262 // If this metadata has operands, we expect all of them to be either from
7263 // before or from after the format change handled here, so we can bail out
7264 // fast if the first (if any) operands is of the new format.
7265 auto MatchesVersion = [](const MDNode *Op) {
7266 return Op->getNumOperands() >= 3 &&
7267 isa<ConstantAsMetadata>(Op->getOperand(2)) &&
7268 cast<ConstantAsMetadata>(Op->getOperand(2))
7269 ->getType()
7270 ->isIntegerTy(64);
7271 };
7272
7273 if (!CFIConsts || !CFIConsts->getNumOperands() ||
7274 MatchesVersion(CFIConsts->getOperand(0)))
7275 return false;
7276
7277 bool Changed = false;
7278 for (unsigned I = 0, E = CFIConsts->getNumOperands(); I != E; ++I) {
7279 MDNode *Op = CFIConsts->getOperand(I);
7280 assert(!MatchesVersion(Op) && "Unexpected mix of CFIConstant formats");
7281 assert(Op->getNumOperands() >= 2 &&
7282 "Expected at least 2 operands - name and linkage type");
7283 MDString *NameMD = dyn_cast<MDString>(Op->getOperand(0));
7284 StringRef Name = NameMD->getString();
7285 GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage(
7286 GlobalValue::dropLLVMManglingEscape(Name));
7287
7288 SmallVector<Metadata *, 4> Elts;
7289 Elts.push_back(Op->getOperand(0));
7290 Elts.push_back(Op->getOperand(1));
7291 Elts.push_back(ConstantAsMetadata::get(
7292 ConstantInt::get(Type::getInt64Ty(M.getContext()), GUID)));
7293
7294 for (unsigned J = 2, EJ = Op->getNumOperands(); J != EJ; ++J)
7295 Elts.push_back(Op->getOperand(J));
7296
7297 CFIConsts->setOperand(I, MDNode::get(M.getContext(), Elts));
7298 Changed = true;
7299 }
7300
7301 return Changed;
7302}
7303
7304void llvm::UpgradeSectionAttributes(Module &M) {
7305 auto TrimSpaces = [](StringRef Section) -> std::string {
7306 SmallVector<StringRef, 5> Components;
7307 Section.split(Components, ',');
7308
7309 SmallString<32> Buffer;
7310 raw_svector_ostream OS(Buffer);
7311
7312 for (auto Component : Components)
7313 OS << ',' << Component.trim();
7314
7315 return std::string(OS.str().substr(1));
7316 };
7317
7318 for (auto &GV : M.globals()) {
7319 if (!GV.hasSection())
7320 continue;
7321
7322 StringRef Section = GV.getSection();
7323
7324 if (!Section.starts_with("__DATA, __objc_catlist"))
7325 continue;
7326
7327 // __DATA, __objc_catlist, regular, no_dead_strip
7328 // __DATA,__objc_catlist,regular,no_dead_strip
7329 GV.setSection(TrimSpaces(Section));
7330 }
7331}
7332
7333namespace {
7334// Prior to LLVM 10.0, the strictfp attribute could be used on individual
7335// callsites within a function that did not also have the strictfp attribute.
7336// Since 10.0, if strict FP semantics are needed within a function, the
7337// function must have the strictfp attribute and all calls within the function
7338// must also have the strictfp attribute. This latter restriction is
7339// necessary to prevent unwanted libcall simplification when a function is
7340// being cloned (such as for inlining).
7341//
7342// The "dangling" strictfp attribute usage was only used to prevent constant
7343// folding and other libcall simplification. The nobuiltin attribute on the
7344// callsite has the same effect.
7345struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
7346 StrictFPUpgradeVisitor() = default;
7347
7348 void visitCallBase(CallBase &Call) {
7349 if (!Call.isStrictFP())
7350 return;
7351 if (isa<ConstrainedFPIntrinsic>(&Call))
7352 return;
7353 // If we get here, the caller doesn't have the strictfp attribute
7354 // but this callsite does. Replace the strictfp attribute with nobuiltin.
7355 Call.removeFnAttr(Attribute::StrictFP);
7356 Call.addFnAttr(Attribute::NoBuiltin);
7357 }
7358};
7359
7360/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
7361struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
7362 : public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
7363 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
7364
7365 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
7366 if (!RMW.isFloatingPointOperation())
7367 return;
7368
7369 MDNode *Empty = MDNode::get(RMW.getContext(), {});
7370 RMW.setMetadata("amdgpu.no.fine.grained.host.memory", Empty);
7371 RMW.setMetadata("amdgpu.no.remote.memory.access", Empty);
7372 RMW.setMetadata("amdgpu.ignore.denormal.mode", Empty);
7373 }
7374};
7375} // namespace
7376
7377void llvm::UpgradeFunctionAttributes(Function &F) {
7378 // If a function definition doesn't have the strictfp attribute,
7379 // convert any callsite strictfp attributes to nobuiltin.
7380 if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
7381 StrictFPUpgradeVisitor SFPV;
7382 SFPV.visit(F);
7383 }
7384
7385 // Remove all incompatibile attributes from function.
7386 F.removeRetAttrs(AttributeFuncs::typeIncompatible(
7387 F.getReturnType(), F.getAttributes().getRetAttrs()));
7388 for (auto &Arg : F.args())
7389 Arg.removeAttrs(
7390 AttributeFuncs::typeIncompatible(Arg.getType(), Arg.getAttributes()));
7391
7392 bool AddingAttrs = false, RemovingAttrs = false;
7393 AttrBuilder AttrsToAdd(F.getContext());
7394 AttributeMask AttrsToRemove;
7395
7396 // Older versions of LLVM treated an "implicit-section-name" attribute
7397 // similarly to directly setting the section on a Function.
7398 if (Attribute A = F.getFnAttribute("implicit-section-name");
7399 A.isValid() && A.isStringAttribute()) {
7400 F.setSection(A.getValueAsString());
7401 AttrsToRemove.addAttribute("implicit-section-name");
7402 RemovingAttrs = true;
7403 }
7404
7405 if (Attribute A = F.getFnAttribute("nooutline");
7406 A.isValid() && A.isStringAttribute()) {
7407 AttrsToRemove.addAttribute("nooutline");
7408 AttrsToAdd.addAttribute(Attribute::NoOutline);
7409 AddingAttrs = RemovingAttrs = true;
7410 }
7411
7412 if (Attribute A = F.getFnAttribute("uniform-work-group-size");
7413 A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
7414 AttrsToRemove.addAttribute("uniform-work-group-size");
7415 RemovingAttrs = true;
7416 if (A.getValueAsString() == "true") {
7417 AttrsToAdd.addAttribute("uniform-work-group-size");
7418 AddingAttrs = true;
7419 }
7420 }
7421
7422 if (!F.empty()) {
7423 // For some reason this is called twice, and the first time is before any
7424 // instructions are loaded into the body.
7425
7426 if (Attribute A = F.getFnAttribute("amdgpu-unsafe-fp-atomics");
7427 A.isValid()) {
7428
7429 if (A.getValueAsBool()) {
7430 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
7431 Visitor.visit(F);
7432 }
7433
7434 // We will leave behind dead attribute uses on external declarations, but
7435 // clang never added these to declarations anyway.
7436 AttrsToRemove.addAttribute("amdgpu-unsafe-fp-atomics");
7437 RemovingAttrs = true;
7438 }
7439 }
7440
7441 DenormalMode DenormalFPMath = DenormalMode::getIEEE();
7442 DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
7443
7444 bool HandleDenormalMode = false;
7445
7446 if (Attribute Attr = F.getFnAttribute("denormal-fp-math"); Attr.isValid()) {
7447 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7448 if (ParsedMode.isValid()) {
7449 DenormalFPMath = ParsedMode;
7450 AttrsToRemove.addAttribute("denormal-fp-math");
7451 AddingAttrs = RemovingAttrs = true;
7452 HandleDenormalMode = true;
7453 }
7454 }
7455
7456 if (Attribute Attr = F.getFnAttribute("denormal-fp-math-f32");
7457 Attr.isValid()) {
7458 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7459 if (ParsedMode.isValid()) {
7460 DenormalFPMathF32 = ParsedMode;
7461 AttrsToRemove.addAttribute("denormal-fp-math-f32");
7462 AddingAttrs = RemovingAttrs = true;
7463 HandleDenormalMode = true;
7464 }
7465 }
7466
7467 if (HandleDenormalMode)
7468 AttrsToAdd.addDenormalFPEnvAttr(
7469 DenormalFPEnv(DenormalFPMath, DenormalFPMathF32));
7470
7471 if (RemovingAttrs)
7472 F.removeFnAttrs(AttrsToRemove);
7473
7474 if (AddingAttrs)
7475 F.addFnAttrs(AttrsToAdd);
7476}
7477
7478// Check if the function attribute is not present and set it.
7479static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName,
7480 StringRef Value) {
7481 if (!F.hasFnAttribute(FnAttrName))
7482 F.addFnAttr(FnAttrName, Value);
7483}
7484
7485// Check if the function attribute is not present and set it if needed.
7486// If the attribute is "false" then removes it.
7487// If the attribute is "true" resets it to a valueless attribute.
7488static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
7489 if (!F.hasFnAttribute(FnAttrName)) {
7490 if (Set)
7491 F.addFnAttr(FnAttrName);
7492 } else {
7493 auto A = F.getFnAttribute(FnAttrName);
7494 if ("false" == A.getValueAsString())
7495 F.removeFnAttr(FnAttrName);
7496 else if ("true" == A.getValueAsString()) {
7497 F.removeFnAttr(FnAttrName);
7498 F.addFnAttr(FnAttrName);
7499 }
7500 }
7501}
7502
7503void llvm::copyModuleAttrToFunctions(Module &M) {
7504 Triple T(M.getTargetTriple());
7505 if (!T.isThumb() && !T.isARM() && !T.isAArch64())
7506 return;
7507
7508 uint64_t BTEValue = 0;
7509 uint64_t BPPLRValue = 0;
7510 uint64_t GCSValue = 0;
7511 uint64_t SRAValue = 0;
7512 uint64_t SRAALLValue = 0;
7513 uint64_t SRABKeyValue = 0;
7514
7515 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7516 if (ModFlags) {
7517 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7518 MDNode *Op = ModFlags->getOperand(I);
7519 if (Op->getNumOperands() != 3)
7520 continue;
7521
7522 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7523 auto *CI = mdconst::dyn_extract<ConstantInt>(Op->getOperand(2));
7524 if (!ID || !CI)
7525 continue;
7526
7527 StringRef IDStr = ID->getString();
7528 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
7529 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
7530 : IDStr == "guarded-control-stack" ? &GCSValue
7531 : IDStr == "sign-return-address" ? &SRAValue
7532 : IDStr == "sign-return-address-all" ? &SRAALLValue
7533 : IDStr == "sign-return-address-with-bkey"
7534 ? &SRABKeyValue
7535 : nullptr;
7536 if (!ValPtr)
7537 continue;
7538
7539 *ValPtr = CI->getZExtValue();
7540 if (*ValPtr == 2)
7541 return;
7542 }
7543 }
7544
7545 bool BTE = BTEValue == 1;
7546 bool BPPLR = BPPLRValue == 1;
7547 bool GCS = GCSValue == 1;
7548 bool SRA = SRAValue == 1;
7549
7550 StringRef SignTypeValue = "non-leaf";
7551 if (SRA && SRAALLValue == 1)
7552 SignTypeValue = "all";
7553
7554 StringRef SignKeyValue = "a_key";
7555 if (SRA && SRABKeyValue == 1)
7556 SignKeyValue = "b_key";
7557
7558 for (Function &F : M.getFunctionList()) {
7559 if (F.isDeclaration())
7560 continue;
7561
7562 if (SRA) {
7563 setFunctionAttrIfNotSet(F, "sign-return-address", SignTypeValue);
7564 setFunctionAttrIfNotSet(F, "sign-return-address-key", SignKeyValue);
7565 } else {
7566 if (auto A = F.getFnAttribute("sign-return-address");
7567 A.isValid() && "none" == A.getValueAsString()) {
7568 F.removeFnAttr("sign-return-address");
7569 F.removeFnAttr("sign-return-address-key");
7570 }
7571 }
7572 ConvertFunctionAttr(F, BTE, "branch-target-enforcement");
7573 ConvertFunctionAttr(F, BPPLR, "branch-protection-pauth-lr");
7574 ConvertFunctionAttr(F, GCS, "guarded-control-stack");
7575 }
7576
7577 if (BTE)
7578 M.setModuleFlag(llvm::Module::Min, "branch-target-enforcement", 2);
7579 if (BPPLR)
7580 M.setModuleFlag(llvm::Module::Min, "branch-protection-pauth-lr", 2);
7581 if (GCS)
7582 M.setModuleFlag(llvm::Module::Min, "guarded-control-stack", 2);
7583 if (SRA) {
7584 M.setModuleFlag(llvm::Module::Min, "sign-return-address", 2);
7585 if (SRAALLValue == 1)
7586 M.setModuleFlag(llvm::Module::Min, "sign-return-address-all", 2);
7587 if (SRABKeyValue == 1)
7588 M.setModuleFlag(llvm::Module::Min, "sign-return-address-with-bkey", 2);
7589 }
7590}
7591
7592/// Return the replacement tags if \p T still uses a removed two-operand form.
7593static const BooleanLoopTags *getOldBooleanLoopTags(const MDTuple *T) {
7594 if (T->getNumOperands() != 2 || !mdconst::hasa<ConstantInt>(T->getOperand(1)))
7595 return nullptr;
7596 auto *Tag = dyn_cast_or_null<MDString>(T->getOperand(0));
7597 return Tag ? findBooleanLoopTags(Tag->getString()) : nullptr;
7598}
7599
7600/// Build the single-operand node that replaces a boolean operand: nonzero
7601/// selects the enable tag, zero the disable tag.
7602static Metadata *makeBooleanLoopNode(LLVMContext &C,
7603 const BooleanLoopTags &Tags,
7604 const MDOperand &Op) {
7605 bool Enable = !mdconst::extract<ConstantInt>(Op)->isZero();
7606 return MDTuple::get(C,
7607 {MDString::get(C, Enable ? Tags.Enable : Tags.Disable)});
7608}
7609
7610static bool isOldLoopArgument(Metadata *MD) {
7611 auto *T = dyn_cast_or_null<MDTuple>(MD);
7612 if (!T)
7613 return false;
7614 if (T->getNumOperands() < 1)
7615 return false;
7616 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
7617 if (!S)
7618 return false;
7619 if (S->getString().starts_with("llvm.vectorizer."))
7620 return true;
7621 return getOldBooleanLoopTags(T) != nullptr;
7622}
7623
7624static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
7625 StringRef OldPrefix = "llvm.vectorizer.";
7626 assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
7627
7628 if (OldTag == "llvm.vectorizer.unroll")
7629 return MDString::get(C, "llvm.loop.interleave.count");
7630
7631 return MDString::get(
7632 C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
7633 .str());
7634}
7635
7636static Metadata *upgradeLoopArgument(Metadata *MD) {
7637 auto *T = dyn_cast_or_null<MDTuple>(MD);
7638 if (!T)
7639 return MD;
7640 if (T->getNumOperands() < 1)
7641 return MD;
7642 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
7643 if (!OldTag)
7644 return MD;
7645
7646 LLVMContext &C = T->getContext();
7647
7648 /// Rewrite a removed two-operand boolean form to the single-operand pair.
7649 if (const BooleanLoopTags *Tags = getOldBooleanLoopTags(T))
7650 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7651
7652 if (!OldTag->getString().starts_with("llvm.vectorizer."))
7653 return MD;
7654
7655 // This has an old tag. Upgrade it.
7656 MDString *NewTag = upgradeLoopTag(C, OldTag->getString());
7657
7658 // The legacy !{!"llvm.vectorizer.enable", i1 X} maps onto the single-operand
7659 // vectorize.enable/disable pair, not a two-operand enable node.
7660 if (T->getNumOperands() == 2 && mdconst::hasa<ConstantInt>(T->getOperand(1)))
7661 if (const BooleanLoopTags *Tags = findBooleanLoopTags(NewTag->getString()))
7662 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7663
7664 SmallVector<Metadata *, 8> Ops;
7665 Ops.reserve(T->getNumOperands());
7666 Ops.push_back(NewTag);
7667 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
7668 Ops.push_back(T->getOperand(I));
7669
7670 return MDTuple::get(C, Ops);
7671}
7672
7673MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
7674 auto *T = dyn_cast<MDTuple>(&N);
7675 if (!T)
7676 return &N;
7677
7678 if (none_of(T->operands(), isOldLoopArgument))
7679 return &N;
7680
7681 // Fix the removed two-operand boolean nodes in place: the Verifier rejects
7682 // any MDNode carrying those tags with more than one operand, so a leftover
7683 // reference (from the distinct loop-ID) would still trigger a diagnostic.
7684 // In-place mutation is safe on distinct MDNodes.
7685 if (T->isDistinct()) {
7686 for (unsigned I = 0, E = T->getNumOperands(); I < E; ++I) {
7687 auto *OpT = dyn_cast_or_null<MDTuple>(T->getOperand(I));
7688 if (OpT && getOldBooleanLoopTags(OpT))
7689 T->replaceOperandWith(I, upgradeLoopArgument(OpT));
7690 }
7691 if (none_of(T->operands(), isOldLoopArgument))
7692 return &N;
7693 }
7694
7695 // Remaining old arguments (e.g. llvm.vectorizer.*) are handled via a wrapper
7696 // attachment; the original distinct loop-ID is kept as the first operand.
7697 SmallVector<Metadata *, 8> Ops;
7698 Ops.reserve(T->getNumOperands());
7699 for (Metadata *MD : T->operands())
7700 Ops.push_back(upgradeLoopArgument(MD));
7701
7702 return MDTuple::get(T->getContext(), Ops);
7703}
7704
7705std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
7706 Triple T(TT);
7707 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
7708 // the address space of globals to 1. This does not apply to SPIRV Logical.
7709 if ((T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical())) &&
7710 !DL.contains("-G") && !DL.starts_with("G")) {
7711 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
7712 }
7713
7714 if (T.isLoongArch64() || T.isRISCV64()) {
7715 // Make i32 a native type for 64-bit LoongArch and RISC-V.
7716 auto I = DL.find("-n64-");
7717 if (I != StringRef::npos)
7718 return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();
7719 return DL.str();
7720 }
7721
7722 // AMDGPU data layout upgrades.
7723 std::string Res = DL.str();
7724 if (T.isAMDGPU()) {
7725 // Define address spaces for constants.
7726 if (!DL.contains("-G") && !DL.starts_with("G"))
7727 Res.append(Res.empty() ? "G1" : "-G1");
7728
7729 // AMDGCN data layout upgrades.
7730 if (T.isAMDGCN()) {
7731
7732 // Add missing non-integral declarations.
7733 // This goes before adding new address spaces to prevent incoherent string
7734 // values.
7735 if (!DL.contains("-ni") && !DL.starts_with("ni"))
7736 Res.append("-ni:7:8:9");
7737 // Update ni:7 to ni:7:8:9.
7738 if (DL.ends_with("ni:7"))
7739 Res.append(":8:9");
7740 if (DL.ends_with("ni:7:8"))
7741 Res.append(":9");
7742
7743 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
7744 // resources) An empty data layout has already been upgraded to G1 by now.
7745 if (!DL.contains("-p7") && !DL.starts_with("p7"))
7746 Res.append("-p7:160:256:256:32");
7747 if (!DL.contains("-p8") && !DL.starts_with("p8"))
7748 Res.append("-p8:128:128:128:48");
7749 constexpr StringRef OldP8("-p8:128:128-");
7750 if (DL.contains(OldP8))
7751 Res.replace(Res.find(OldP8), OldP8.size(), "-p8:128:128:128:48-");
7752 if (!DL.contains("-p9") && !DL.starts_with("p9"))
7753 Res.append("-p9:192:256:256:32");
7754 }
7755
7756 // Upgrade the ELF mangling mode.
7757 if (!DL.contains("m:e"))
7758 Res = Res.empty() ? "m:e" : "m:e-" + Res;
7759
7760 return Res;
7761 }
7762
7763 if (T.isSystemZ() && !DL.empty()) {
7764 // Make sure the stack alignment is present.
7765 if (!DL.contains("-S64"))
7766 return "E-S64" + DL.drop_front(1).str();
7767 return DL.str();
7768 }
7769
7770 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
7771 // If the datalayout matches the expected format, add pointer size address
7772 // spaces to the datalayout.
7773 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
7774 if (!DL.contains(AddrSpaces)) {
7775 SmallVector<StringRef, 4> Groups;
7776 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
7777 if (R.match(Res, &Groups))
7778 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
7779 }
7780 };
7781
7782 // AArch64 data layout upgrades.
7783 if (T.isAArch64()) {
7784 // Add "-Fn32"
7785 if (!DL.empty() && !DL.contains("-Fn32"))
7786 Res.append("-Fn32");
7787 AddPtr32Ptr64AddrSpaces();
7788 return Res;
7789 }
7790
7791 if (T.isSPARC() || (T.isMIPS64() && !DL.contains("m:m")) || T.isPPC64() ||
7792 T.isWasm()) {
7793 // Mips64 with o32 ABI did not add "-i128:128".
7794 // Add "-i128:128"
7795 std::string I64 = "-i64:64";
7796 std::string I128 = "-i128:128";
7797 if (!StringRef(Res).contains(I128)) {
7798 size_t Pos = Res.find(I64);
7799 if (Pos != size_t(-1))
7800 Res.insert(Pos + I64.size(), I128);
7801 }
7802 }
7803
7804 if (T.isPPC() && T.isOSAIX() && !DL.contains("f64:32:64") && !DL.empty()) {
7805 size_t Pos = Res.find("-S128");
7806 if (Pos == StringRef::npos)
7807 Pos = Res.size();
7808 Res.insert(Pos, "-f64:32:64");
7809 }
7810
7811 if (!T.isX86())
7812 return Res;
7813
7814 AddPtr32Ptr64AddrSpaces();
7815
7816 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
7817 // for i128 operations prior to this being reflected in the data layout, and
7818 // clang mostly produced LLVM IR that already aligned i128 to 16 byte
7819 // boundaries, so although this is a breaking change, the upgrade is expected
7820 // to fix more IR than it breaks.
7821 // Intel MCU is an exception and uses 4-byte-alignment.
7822 if (!T.isOSIAMCU()) {
7823 std::string I128 = "-i128:128";
7824 if (StringRef Ref = Res; !Ref.contains(I128)) {
7825 SmallVector<StringRef, 4> Groups;
7826 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
7827 if (R.match(Res, &Groups))
7828 Res = (Groups[1] + I128 + Groups[3]).str();
7829 }
7830 }
7831
7832 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
7833 // Raising the alignment is safe because Clang did not produce f80 values in
7834 // the MSVC environment before this upgrade was added.
7835 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
7836 StringRef Ref = Res;
7837 auto I = Ref.find("-f80:32-");
7838 if (I != StringRef::npos)
7839 Res = (Ref.take_front(I) + "-f80:128-" + Ref.drop_front(I + 8)).str();
7840 }
7841
7842 return Res;
7843}
7844
7845void llvm::UpgradeAttributes(AttrBuilder &B) {
7846 StringRef FramePointer;
7847 Attribute A = B.getAttribute("no-frame-pointer-elim");
7848 if (A.isValid()) {
7849 // The value can be "true" or "false".
7850 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
7851 B.removeAttribute("no-frame-pointer-elim");
7852 }
7853 if (B.contains("no-frame-pointer-elim-non-leaf")) {
7854 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
7855 if (FramePointer != "all")
7856 FramePointer = "non-leaf";
7857 B.removeAttribute("no-frame-pointer-elim-non-leaf");
7858 }
7859 if (!FramePointer.empty())
7860 B.addAttribute("frame-pointer", FramePointer);
7861
7862 A = B.getAttribute("null-pointer-is-valid");
7863 if (A.isValid()) {
7864 // The value can be "true" or "false".
7865 bool NullPointerIsValid = A.getValueAsString() == "true";
7866 B.removeAttribute("null-pointer-is-valid");
7867 if (NullPointerIsValid)
7868 B.addAttribute(Attribute::NullPointerIsValid);
7869 }
7870
7871 A = B.getAttribute("uniform-work-group-size");
7872 if (A.isValid()) {
7873 StringRef Val = A.getValueAsString();
7874 if (!Val.empty()) {
7875 bool IsTrue = Val == "true";
7876 B.removeAttribute("uniform-work-group-size");
7877 if (IsTrue)
7878 B.addAttribute("uniform-work-group-size");
7879 }
7880 }
7881}
7882
7883void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
7884 // clang.arc.attachedcall bundles are now required to have an operand.
7885 // If they don't, it's okay to drop them entirely: when there is an operand,
7886 // the "attachedcall" is meaningful and required, but without an operand,
7887 // it's just a marker NOP. Dropping it merely prevents an optimization.
7888 erase_if(Bundles, [&](OperandBundleDef &OBD) {
7889 return OBD.getTag() == "clang.arc.attachedcall" &&
7890 OBD.inputs().empty();
7891 });
7892}
assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPUAddrSpace.h
AMDGPU address space definition.
Imm
unsigned Imm
Definition AMDGPUDisassembler.cpp:510
uint64_t
unsigned uint64_t
Definition AMDGPUDisassembler.cpp:511
Select
AMDGPU Register Bank Select
Definition AMDGPURegBankSelect.cpp:70
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73
Attributes.h
This file contains the simple types necessary to represent the attributes associated with functions a...
upgradeIntrinsicDeclWithDefaultArgs
static bool upgradeIntrinsicDeclWithDefaultArgs(Function *F, Function *&NewFn)
Definition AutoUpgrade.cpp:1611
upgradeX86VPERMT2Intrinsics
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
Definition AutoUpgrade.cpp:2579
isLegacyNVPTXBF16IntSignature
static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:1440
G2S_ID
#define G2S_ID(ID_SUFFIX, NAME)
upgradeLoopArgument
static Metadata * upgradeLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7636
isXYZ
static bool isXYZ(StringRef S)
Definition AutoUpgrade.cpp:6728
upgradeIntrinsicFunction1
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
Definition AutoUpgrade.cpp:1641
upgradeX86PSLLDQIntrinsics
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2419
shouldUpgradeNVPTXSharedClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1303
upgradeVPIntrinsicCall
static Value * upgradeVPIntrinsicCall(StringRef Name, CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5566
getNVPTXTMAReductionOp
static std::optional< unsigned > getNVPTXTMAReductionOp(StringRef Name)
Definition AutoUpgrade.cpp:1270
shouldUpgradeNVPTXTMAReductionIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name)
Definition AutoUpgrade.cpp:1283
upgradeRetainReleaseMarker
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
Definition AutoUpgrade.cpp:6834
upgradeX86vpcom
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
Definition AutoUpgrade.cpp:2680
upgradeMaskToInt
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2906
convertIntrinsicValidType
static bool convertIntrinsicValidType(StringRef Name, const FunctionType *FuncTy)
Definition AutoUpgrade.cpp:1591
upgradeX86Rotate
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
Definition AutoUpgrade.cpp:2654
upgradeX86MultiplyAddBytes
static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:126
getFunctionalIntrinsicIDForVP
static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name)
Definition AutoUpgrade.cpp:1544
setFunctionAttrIfNotSet
static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName, StringRef Value)
Definition AutoUpgrade.cpp:7479
shouldUpgradeNVPTXBF16Intrinsic
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1374
upgradeSingleNVVMAnnotation
static bool upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K, const Metadata *V)
Definition AutoUpgrade.cpp:6732
unwrapMAVOp
static MDNode * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
Definition AutoUpgrade.cpp:5426
upgradeLoopTag
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
Definition AutoUpgrade.cpp:7624
getVPIntPredicateFromMD
static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5526
upgradeNVVMFnVectorAttr
static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC, GlobalValue *GV, const Metadata *V)
Definition AutoUpgrade.cpp:6695
upgradeX86MaskedFPCompare
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:113
upgradeX86ALIGNIntrinsics
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
Definition AutoUpgrade.cpp:2534
upgradeAbs
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2791
shouldUpgradeVPIntrinsic
static bool shouldUpgradeVPIntrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1587
emitX86Select
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2505
upgradeAArch64IntrinsicCall
static Value * upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5063
G2S_CTA_ID
#define G2S_CTA_ID(ID_SUFFIX, NAME)
upgradeMaskedMove
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2892
getOldBooleanLoopTags
static const BooleanLoopTags * getOldBooleanLoopTags(const MDTuple *T)
Return the replacement tags if T still uses a removed two-operand form.
Definition AutoUpgrade.cpp:7593
upgradeX86IntrinsicFunction
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:551
applyX86MaskOn1BitsVec
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
Definition AutoUpgrade.cpp:2831
shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1354
getModuleFlagNameSafely
static std::optional< StringRef > getModuleFlagNameSafely(const MDNode &Flag)
Definition AutoUpgrade.cpp:6641
consumeNVVMPtrAddrSpace
static bool consumeNVVMPtrAddrSpace(StringRef &Name)
Definition AutoUpgrade.cpp:1501
makeBooleanLoopNode
static Metadata * makeBooleanLoopNode(LLVMContext &C, const BooleanLoopTags &Tags, const MDOperand &Op)
Build the single-operand node that replaces a boolean operand: nonzero selects the enable tag,...
Definition AutoUpgrade.cpp:7602
G2S_CLUSTER_CASE
#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME)
shouldUpgradeX86Intrinsic
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:180
upgradeX86PSRLDQIntrinsics
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2453
getFunctionalOpcodeForVP
static unsigned getFunctionalOpcodeForVP(StringRef Name)
Definition AutoUpgrade.cpp:1507
shouldUpgradeNVPTXTMAG2SIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
Definition AutoUpgrade.cpp:1199
shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1329
getNVVMFAddUpgrade
static std::optional< std::pair< Intrinsic::ID, RoundingMode > > getNVVMFAddUpgrade(StringRef Name)
Definition AutoUpgrade.cpp:1474
isOldLoopArgument
static bool isOldLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7610
upgradeARMIntrinsicCall
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5133
upgradeX86IntrinsicsWith8BitMask
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:97
upgradeVectorSplice
static Value * upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5496
upgradeAMDGCNIntrinsicCall
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5223
upgradeMaskedLoad
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2770
unwrapMAVMetadataOp
static Metadata * unwrapMAVMetadataOp(CallBase *CI, unsigned Op)
Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
Definition AutoUpgrade.cpp:5438
upgradeX86BF16Intrinsic
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:160
upgradeArmOrAarch64IntrinsicFunction
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:789
upgradeIntrinsicCallWithDefaultArgs
static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5597
getX86MaskVec
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
Definition AutoUpgrade.cpp:2485
emitX86ScalarSelect
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2517
upgradeX86ConcatShift
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
Definition AutoUpgrade.cpp:2719
shouldUpgradeNVPTXTcgen05MMAIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05MMAIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1461
rename
static void rename(GlobalValue *GV)
Definition AutoUpgrade.cpp:68
upgradePTESTIntrinsic
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:81
upgradeX86BF16DPIntrinsic
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:170
NVVM_TMA_G2S_MODES
#define NVVM_TMA_G2S_MODES(M)
Definition AutoUpgrade.cpp:1174
DisableAutoUpgradeDebugInfo
static cl::opt< bool > DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info", cl::desc("Disable autoupgrade of debug info"))
upgradeMaskedCompare
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
Definition AutoUpgrade.cpp:2853
upgradeX86BinaryIntrinsics
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2639
upgradeNVVMIntrinsicCall
static Value * upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3163
upgradeX86MaskedShift
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2885
upgradeAVX512MaskToSelect
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
Definition AutoUpgrade.cpp:2915
upgradeDbgIntrinsicToDbgRecord
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
Definition AutoUpgrade.cpp:5448
ConvertFunctionAttr
static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName)
Definition AutoUpgrade.cpp:7488
upgradePMULDQ
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
Definition AutoUpgrade.cpp:2801
reportFatalUsageErrorWithCI
static void reportFatalUsageErrorWithCI(StringRef reason, CallBase *CI)
Definition AutoUpgrade.cpp:72
upgradeMaskedStore
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2752
shouldUpgradeNVPTXTMAG2SCTAIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1242
upgradeConvertIntrinsicCall
static Value * upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5509
G2S_CTA_CASE
#define G2S_CTA_CASE(ID_SUFFIX, NAME)
upgradeX86MultiplyAddWords
static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:144
upgradePtrauthInitFiniArrays
static bool upgradePtrauthInitFiniArrays(Module &M)
Definition AutoUpgrade.cpp:6989
upgradeX86IntrinsicCall
static Value * upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3371
getVPFPPredicateFromMD
static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5544
C
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
A
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Constants.h
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Enable
@ Enable
Definition DwarfDebug.cpp:87
Dwarf.h
This file contains constants used for implementing Dwarf debug support.
Module.h
Module.h This file contains the declarations for the Module class.
Ops
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Definition ItaniumDemangle.h:3392
isZero
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:539
F
#define F(x, y, z)
Definition MD5.cpp:54
I
#define I(x, y, z)
Definition MD5.cpp:57
R2
#define R2(n)
Metadata.h
This file contains the declarations for metadata subclasses.
T
#define T
Definition Mips16ISelLowering.cpp:282
T1
#define T1
Definition Mips16ISelLowering.cpp:281
NVPTXAddrSpace.h
NVPTX address space definition.
High
uint64_t High
Definition NVVMIntrRange.cpp:46
NVVMIntrinsicUtils.h
This file contains the definitions of the enumerations and flags associated with NVVM Intrinsics,...
contains
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:484
StringExtras.h
This file contains some functions that are useful when dealing with strings.
StringSwitch.h
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
getType
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
Empty
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
Definition TypeHashing.cpp:17
Groups
static const X86InstrFMA3Group Groups[]
Definition X86InstrFMA3Info.cpp:81
RHS
Value * RHS
Definition X86PartialReduction.cpp:83
LHS
Value * LHS
Definition X86PartialReduction.cpp:82
llvm::APInt
Class for arbitrary precision integers.
Definition APInt.h:78
llvm::ArrayRef
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
llvm::ArrayType
Class to represent array types.
Definition DerivedTypes.h:458
llvm::ArrayType::get
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
llvm::ArrayType::getElementType
Type * getElementType() const
Definition DerivedTypes.h:471
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition Instructions.h:776
llvm::AtomicRMWInst::setVolatile
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
Definition Instructions.h:936
llvm::AtomicRMWInst::BinOp
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition Instructions.h:788
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition Instructions.h:792
llvm::AtomicRMWInst::FAdd
@ FAdd
*p = old + v
Definition Instructions.h:813
llvm::AtomicRMWInst::USubCond
@ USubCond
Subtract only if no unsigned overflow.
Definition Instructions.h:852
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition Instructions.h:806
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition Instructions.h:796
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition Instructions.h:802
llvm::AtomicRMWInst::USubSat
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
Definition Instructions.h:856
llvm::AtomicRMWInst::UIncWrap
@ UIncWrap
Increment one up to a maximum value.
Definition Instructions.h:844
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition Instructions.h:804
llvm::AtomicRMWInst::FMin
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition Instructions.h:824
llvm::AtomicRMWInst::FMax
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition Instructions.h:820
llvm::AtomicRMWInst::UDecWrap
@ UDecWrap
Decrement one until a minimum value or zero.
Definition Instructions.h:848
llvm::AtomicRMWInst::isFloatingPointOperation
bool isFloatingPointOperation() const
Definition Instructions.h:983
llvm::AttributeMask
This class stores enough information to efficiently remove some attributes from an existing AttrBuild...
Definition AttributeMask.h:29
llvm::AttributeMask::addAttribute
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
Definition AttributeMask.h:44
llvm::Attribute
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:106
llvm::Attribute::getWithStackAlignment
static LLVM_ABI Attribute getWithStackAlignment(LLVMContext &Context, Align Alignment)
Definition Attributes.cpp:226
llvm::Attribute::get
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition Attributes.cpp:95
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition InstrTypes.h:1181
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition InstrTypes.h:1479
llvm::CallBase::getOperandBundlesAsDefs
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition Instructions.cpp:511
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition InstrTypes.h:1417
llvm::CallBase::getCallingConv
CallingConv::ID getCallingConv() const
Definition InstrTypes.h:1475
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition InstrTypes.h:1409
llvm::CallBase::setAttributes
void setAttributes(AttributeList A)
Set the attributes for this call.
Definition InstrTypes.h:1496
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition InstrTypes.h:1361
llvm::CallBase::getFunctionType
FunctionType * getFunctionType() const
Definition InstrTypes.h:1274
llvm::CallBase::getIntrinsicID
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Definition Instructions.cpp:359
llvm::CallBase::args
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition InstrTypes.h:1352
llvm::CallBase::setCalledOperand
void setCalledOperand(Value *V)
Definition InstrTypes.h:1453
llvm::CallBase::arg_size
unsigned arg_size() const
Definition InstrTypes.h:1359
llvm::CallBase::getAttributes
AttributeList getAttributes() const
Return the attributes for this call.
Definition InstrTypes.h:1493
llvm::CallBase::setCalledFunction
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition InstrTypes.h:1456
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition Instructions.h:1597
llvm::CallInst::setTailCallKind
void setTailCallKind(TailCallKind TCK)
Definition Instructions.h:1717
llvm::CastInst::Create
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Definition Instructions.cpp:3109
llvm::CastInst::castIsValid
static LLVM_ABI bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Definition Instructions.cpp:3392
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
llvm::CmpInst::FCMP_OEQ
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
llvm::CmpInst::FCMP_OLT
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
llvm::CmpInst::FCMP_ULE
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
llvm::CmpInst::FCMP_OGT
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
llvm::CmpInst::FCMP_OGE
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
llvm::CmpInst::ICMP_UGE
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
llvm::CmpInst::FCMP_ULT
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
llvm::CmpInst::FCMP_ONE
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
llvm::CmpInst::FCMP_UEQ
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
llvm::CmpInst::FCMP_UGT
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:752
llvm::CmpInst::FCMP_OLE
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
llvm::CmpInst::FCMP_ORD
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition InstrTypes.h:762
llvm::CmpInst::ICMP_SGE
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
llvm::CmpInst::FCMP_UNE
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:756
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
llvm::CmpInst::FCMP_UGE
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
llvm::CmpInst::FCMP_UNO
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
llvm::ConstantAggregateZero::get
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
Definition Constants.cpp:1810
llvm::ConstantArray::get
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1432
llvm::ConstantAsMetadata::get
static ConstantAsMetadata * get(Constant *C)
Definition Metadata.h:537
llvm::ConstantExpr::getIntToPtr
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2470
llvm::ConstantExpr::getPointerCast
static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition Constants.cpp:2402
llvm::ConstantExpr::getPtrToInt
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2456
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition Constants.h:87
llvm::ConstantInt::isZero
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
llvm::ConstantPointerNull::get
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition Constants.cpp:1976
llvm::ConstantStruct::get
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1499
llvm::ConstantStruct::getType
StructType * getType() const
Specialization - reduce amount of casting.
Definition Constants.h:661
llvm::ConstantTokenNone::get
static LLVM_ABI ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition Constants.cpp:1659
llvm::Constant
This is an important base class in LLVM.
Definition Constant.h:43
llvm::Constant::getAllOnesValue
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
Definition Constants.cpp:427
llvm::Constant::getNullValue
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition Constants.cpp:367
llvm::DIExpression
DWARF expression.
Definition DebugInfoMetadata.h:3474
llvm::DIExpression::append
static LLVM_ABI DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
Definition DebugInfoMetadata.cpp:2335
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
llvm::DbgLabelRecord::createUnresolvedDbgLabelRecord
static LLVM_ABI DbgLabelRecord * createUnresolvedDbgLabelRecord(MDNode *Label)
For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved MDNodes.
Definition DebugProgramInstruction.cpp:159
llvm::DbgRecord
Base class for non-instruction debug metadata records that have positions within IR.
Definition DebugProgramInstruction.h:139
llvm::DbgRecord::setDebugLoc
void setDebugLoc(DebugLoc Loc)
Definition DebugProgramInstruction.h:217
llvm::DbgVariableRecord::createUnresolvedDbgVariableRecord
static LLVM_ABI DbgVariableRecord * createUnresolvedDbgVariableRecord(LocationType Type, Metadata *Val, MDNode *Variable, MDNode *Expression, MDNode *AssignID, Metadata *Address, MDNode *AddressExpression)
Used to create DbgVariableRecords during parsing, where some metadata references may still be unresol...
Definition DebugProgramInstruction.cpp:172
llvm::DiagnosticInfoDebugMetadataVersion
Diagnostic information for debug metadata version reporting.
Definition DiagnosticInfo.h:214
llvm::DiagnosticInfoIgnoringInvalidDebugMetadata
Diagnostic information for stripping invalid debug metadata.
Definition DiagnosticInfo.h:242
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
llvm::FastMathFlags::setApproxFunc
void setApproxFunc(bool B=true)
Definition FMF.h:93
llvm::FixedVectorType::get
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:867
llvm::FunctionType
Class to represent function types.
Definition DerivedTypes.h:165
llvm::FunctionType::getNumParams
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition DerivedTypes.h:204
llvm::FunctionType::getParamType
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition DerivedTypes.h:197
llvm::FunctionType::getReturnType
Type * getReturnType() const
Definition DerivedTypes.h:186
llvm::FunctionType::get
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition Function.h:169
llvm::Function::getFunctionType
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:212
llvm::Function::getIntrinsicID
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:247
llvm::Function::getFunction
const Function & getFunction() const
Definition Function.h:167
llvm::Function::eraseFromParent
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition Function.cpp:451
llvm::Function::arg_size
size_t arg_size() const
Definition Function.h:886
llvm::Function::getReturnType
Type * getReturnType() const
Returns the type of the ret val.
Definition Function.h:217
llvm::Function::getArg
Argument * getArg(unsigned i) const
Definition Function.h:871
llvm::GlobalValue::getGUIDAssumingExternalLinkage
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
Definition Globals.cpp:80
llvm::GlobalValue::getLinkage
LinkageTypes getLinkage() const
Definition GlobalValue.h:556
llvm::GlobalValue::GUID
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
Definition GlobalValue.h:585
llvm::GlobalValue::dropLLVMManglingEscape
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition GlobalValue.h:577
llvm::GlobalValue::getValueType
Type * getValueType() const
Definition GlobalValue.h:302
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition GlobalVariable.h:155
llvm::GlobalVariable::hasInitializer
bool hasInitializer() const
Definitions have initializers, declarations don't.
Definition GlobalVariable.h:111
llvm::IRBuilderBase::getPtrTy
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Definition IRBuilder.h:577
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2910
llvm::InstVisitor
Base class for instruction visitors.
Definition InstVisitor.h:78
llvm::Instruction::isCast
bool isCast() const
Definition Instruction.h:321
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition Instruction.h:519
llvm::Instruction::getModule
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition Instruction.cpp:83
llvm::Instruction::isBinaryOp
bool isBinaryOp() const
Definition Instruction.h:317
llvm::Instruction::eraseFromParent
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition Instruction.cpp:109
llvm::Instruction::setMetadata
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition Metadata.cpp:1809
llvm::Instruction::getFastMathFlags
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition Instruction.cpp:704
llvm::Instruction::isUnaryOp
bool isUnaryOp() const
Definition Instruction.h:316
llvm::Instruction::copyMetadata
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Definition Instruction.cpp:1526
llvm::Instruction::getDataLayout
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Definition Instruction.cpp:91
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:68
llvm::LLVMContext::getOrInsertSyncScopeID
LLVM_ABI SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
Definition LLVMContext.cpp:309
llvm::LoadInst
An instruction for reading from memory.
Definition Instructions.h:197
llvm::MDBuilder::createRange
LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
Definition MDBuilder.cpp:96
llvm::MDNode
Metadata node.
Definition Metadata.h:1069
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1426
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1567
llvm::MDNode::getNumOperands
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1432
llvm::MDNode::getContext
LLVMContext & getContext() const
Definition Metadata.h:1233
llvm::MDOperand
Tracking metadata reference owned by Metadata.
Definition Metadata.h:891
llvm::MDString
A single uniqued string.
Definition Metadata.h:722
llvm::MDString::getString
LLVM_ABI StringRef getString() const
Definition Metadata.cpp:633
llvm::MDString::get
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
Definition Metadata.cpp:615
llvm::MDTuple
Tuple of metadata.
Definition Metadata.h:1484
llvm::MDTuple::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1513
llvm::MetadataAsValue
Metadata wrapper in the Value hierarchy.
Definition Metadata.h:184
llvm::MetadataAsValue::get
static LLVM_ABI MetadataAsValue * get(LLVMContext &Context, Metadata *MD)
Definition Metadata.cpp:111
llvm::Metadata
Root of the metadata hierarchy.
Definition Metadata.h:64
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
llvm::Module::ModFlagBehavior
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
Definition Module.h:118
llvm::Module::Override
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
Definition Module.h:139
llvm::Module::Error
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
Definition Module.h:121
llvm::Module::Min
@ Min
Takes the min of the two values, which are required to be integers.
Definition Module.h:153
llvm::Module::Max
@ Max
Takes the max of the two values, which are required to be integers.
Definition Module.h:150
llvm::NamedMDNode
A tuple of MDNodes.
Definition Metadata.h:1755
llvm::NamedMDNode::setOperand
LLVM_ABI void setOperand(unsigned I, MDNode *New)
Definition Metadata.cpp:1593
llvm::NamedMDNode::getOperand
LLVM_ABI MDNode * getOperand(unsigned i) const
Definition Metadata.cpp:1585
llvm::NamedMDNode::getNumOperands
LLVM_ABI unsigned getNumOperands() const
Definition Metadata.cpp:1581
llvm::NamedMDNode::clearOperands
LLVM_ABI void clearOperands()
Drop all references to this node's operands.
Definition Metadata.cpp:1600
llvm::NamedMDNode::operands
iterator_range< op_iterator > operands()
Definition Metadata.h:1851
llvm::NamedMDNode::addOperand
LLVM_ABI void addOperand(MDNode *M)
Definition Metadata.cpp:1591
llvm::OperandBundleDefT::inputs
ArrayRef< InputTy > inputs() const
Definition InstrTypes.h:1149
llvm::OperandBundleDefT::getTag
StringRef getTag() const
Definition InstrTypes.h:1157
llvm::PoisonValue::get
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition Constants.cpp:2034
llvm::Regex::match
LLVM_ABI bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Definition Regex.cpp:83
llvm::ScalableVectorType::get
static LLVM_ABI ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
Definition Type.cpp:889
llvm::ShuffleVectorInst::getShuffleMask
ArrayRef< int > getShuffleMask() const
Definition Instructions.h:2097
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition SmallPtrSet.h:373
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition SmallPtrSet.h:519
llvm::SmallString
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition SmallString.h:26
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition SmallVector.h:592
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition SmallVector.h:993
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition SmallVector.h:718
llvm::SmallVectorTemplateBase::push_back
void push_back(const T &Elt)
Definition SmallVector.h:423
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition SmallVector.h:1252
llvm::StoreInst
An instruction for storing to memory.
Definition Instructions.h:338
llvm::StringLiteral
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Definition StringRef.h:888
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::StringRef::split
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition StringRef.h:736
llvm::StringRef::npos
static constexpr size_t npos
Definition StringRef.h:58
llvm::StringRef::substr
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition StringRef.h:597
llvm::StringRef::starts_with
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
llvm::StringRef::empty
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
llvm::StringRef::drop_front
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition StringRef.h:635
llvm::StringRef::size
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
llvm::StringRef::trim
StringRef trim(char Char) const
Return string with consecutive Char characters starting from the left and right removed.
Definition StringRef.h:850
llvm::StringSwitch
A switch()-like statement whose cases are string literals.
Definition StringSwitch.h:47
llvm::StringSwitch::Case
StringSwitch & Case(StringLiteral S, T Value)
Definition StringSwitch.h:69
llvm::StringSwitch::StartsWith
StringSwitch & StartsWith(StringLiteral S, T Value)
Definition StringSwitch.h:81
llvm::StringSwitch::Cases
StringSwitch & Cases(std::initializer_list< StringLiteral > CaseStrings, T Value)
Definition StringSwitch.h:88
llvm::StructType
Class to represent struct types.
Definition DerivedTypes.h:278
llvm::StructType::get
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:477
llvm::StructType::getNumElements
unsigned getNumElements() const
Random access to the elements.
Definition DerivedTypes.h:428
llvm::StructType::getElementType
Type * getElementType(unsigned N) const
Definition DerivedTypes.h:429
llvm::TimeTraceScope
The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.
Definition TimeProfiler.h:182
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:48
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
llvm::Type::getInt64Ty
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:310
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:288
llvm::Type::getInt32Ty
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:309
llvm::Type::isFloatTy
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
llvm::Type::isBFloatTy
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
Definition Type.h:147
llvm::Type::getPointerAddressSpace
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition DerivedTypes.h:823
llvm::Type::getInt8Ty
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:307
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:368
llvm::Type::getPrimitiveSizeInBits
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:197
llvm::Type::getInt16Ty
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
Definition Type.cpp:308
llvm::Type::getScalarSizeInBits
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:232
llvm::Type::isPtrOrPtrVectorTy
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:285
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:257
llvm::Type::isFPOrFPVectorTy
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:227
llvm::Type::getFloatTy
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:286
llvm::Type::getBFloatTy
static LLVM_ABI Type * getBFloatTy(LLVMContext &C)
Definition Type.cpp:285
llvm::Type::getHalfTy
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:284
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition User.h:207
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition User.h:229
llvm::Value
LLVM Value Representation.
Definition Value.h:75
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:255
llvm::Value::print
LLVM_ABI void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
Definition AsmWriter.cpp:5333
llvm::Value::setName
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:394
llvm::Value::replaceAllUsesWith
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:553
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:258
llvm::Value::users
iterator_range< user_iterator > users()
Definition Value.h:426
llvm::Value::stripPointerCasts
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:713
llvm::Value::use_empty
bool use_empty() const
Definition Value.h:346
llvm::Value::hasName
bool hasName() const
Definition Value.h:261
llvm::Value::getName
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:319
llvm::Value::takeName
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:400
llvm::VectorType
Base class of all SIMD vector types.
Definition DerivedTypes.h:490
llvm::VectorType::getInteger
static VectorType * getInteger(VectorType *VTy)
This static method gets a VectorType with the same number of elements as the input type,...
Definition DerivedTypes.h:541
llvm::VectorType::get
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
llvm::details::FixedOrScalableQuantity::getFixedValue
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:200
llvm::ilist_detail::node_parent_access::getParent
const ParentTy * getParent() const
Definition ilist_node.h:34
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition ilist_node.h:123
llvm::raw_svector_ostream
A raw_ostream that writes to an SmallVector or SmallString.
Definition raw_ostream.h:694
llvm::raw_svector_ostream::str
StringRef str() const
Return a StringRef for the vector contents.
Definition raw_ostream.h:723
uint32_t
uint8_t
Call
CallInst * Call
Definition ObjCARCOpts.cpp:2458
Changed
Changed
Definition ObjCARCOpts.cpp:2468
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::AMDGPUAS::LOCAL_ADDRESS
@ LOCAL_ADDRESS
Address space for local memory.
Definition AMDGPUAddrSpace.h:36
llvm::AMDGPUAS::FLAT_ADDRESS
@ FLAT_ADDRESS
Address space for flat memory.
Definition AMDGPUAddrSpace.h:32
llvm::AMDGPUAS::PRIVATE_ADDRESS
@ PRIVATE_ADDRESS
Address space for private memory.
Definition AMDGPUAddrSpace.h:38
llvm::ARM_MB::ST
@ ST
Definition ARMBaseInfo.h:73
llvm::CallingConv::PTX_Kernel
@ PTX_Kernel
Call to a PTX kernel. Passes all arguments in parameter space.
Definition CallingConv.h:125
llvm::FloatABI::parseABIType
std::optional< ABIType > parseABIType(StringRef S)
Parse the string spelling used by the "float-abi" IR module flag into an ABIType.
Definition CodeGen.h:117
llvm::Intrinsic::remangleIntrinsicFunction
LLVM_ABI std::optional< Function * > remangleIntrinsicFunction(Function *F)
Definition Intrinsics.cpp:1423
llvm::Intrinsic::getOrInsertDeclaration
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
Definition Intrinsics.cpp:801
llvm::Intrinsic::lookupIntrinsicID
LLVM_ABI ID lookupIntrinsicID(StringRef Name)
This does the actual lookup of an intrinsic ID which matches the given function name.
Definition Intrinsics.cpp:757
llvm::Intrinsic::getAttributes
LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)
Return the attributes for an intrinsic.
llvm::Intrinsic::isOverloaded
LLVM_ABI bool isOverloaded(ID id)
Returns true if the intrinsic can be overloaded.
Definition Intrinsics.cpp:649
llvm::Intrinsic::getType
LLVM_ABI FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > OverloadTys={})
Return the function type for an intrinsic.
Definition Intrinsics.cpp:636
llvm::Intrinsic::isSignatureValid
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
Definition Intrinsics.cpp:1404
llvm::Intrinsic::hasStructReturnType
LLVM_ABI bool hasStructReturnType(ID id)
Returns true if id has a struct return type.
Definition Intrinsics.cpp:1397
llvm::Intrinsic::getAllDefaultArgValues
LLVM_ABI std::pair< unsigned, ArrayRef< uint64_t > > getAllDefaultArgValues(ID IID)
Returns the first default argument index and an ArrayRef of all default values for the trailing param...
llvm::NVVMAttr::GridConstant
constexpr StringLiteral GridConstant("nvvm.grid_constant")
llvm::NVVMAttr::MaxNTID
constexpr StringLiteral MaxNTID("nvvm.maxntid")
llvm::NVVMAttr::MaxNReg
constexpr StringLiteral MaxNReg("nvvm.maxnreg")
llvm::NVVMAttr::MinCTASm
constexpr StringLiteral MinCTASm("nvvm.minctasm")
llvm::NVVMAttr::ReqNTID
constexpr StringLiteral ReqNTID("nvvm.reqntid")
llvm::NVVMAttr::MaxClusterRank
constexpr StringLiteral MaxClusterRank("nvvm.maxclusterrank")
llvm::NVVMAttr::ClusterDim
constexpr StringLiteral ClusterDim("nvvm.cluster_dim")
llvm::mdconst::dyn_extract_or_null
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract_or_null(Y &&MD)
Extract a Value from Metadata, if any, allowing null.
Definition Metadata.h:709
llvm::mdconst::hasa
std::enable_if_t< detail::IsValidPointer< X, Y >::value, bool > hasa(Y &&MD)
Check whether Metadata has a Value.
Definition Metadata.h:651
llvm::mdconst::dyn_extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
Definition Metadata.h:696
llvm::mdconst::extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:668
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
Definition MicrosoftDemangle.h:132
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition FunctionInfo.h:25
llvm::Offset
@ Offset
Definition DWP.cpp:577
llvm::Length
@ Length
Definition DWP.cpp:577
llvm::UpgradeIntrinsicCall
LLVM_ABI void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
Definition AutoUpgrade.cpp:5652
llvm::UpgradeSectionAttributes
LLVM_ABI void UpgradeSectionAttributes(Module &M)
Definition AutoUpgrade.cpp:7304
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1669
llvm::UpgradeInlineAsmString
LLVM_ABI void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
Definition AutoUpgrade.cpp:3154
llvm::isValidAtomicOrdering
bool isValidAtomicOrdering(Int I)
Definition AtomicOrdering.h:75
llvm::dyn_cast
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
llvm::VectorInstrContext::Load
@ Load
The value being inserted comes from a load (InsertElement only).
Definition TargetTransformInfo.h:191
llvm::getLongDoubleFormatName
StringRef getLongDoubleFormatName(LongDoubleFormat Format)
Returns the IR floating-point type name for a LongDoubleFormat.
Definition CodeGen.h:76
llvm::LongDoubleFormat
LongDoubleFormat
The floating-point format used for the target's "long double" type.
Definition CodeGen.h:67
llvm::UpgradeIntrinsicFunction
LLVM_ABI bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
Definition AutoUpgrade.cpp:2367
llvm::upgradeInstructionLoopAttachment
LLVM_ABI MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
Definition AutoUpgrade.cpp:7673
llvm::dyn_cast_if_present
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
llvm::UpgradeAttributes
LLVM_ABI void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
Definition AutoUpgrade.cpp:7845
llvm::UpgradeCallsToIntrinsic
LLVM_ABI void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
Definition AutoUpgrade.cpp:6554
llvm::UpgradeNVVMAnnotations
LLVM_ABI void UpgradeNVVMAnnotations(Module &M)
Convert legacy nvvm.annotations metadata to appropriate function attributes.
Definition AutoUpgrade.cpp:6793
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:633
llvm::UpgradeModuleFlags
LLVM_ABI bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
Definition AutoUpgrade.cpp:7088
llvm::utostr
std::string utostr(uint64_t X, bool isNeg=false)
Definition StringExtras.h:322
llvm::isPowerOf2_64
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
llvm::UpgradeCFIFunctionsMetadata
LLVM_ABI bool UpgradeCFIFunctionsMetadata(Module &M)
Upgrade the cfi.functions metadata node by calculating and inserting the GUID for each function entry...
Definition AutoUpgrade.cpp:7260
llvm::copyModuleAttrToFunctions
LLVM_ABI void copyModuleAttrToFunctions(Module &M)
Copies module attributes to the functions in the module.
Definition AutoUpgrade.cpp:7503
llvm::UpgradeOperandBundles
LLVM_ABI void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
Definition AutoUpgrade.cpp:7883
llvm::UpgradeBitCastExpr
LLVM_ABI Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
Definition AutoUpgrade.cpp:6621
llvm::dyn_cast_or_null
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
llvm::dbgs
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
llvm::UpgradeDataLayoutString
LLVM_ABI std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
Definition AutoUpgrade.cpp:7705
llvm::none_of
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1753
llvm::report_fatal_error
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
llvm::isa
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
llvm::UpgradeGlobalVariable
LLVM_ABI GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
Definition AutoUpgrade.cpp:2386
llvm::errs
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition raw_ostream.cpp:890
llvm::StripDebugInfo
LLVM_ABI bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
Definition DebugInfo.cpp:711
llvm::drop_end
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Definition STLExtras.h:322
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition AtomicOrdering.h:56
llvm::ModRefInfo::Ref
@ Ref
The access may reference the value stored in memory.
Definition ModRef.h:32
llvm::join
std::string join(IteratorT Begin, IteratorT End, StringRef Separator)
Joins the strings in the range [Begin, End), adding Separator between the elements.
Definition StringExtras.h:492
llvm::findBooleanLoopTags
const BooleanLoopTags * findBooleanLoopTags(StringRef Name)
Return the replacement tags for the enable tag Name, or nullptr.
Definition AutoUpgrade.h:129
llvm::OperandBundleDef
OperandBundleDefT< Value * > OperandBundleDef
Definition AutoUpgrade.h:34
llvm::UpgradeBitCastInst
LLVM_ABI Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
Definition AutoUpgrade.cpp:6599
llvm::RecurKind::FAdd
@ FAdd
Sum of floats.
Definition IVDescriptors.h:49
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:24
llvm::RoundingMode
RoundingMode
Rounding mode.
Definition FloatingPointMode.h:38
llvm::RoundingMode::TowardZero
@ TowardZero
roundTowardZero.
Definition FloatingPointMode.h:40
llvm::RoundingMode::NearestTiesToEven
@ NearestTiesToEven
roundTiesToEven.
Definition FloatingPointMode.h:41
llvm::RoundingMode::Dynamic
@ Dynamic
Denotes mode unknown at compile time.
Definition FloatingPointMode.h:47
llvm::RoundingMode::TowardPositive
@ TowardPositive
roundTowardPositive.
Definition FloatingPointMode.h:42
llvm::RoundingMode::TowardNegative
@ TowardNegative
roundTowardNegative.
Definition FloatingPointMode.h:43
llvm::ArrayRef
ArrayRef(const T &OneElt) -> ArrayRef< T >
llvm::parseDenormalFPAttribute
DenormalMode parseDenormalFPAttribute(StringRef Str)
Returns the denormal mode to use for inputs and outputs.
Definition FloatingPointMode.h:254
llvm::cast
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1772
llvm::erase_if
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2192
llvm::is_contained
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1947
llvm::UpgradeDebugInfo
LLVM_ABI bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
Definition AutoUpgrade.cpp:6651
llvm::UpgradeFunctionAttributes
LLVM_ABI void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
Definition AutoUpgrade.cpp:7377
llvm::UpgradeTBAANode
LLVM_ABI MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
Definition AutoUpgrade.cpp:6573
llvm::UpgradeARCRuntime
LLVM_ABI void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
Definition AutoUpgrade.cpp:6858
llvm::DEBUG_METADATA_VERSION
@ DEBUG_METADATA_VERSION
Definition Metadata.h:54
llvm::ValueUniformity::Default
@ Default
The result value is uniform if and only if all operands are uniform.
Definition Uniformity.h:20
llvm::verifyModule
LLVM_ABI bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
Definition Verifier.cpp:7811
llvm::reportFatalUsageError
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:880
N
#define N
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
llvm::BooleanLoopTags
Single-operand tags replacing a removed two-operand form !
Definition AutoUpgrade.h:115
llvm::BooleanLoopTags::Disable
StringLiteral Disable
Definition AutoUpgrade.h:117
llvm::BooleanLoopTags::Enable
StringLiteral Enable
Definition AutoUpgrade.h:116
llvm::DenormalFPEnv
Represents the full denormal controls for a function, including the default mode and the f32 specific...
Definition FloatingPointMode.h:279
llvm::DenormalMode
Represent subnormal handling kind for floating point instruction inputs and outputs.
Definition FloatingPointMode.h:71
llvm::DenormalMode::getInvalid
static constexpr DenormalMode getInvalid()
Definition FloatingPointMode.h:106
llvm::DenormalMode::isValid
constexpr bool isValid() const
Definition FloatingPointMode.h:156
llvm::DenormalMode::getIEEE
static constexpr DenormalMode getIEEE()
Definition FloatingPointMode.h:115
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:106
Generated on for LLVM by doxygen 1.14.0