LLVM 24.0.0git
AutoUpgrade.cpp
Go to the documentation of this file.
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "IROptions.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/StringExtras.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/ADT/StringSwitch.h"
21#include "llvm/BinaryFormat/Dwarf.h"
22#include "llvm/IR/AttributeMask.h"
23#include "llvm/IR/Attributes.h"
24#include "llvm/IR/CallingConv.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/DebugInfo.h"
27#include "llvm/IR/DebugInfoMetadata.h"
28#include "llvm/IR/DiagnosticInfo.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/GlobalValue.h"
31#include "llvm/IR/IRBuilder.h"
32#include "llvm/IR/InstVisitor.h"
33#include "llvm/IR/Instruction.h"
34#include "llvm/IR/IntrinsicInst.h"
35#include "llvm/IR/Intrinsics.h"
36#include "llvm/IR/IntrinsicsAArch64.h"
37#include "llvm/IR/IntrinsicsAMDGPU.h"
38#include "llvm/IR/IntrinsicsARM.h"
39#include "llvm/IR/IntrinsicsNVPTX.h"
40#include "llvm/IR/IntrinsicsRISCV.h"
41#include "llvm/IR/IntrinsicsWebAssembly.h"
42#include "llvm/IR/IntrinsicsX86.h"
43#include "llvm/IR/LLVMContext.h"
44#include "llvm/IR/MDBuilder.h"
45#include "llvm/IR/Metadata.h"
46#include "llvm/IR/Module.h"
47#include "llvm/IR/NVVMIntrinsicUtils.h"
48#include "llvm/IR/Value.h"
49#include "llvm/IR/Verifier.h"
50#include "llvm/Support/AMDGPUAddrSpace.h"
51#include "llvm/Support/CodeGen.h"
52#include "llvm/Support/Debug.h"
53#include "llvm/Support/ErrorHandling.h"
54#include "llvm/Support/NVPTXAddrSpace.h"
55#include "llvm/Support/NVVMAttributes.h"
56#include "llvm/Support/Regex.h"
57#include "llvm/Support/TimeProfiler.h"
58#include "llvm/TargetParser/Triple.h"
59#include <cstdint>
60#include <cstring>
61#include <numeric>
62
63using namespace llvm;
64
65#define DEBUG_TYPE "auto-upgrade"
66
67static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
68
69// Report a fatal error along with the
70// Call Instruction which caused the error
71[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
72 CallBase *CI) {
73 CI->print(llvm::errs());
74 llvm::errs() << "\n";
75 reportFatalUsageError(reason);
76}
77
78// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
79// changed their type from v4f32 to v2i64.
80static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
81 Function *&NewFn) {
82 // Check whether this is an old version of the function, which received
83 // v4f32 arguments.
84 Type *Arg0Type = F->getFunctionType()->getParamType(0);
85 if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
86 return false;
87
88 // Yes, it's old, replace it with new version.
89 rename(F);
90 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
91 return true;
92}
93
94// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
95// arguments have changed their type from i32 to i8.
96static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
97 Function *&NewFn) {
98 // Check that the last argument is an i32.
99 Type *LastArgType = F->getFunctionType()->getParamType(
100 F->getFunctionType()->getNumParams() - 1);
101 if (!LastArgType->isIntegerTy(32))
102 return false;
103
104 // Move this function aside and map down.
105 rename(F);
106 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
107 return true;
108}
109
110// Upgrade the declaration of fp compare intrinsics that change return type
111// from scalar to vXi1 mask.
112static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
113 Function *&NewFn) {
114 // Check if the return type is a vector.
115 if (F->getReturnType()->isVectorTy())
116 return false;
117
118 rename(F);
119 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
120 return true;
121}
122
123// Upgrade the declaration of multiply and add bytes intrinsics whose input
124// arguments' types have changed from vectors of i32 to vectors of i8
125static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID,
126 Function *&NewFn) {
127 // check if input argument type is a vector of i8
128 Type *Arg1Type = F->getFunctionType()->getParamType(1);
129 Type *Arg2Type = F->getFunctionType()->getParamType(2);
130 if (Arg1Type->isVectorTy() &&
131 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(8) &&
132 Arg2Type->isVectorTy() &&
133 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(8))
134 return false;
135
136 rename(F);
137 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
138 return true;
139}
140
141// Upgrade the declaration of multipy and add words intrinsics whose input
142// arguments' types have changed to vectors of i32 to vectors of i16
143static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID,
144 Function *&NewFn) {
145 // check if input argument type is a vector of i16
146 Type *Arg1Type = F->getFunctionType()->getParamType(1);
147 Type *Arg2Type = F->getFunctionType()->getParamType(2);
148 if (Arg1Type->isVectorTy() &&
149 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(16) &&
150 Arg2Type->isVectorTy() &&
151 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(16))
152 return false;
153
154 rename(F);
155 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
156 return true;
157}
158
159static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
160 Function *&NewFn) {
161 if (F->getReturnType()->getScalarType()->isBFloatTy())
162 return false;
163
164 rename(F);
165 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
166 return true;
167}
168
169static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
170 Function *&NewFn) {
171 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
172 return false;
173
174 rename(F);
175 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
176 return true;
177}
178
179static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
180 // All of the intrinsics matches below should be marked with which llvm
181 // version started autoupgrading them. At some point in the future we would
182 // like to use this information to remove upgrade code for some older
183 // intrinsics. It is currently undecided how we will determine that future
184 // point.
185 if (Name.consume_front("avx."))
186 return (Name.starts_with("blend.p") || // Added in 3.7
187 Name == "cvt.ps2.pd.256" || // Added in 3.9
188 Name == "cvtdq2.pd.256" || // Added in 3.9
189 Name == "cvtdq2.ps.256" || // Added in 7.0
190 Name.starts_with("movnt.") || // Added in 3.2
191 Name.starts_with("sqrt.p") || // Added in 7.0
192 Name.starts_with("storeu.") || // Added in 3.9
193 Name.starts_with("vbroadcast.s") || // Added in 3.5
194 Name.starts_with("vbroadcastf128") || // Added in 4.0
195 Name.starts_with("vextractf128.") || // Added in 3.7
196 Name.starts_with("vinsertf128.") || // Added in 3.7
197 Name.starts_with("vperm2f128.") || // Added in 6.0
198 Name.starts_with("vpermil.")); // Added in 3.1
199
200 if (Name.consume_front("avx2."))
201 return (Name == "movntdqa" || // Added in 5.0
202 Name.starts_with("pabs.") || // Added in 6.0
203 Name.starts_with("padds.") || // Added in 8.0
204 Name.starts_with("paddus.") || // Added in 8.0
205 Name.starts_with("pblendd.") || // Added in 3.7
206 Name == "pblendw" || // Added in 3.7
207 Name.starts_with("pbroadcast") || // Added in 3.8
208 Name.starts_with("pcmpeq.") || // Added in 3.1
209 Name.starts_with("pcmpgt.") || // Added in 3.1
210 Name.starts_with("pmax") || // Added in 3.9
211 Name.starts_with("pmin") || // Added in 3.9
212 Name.starts_with("pmovsx") || // Added in 3.9
213 Name.starts_with("pmovzx") || // Added in 3.9
214 Name.starts_with("pmulh.w") || // Added in 24.0
215 Name.starts_with("pmulhu.w") || // Added in 24.0
216 Name == "pmul.dq" || // Added in 7.0
217 Name == "pmulu.dq" || // Added in 7.0
218 Name.starts_with("psll.dq") || // Added in 3.7
219 Name.starts_with("psrl.dq") || // Added in 3.7
220 Name.starts_with("psubs.") || // Added in 8.0
221 Name.starts_with("psubus.") || // Added in 8.0
222 Name.starts_with("vbroadcast") || // Added in 3.8
223 Name == "vbroadcasti128" || // Added in 3.7
224 Name == "vextracti128" || // Added in 3.7
225 Name == "vinserti128" || // Added in 3.7
226 Name == "vperm2i128"); // Added in 6.0
227
228 if (Name.consume_front("avx512.")) {
229 if (Name.consume_front("mask."))
230 // 'avx512.mask.*'
231 return (Name.starts_with("add.p") || // Added in 7.0. 128/256 in 4.0
232 Name.starts_with("and.") || // Added in 3.9
233 Name.starts_with("andn.") || // Added in 3.9
234 Name.starts_with("broadcast.s") || // Added in 3.9
235 Name.starts_with("broadcastf32x4.") || // Added in 6.0
236 Name.starts_with("broadcastf32x8.") || // Added in 6.0
237 Name.starts_with("broadcastf64x2.") || // Added in 6.0
238 Name.starts_with("broadcastf64x4.") || // Added in 6.0
239 Name.starts_with("broadcasti32x4.") || // Added in 6.0
240 Name.starts_with("broadcasti32x8.") || // Added in 6.0
241 Name.starts_with("broadcasti64x2.") || // Added in 6.0
242 Name.starts_with("broadcasti64x4.") || // Added in 6.0
243 Name.starts_with("cmp.b") || // Added in 5.0
244 Name.starts_with("cmp.d") || // Added in 5.0
245 Name.starts_with("cmp.q") || // Added in 5.0
246 Name.starts_with("cmp.w") || // Added in 5.0
247 Name.starts_with("compress.b") || // Added in 9.0
248 Name.starts_with("compress.d") || // Added in 9.0
249 Name.starts_with("compress.p") || // Added in 9.0
250 Name.starts_with("compress.q") || // Added in 9.0
251 Name.starts_with("compress.store.") || // Added in 7.0
252 Name.starts_with("compress.w") || // Added in 9.0
253 Name.starts_with("conflict.") || // Added in 9.0
254 Name.starts_with("cvtdq2pd.") || // Added in 4.0
255 Name.starts_with("cvtdq2ps.") || // Added in 7.0 updated 9.0
256 Name == "cvtpd2dq.256" || // Added in 7.0
257 Name == "cvtpd2ps.256" || // Added in 7.0
258 Name == "cvtps2pd.128" || // Added in 7.0
259 Name == "cvtps2pd.256" || // Added in 7.0
260 Name.starts_with("cvtqq2pd.") || // Added in 7.0 updated 9.0
261 Name == "cvtqq2ps.256" || // Added in 9.0
262 Name == "cvtqq2ps.512" || // Added in 9.0
263 Name == "cvttpd2dq.256" || // Added in 7.0
264 Name == "cvttps2dq.128" || // Added in 7.0
265 Name == "cvttps2dq.256" || // Added in 7.0
266 Name.starts_with("cvtudq2pd.") || // Added in 4.0
267 Name.starts_with("cvtudq2ps.") || // Added in 7.0 updated 9.0
268 Name.starts_with("cvtuqq2pd.") || // Added in 7.0 updated 9.0
269 Name == "cvtuqq2ps.256" || // Added in 9.0
270 Name == "cvtuqq2ps.512" || // Added in 9.0
271 Name.starts_with("dbpsadbw.") || // Added in 7.0
272 Name.starts_with("div.p") || // Added in 7.0. 128/256 in 4.0
273 Name.starts_with("expand.b") || // Added in 9.0
274 Name.starts_with("expand.d") || // Added in 9.0
275 Name.starts_with("expand.load.") || // Added in 7.0
276 Name.starts_with("expand.p") || // Added in 9.0
277 Name.starts_with("expand.q") || // Added in 9.0
278 Name.starts_with("expand.w") || // Added in 9.0
279 Name.starts_with("fpclass.p") || // Added in 7.0
280 Name.starts_with("insert") || // Added in 4.0
281 Name.starts_with("load.") || // Added in 3.9
282 Name.starts_with("loadu.") || // Added in 3.9
283 Name.starts_with("lzcnt.") || // Added in 5.0
284 Name.starts_with("max.p") || // Added in 7.0. 128/256 in 5.0
285 Name.starts_with("min.p") || // Added in 7.0. 128/256 in 5.0
286 Name.starts_with("movddup") || // Added in 3.9
287 Name.starts_with("move.s") || // Added in 4.0
288 Name.starts_with("movshdup") || // Added in 3.9
289 Name.starts_with("movsldup") || // Added in 3.9
290 Name.starts_with("mul.p") || // Added in 7.0. 128/256 in 4.0
291 Name.starts_with("or.") || // Added in 3.9
292 Name.starts_with("pabs.") || // Added in 6.0
293 Name.starts_with("packssdw.") || // Added in 5.0
294 Name.starts_with("packsswb.") || // Added in 5.0
295 Name.starts_with("packusdw.") || // Added in 5.0
296 Name.starts_with("packuswb.") || // Added in 5.0
297 Name.starts_with("padd.") || // Added in 4.0
298 Name.starts_with("padds.") || // Added in 8.0
299 Name.starts_with("paddus.") || // Added in 8.0
300 Name.starts_with("palignr.") || // Added in 3.9
301 Name.starts_with("pand.") || // Added in 3.9
302 Name.starts_with("pandn.") || // Added in 3.9
303 Name.starts_with("pavg") || // Added in 6.0
304 Name.starts_with("pbroadcast") || // Added in 6.0
305 Name.starts_with("pcmpeq.") || // Added in 3.9
306 Name.starts_with("pcmpgt.") || // Added in 3.9
307 Name.starts_with("perm.df.") || // Added in 3.9
308 Name.starts_with("perm.di.") || // Added in 3.9
309 Name.starts_with("permvar.") || // Added in 7.0
310 Name.starts_with("pmaddubs.w.") || // Added in 7.0
311 Name.starts_with("pmaddw.d.") || // Added in 7.0
312 Name.starts_with("pmax") || // Added in 4.0
313 Name.starts_with("pmin") || // Added in 4.0
314 Name == "pmov.qd.256" || // Added in 9.0
315 Name == "pmov.qd.512" || // Added in 9.0
316 Name == "pmov.wb.256" || // Added in 9.0
317 Name == "pmov.wb.512" || // Added in 9.0
318 Name.starts_with("pmovsx") || // Added in 4.0
319 Name.starts_with("pmovzx") || // Added in 4.0
320 Name.starts_with("pmul.dq.") || // Added in 4.0
321 Name.starts_with("pmul.hr.sw.") || // Added in 7.0
322 Name.starts_with("pmulh.w.") || // Added in 7.0
323 Name.starts_with("pmulhu.w.") || // Added in 7.0
324 Name.starts_with("pmull.") || // Added in 4.0
325 Name.starts_with("pmultishift.qb.") || // Added in 8.0
326 Name.starts_with("pmulu.dq.") || // Added in 4.0
327 Name.starts_with("por.") || // Added in 3.9
328 Name.starts_with("prol.") || // Added in 8.0
329 Name.starts_with("prolv.") || // Added in 8.0
330 Name.starts_with("pror.") || // Added in 8.0
331 Name.starts_with("prorv.") || // Added in 8.0
332 Name.starts_with("pshuf.b.") || // Added in 4.0
333 Name.starts_with("pshuf.d.") || // Added in 3.9
334 Name.starts_with("pshufh.w.") || // Added in 3.9
335 Name.starts_with("pshufl.w.") || // Added in 3.9
336 Name.starts_with("psll.d") || // Added in 4.0
337 Name.starts_with("psll.q") || // Added in 4.0
338 Name.starts_with("psll.w") || // Added in 4.0
339 Name.starts_with("pslli") || // Added in 4.0
340 Name.starts_with("psllv") || // Added in 4.0
341 Name.starts_with("psra.d") || // Added in 4.0
342 Name.starts_with("psra.q") || // Added in 4.0
343 Name.starts_with("psra.w") || // Added in 4.0
344 Name.starts_with("psrai") || // Added in 4.0
345 Name.starts_with("psrav") || // Added in 4.0
346 Name.starts_with("psrl.d") || // Added in 4.0
347 Name.starts_with("psrl.q") || // Added in 4.0
348 Name.starts_with("psrl.w") || // Added in 4.0
349 Name.starts_with("psrli") || // Added in 4.0
350 Name.starts_with("psrlv") || // Added in 4.0
351 Name.starts_with("psub.") || // Added in 4.0
352 Name.starts_with("psubs.") || // Added in 8.0
353 Name.starts_with("psubus.") || // Added in 8.0
354 Name.starts_with("pternlog.") || // Added in 7.0
355 Name.starts_with("punpckh") || // Added in 3.9
356 Name.starts_with("punpckl") || // Added in 3.9
357 Name.starts_with("pxor.") || // Added in 3.9
358 Name.starts_with("shuf.f") || // Added in 6.0
359 Name.starts_with("shuf.i") || // Added in 6.0
360 Name.starts_with("shuf.p") || // Added in 4.0
361 Name.starts_with("sqrt.p") || // Added in 7.0
362 Name.starts_with("store.b.") || // Added in 3.9
363 Name.starts_with("store.d.") || // Added in 3.9
364 Name.starts_with("store.p") || // Added in 3.9
365 Name.starts_with("store.q.") || // Added in 3.9
366 Name.starts_with("store.w.") || // Added in 3.9
367 Name == "store.ss" || // Added in 7.0
368 Name.starts_with("storeu.") || // Added in 3.9
369 Name.starts_with("sub.p") || // Added in 7.0. 128/256 in 4.0
370 Name.starts_with("ucmp.") || // Added in 5.0
371 Name.starts_with("unpckh.") || // Added in 3.9
372 Name.starts_with("unpckl.") || // Added in 3.9
373 Name.starts_with("valign.") || // Added in 4.0
374 Name == "vcvtph2ps.128" || // Added in 11.0
375 Name == "vcvtph2ps.256" || // Added in 11.0
376 Name.starts_with("vextract") || // Added in 4.0
377 Name.starts_with("vfmadd.") || // Added in 7.0
378 Name.starts_with("vfmaddsub.") || // Added in 7.0
379 Name.starts_with("vfnmadd.") || // Added in 7.0
380 Name.starts_with("vfnmsub.") || // Added in 7.0
381 Name.starts_with("vpdpbusd.") || // Added in 7.0
382 Name.starts_with("vpdpbusds.") || // Added in 7.0
383 Name.starts_with("vpdpwssd.") || // Added in 7.0
384 Name.starts_with("vpdpwssds.") || // Added in 7.0
385 Name.starts_with("vpermi2var.") || // Added in 7.0
386 Name.starts_with("vpermil.p") || // Added in 3.9
387 Name.starts_with("vpermilvar.") || // Added in 4.0
388 Name.starts_with("vpermt2var.") || // Added in 7.0
389 Name.starts_with("vpmadd52") || // Added in 7.0
390 Name.starts_with("vpshld.") || // Added in 7.0
391 Name.starts_with("vpshldv.") || // Added in 8.0
392 Name.starts_with("vpshrd.") || // Added in 7.0
393 Name.starts_with("vpshrdv.") || // Added in 8.0
394 Name.starts_with("vpshufbitqmb.") || // Added in 8.0
395 Name.starts_with("xor.")); // Added in 3.9
396
397 if (Name.consume_front("mask3."))
398 // 'avx512.mask3.*'
399 return (Name.starts_with("vfmadd.") || // Added in 7.0
400 Name.starts_with("vfmaddsub.") || // Added in 7.0
401 Name.starts_with("vfmsub.") || // Added in 7.0
402 Name.starts_with("vfmsubadd.") || // Added in 7.0
403 Name.starts_with("vfnmsub.")); // Added in 7.0
404
405 if (Name.consume_front("maskz."))
406 // 'avx512.maskz.*'
407 return (Name.starts_with("pternlog.") || // Added in 7.0
408 Name.starts_with("vfmadd.") || // Added in 7.0
409 Name.starts_with("vfmaddsub.") || // Added in 7.0
410 Name.starts_with("vpdpbusd.") || // Added in 7.0
411 Name.starts_with("vpdpbusds.") || // Added in 7.0
412 Name.starts_with("vpdpwssd.") || // Added in 7.0
413 Name.starts_with("vpdpwssds.") || // Added in 7.0
414 Name.starts_with("vpermt2var.") || // Added in 7.0
415 Name.starts_with("vpmadd52") || // Added in 7.0
416 Name.starts_with("vpshldv.") || // Added in 8.0
417 Name.starts_with("vpshrdv.")); // Added in 8.0
418
419 // 'avx512.*'
420 return (Name == "movntdqa" || // Added in 5.0
421 Name == "pmul.dq.512" || // Added in 7.0
422 Name == "pmulu.dq.512" || // Added in 7.0
423 Name.starts_with("broadcastm") || // Added in 6.0
424 Name.starts_with("cmp.p") || // Added in 12.0
425 Name.starts_with("cvtb2mask.") || // Added in 7.0
426 Name.starts_with("cvtd2mask.") || // Added in 7.0
427 Name.starts_with("cvtmask2") || // Added in 5.0
428 Name.starts_with("cvtq2mask.") || // Added in 7.0
429 Name == "cvtusi2sd" || // Added in 7.0
430 Name.starts_with("cvtw2mask.") || // Added in 7.0
431 Name == "kand.w" || // Added in 7.0
432 Name == "kandn.w" || // Added in 7.0
433 Name == "knot.w" || // Added in 7.0
434 Name == "kor.w" || // Added in 7.0
435 Name == "kortestc.w" || // Added in 7.0
436 Name == "kortestz.w" || // Added in 7.0
437 Name.starts_with("kunpck") || // added in 6.0
438 Name == "kxnor.w" || // Added in 7.0
439 Name == "kxor.w" || // Added in 7.0
440 Name.starts_with("padds.") || // Added in 8.0
441 Name.starts_with("pbroadcast") || // Added in 3.9
442 Name.starts_with("pmulh.w") || // Added in 24.0
443 Name.starts_with("pmulhu.w") || // Added in 24.0
444 Name.starts_with("prol") || // Added in 8.0
445 Name.starts_with("pror") || // Added in 8.0
446 Name.starts_with("psll.dq") || // Added in 3.9
447 Name.starts_with("psrl.dq") || // Added in 3.9
448 Name.starts_with("psubs.") || // Added in 8.0
449 Name.starts_with("ptestm") || // Added in 6.0
450 Name.starts_with("ptestnm") || // Added in 6.0
451 Name.starts_with("storent.") || // Added in 3.9
452 Name.starts_with("vbroadcast.s") || // Added in 7.0
453 Name.starts_with("vpshld.") || // Added in 8.0
454 Name.starts_with("vpshrd.")); // Added in 8.0
455 }
456
457 if (Name.consume_front("fma."))
458 return (Name.starts_with("vfmadd.") || // Added in 7.0
459 Name.starts_with("vfmsub.") || // Added in 7.0
460 Name.starts_with("vfmsubadd.") || // Added in 7.0
461 Name.starts_with("vfnmadd.") || // Added in 7.0
462 Name.starts_with("vfnmsub.")); // Added in 7.0
463
464 if (Name.consume_front("fma4."))
465 return Name.starts_with("vfmadd.s"); // Added in 7.0
466
467 if (Name.consume_front("sse."))
468 return (Name == "add.ss" || // Added in 4.0
469 Name == "cvtsi2ss" || // Added in 7.0
470 Name == "cvtsi642ss" || // Added in 7.0
471 Name == "div.ss" || // Added in 4.0
472 Name == "mul.ss" || // Added in 4.0
473 Name.starts_with("sqrt.p") || // Added in 7.0
474 Name == "sqrt.ss" || // Added in 7.0
475 Name.starts_with("storeu.") || // Added in 3.9
476 Name == "sub.ss"); // Added in 4.0
477
478 if (Name.consume_front("sse2."))
479 return (Name == "add.sd" || // Added in 4.0
480 Name == "cvtdq2pd" || // Added in 3.9
481 Name == "cvtdq2ps" || // Added in 7.0
482 Name == "cvtps2pd" || // Added in 3.9
483 Name == "cvtsi2sd" || // Added in 7.0
484 Name == "cvtsi642sd" || // Added in 7.0
485 Name == "cvtss2sd" || // Added in 7.0
486 Name == "div.sd" || // Added in 4.0
487 Name == "mul.sd" || // Added in 4.0
488 Name.starts_with("padds.") || // Added in 8.0
489 Name.starts_with("paddus.") || // Added in 8.0
490 Name.starts_with("pcmpeq.") || // Added in 3.1
491 Name.starts_with("pcmpgt.") || // Added in 3.1
492 Name == "pmaxs.w" || // Added in 3.9
493 Name == "pmaxu.b" || // Added in 3.9
494 Name == "pmins.w" || // Added in 3.9
495 Name == "pminu.b" || // Added in 3.9
496 Name == "pmulh.w" || // Added in 24.0
497 Name == "pmulhu.w" || // Added in 24.0
498 Name == "pmulu.dq" || // Added in 7.0
499 Name.starts_with("pshuf") || // Added in 3.9
500 Name.starts_with("psll.dq") || // Added in 3.7
501 Name.starts_with("psrl.dq") || // Added in 3.7
502 Name.starts_with("psubs.") || // Added in 8.0
503 Name.starts_with("psubus.") || // Added in 8.0
504 Name.starts_with("sqrt.p") || // Added in 7.0
505 Name == "sqrt.sd" || // Added in 7.0
506 Name == "storel.dq" || // Added in 3.9
507 Name.starts_with("storeu.") || // Added in 3.9
508 Name == "sub.sd"); // Added in 4.0
509
510 if (Name.consume_front("sse41."))
511 return (Name.starts_with("blendp") || // Added in 3.7
512 Name == "movntdqa" || // Added in 5.0
513 Name == "pblendw" || // Added in 3.7
514 Name == "pmaxsb" || // Added in 3.9
515 Name == "pmaxsd" || // Added in 3.9
516 Name == "pmaxud" || // Added in 3.9
517 Name == "pmaxuw" || // Added in 3.9
518 Name == "pminsb" || // Added in 3.9
519 Name == "pminsd" || // Added in 3.9
520 Name == "pminud" || // Added in 3.9
521 Name == "pminuw" || // Added in 3.9
522 Name.starts_with("pmovsx") || // Added in 3.8
523 Name.starts_with("pmovzx") || // Added in 3.9
524 Name == "pmuldq"); // Added in 7.0
525
526 if (Name.consume_front("sse42."))
527 return Name == "crc32.64.8"; // Added in 3.4
528
529 if (Name.consume_front("sse4a."))
530 return Name.starts_with("movnt."); // Added in 3.9
531
532 if (Name.consume_front("ssse3."))
533 return (Name == "pabs.b.128" || // Added in 6.0
534 Name == "pabs.d.128" || // Added in 6.0
535 Name == "pabs.w.128"); // Added in 6.0
536
537 if (Name.consume_front("xop."))
538 return (Name == "vpcmov" || // Added in 3.8
539 Name == "vpcmov.256" || // Added in 5.0
540 Name.starts_with("vpcom") || // Added in 3.2, Updated in 9.0
541 Name.starts_with("vprot")); // Added in 8.0
542
543 if (Name.consume_front("bmi."))
544 return (Name.starts_with("pdep.") || // Added in 23.0
545 Name.starts_with("pext.")); // Added in 23.0
546
547 return (Name == "addcarry.u32" || // Added in 8.0
548 Name == "addcarry.u64" || // Added in 8.0
549 Name == "addcarryx.u32" || // Added in 8.0
550 Name == "addcarryx.u64" || // Added in 8.0
551 Name == "subborrow.u32" || // Added in 8.0
552 Name == "subborrow.u64" || // Added in 8.0
553 Name.starts_with("vcvtph2ps.")); // Added in 11.0
554}
555
556static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
557 Function *&NewFn) {
558 // Only handle intrinsics that start with "x86.".
559 if (!Name.consume_front("x86."))
560 return false;
561
562 if (shouldUpgradeX86Intrinsic(F, Name)) {
563 NewFn = nullptr;
564 return true;
565 }
566
567 if (Name == "rdtscp") { // Added in 8.0
568 // If this intrinsic has 0 operands, it's the new version.
569 if (F->getFunctionType()->getNumParams() == 0)
570 return false;
571
572 rename(F);
573 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
574 Intrinsic::x86_rdtscp);
575 return true;
576 }
577
578 Intrinsic::ID ID;
579
580 // SSE4.1 ptest functions may have an old signature.
581 if (Name.consume_front("sse41.ptest")) { // Added in 3.2
582 ID = StringSwitch<Intrinsic::ID>(Name)
583 .Case("c", Intrinsic::x86_sse41_ptestc)
584 .Case("z", Intrinsic::x86_sse41_ptestz)
585 .Case("nzc", Intrinsic::x86_sse41_ptestnzc)
586 .Default(Intrinsic::not_intrinsic);
587 if (ID != Intrinsic::not_intrinsic)
588 return upgradePTESTIntrinsic(F, ID, NewFn);
589
590 return false;
591 }
592
593 // Several blend and other instructions with masks used the wrong number of
594 // bits.
595
596 // Added in 3.6
597 ID = StringSwitch<Intrinsic::ID>(Name)
598 .Case("sse41.insertps", Intrinsic::x86_sse41_insertps)
599 .Case("sse41.dppd", Intrinsic::x86_sse41_dppd)
600 .Case("sse41.dpps", Intrinsic::x86_sse41_dpps)
601 .Case("sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
602 .Case("avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
603 .Case("avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
604 .Default(Intrinsic::not_intrinsic);
605 if (ID != Intrinsic::not_intrinsic)
606 return upgradeX86IntrinsicsWith8BitMask(F, ID, NewFn);
607
608 if (Name.consume_front("avx512.")) {
609 if (Name.consume_front("mask.cmp.")) {
610 // Added in 7.0
611 ID = StringSwitch<Intrinsic::ID>(Name)
612 .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
613 .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
614 .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
615 .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
616 .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
617 .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
618 .Default(Intrinsic::not_intrinsic);
619 if (ID != Intrinsic::not_intrinsic)
620 return upgradeX86MaskedFPCompare(F, ID, NewFn);
621 } else if (Name.starts_with("vpdpbusd.") ||
622 Name.starts_with("vpdpbusds.")) {
623 // Added in 21.1
624 ID = StringSwitch<Intrinsic::ID>(Name)
625 .Case("vpdpbusd.128", Intrinsic::x86_avx512_vpdpbusd_128)
626 .Case("vpdpbusd.256", Intrinsic::x86_avx512_vpdpbusd_256)
627 .Case("vpdpbusd.512", Intrinsic::x86_avx512_vpdpbusd_512)
628 .Case("vpdpbusds.128", Intrinsic::x86_avx512_vpdpbusds_128)
629 .Case("vpdpbusds.256", Intrinsic::x86_avx512_vpdpbusds_256)
630 .Case("vpdpbusds.512", Intrinsic::x86_avx512_vpdpbusds_512)
631 .Default(Intrinsic::not_intrinsic);
632 if (ID != Intrinsic::not_intrinsic)
633 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
634 } else if (Name.starts_with("vpdpwssd.") ||
635 Name.starts_with("vpdpwssds.")) {
636 // Added in 21.1
637 ID = StringSwitch<Intrinsic::ID>(Name)
638 .Case("vpdpwssd.128", Intrinsic::x86_avx512_vpdpwssd_128)
639 .Case("vpdpwssd.256", Intrinsic::x86_avx512_vpdpwssd_256)
640 .Case("vpdpwssd.512", Intrinsic::x86_avx512_vpdpwssd_512)
641 .Case("vpdpwssds.128", Intrinsic::x86_avx512_vpdpwssds_128)
642 .Case("vpdpwssds.256", Intrinsic::x86_avx512_vpdpwssds_256)
643 .Case("vpdpwssds.512", Intrinsic::x86_avx512_vpdpwssds_512)
644 .Default(Intrinsic::not_intrinsic);
645 if (ID != Intrinsic::not_intrinsic)
646 return upgradeX86MultiplyAddWords(F, ID, NewFn);
647 }
648 return false; // No other 'x86.avx512.*'.
649 }
650
651 if (Name.consume_front("avx2.")) {
652 if (Name.consume_front("vpdpb")) {
653 // Added in 21.1
654 ID = StringSwitch<Intrinsic::ID>(Name)
655 .Case("ssd.128", Intrinsic::x86_avx2_vpdpbssd_128)
656 .Case("ssd.256", Intrinsic::x86_avx2_vpdpbssd_256)
657 .Case("ssds.128", Intrinsic::x86_avx2_vpdpbssds_128)
658 .Case("ssds.256", Intrinsic::x86_avx2_vpdpbssds_256)
659 .Case("sud.128", Intrinsic::x86_avx2_vpdpbsud_128)
660 .Case("sud.256", Intrinsic::x86_avx2_vpdpbsud_256)
661 .Case("suds.128", Intrinsic::x86_avx2_vpdpbsuds_128)
662 .Case("suds.256", Intrinsic::x86_avx2_vpdpbsuds_256)
663 .Case("uud.128", Intrinsic::x86_avx2_vpdpbuud_128)
664 .Case("uud.256", Intrinsic::x86_avx2_vpdpbuud_256)
665 .Case("uuds.128", Intrinsic::x86_avx2_vpdpbuuds_128)
666 .Case("uuds.256", Intrinsic::x86_avx2_vpdpbuuds_256)
667 .Default(Intrinsic::not_intrinsic);
668 if (ID != Intrinsic::not_intrinsic)
669 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
670 } else if (Name.consume_front("vpdpw")) {
671 // Added in 21.1
672 ID = StringSwitch<Intrinsic::ID>(Name)
673 .Case("sud.128", Intrinsic::x86_avx2_vpdpwsud_128)
674 .Case("sud.256", Intrinsic::x86_avx2_vpdpwsud_256)
675 .Case("suds.128", Intrinsic::x86_avx2_vpdpwsuds_128)
676 .Case("suds.256", Intrinsic::x86_avx2_vpdpwsuds_256)
677 .Case("usd.128", Intrinsic::x86_avx2_vpdpwusd_128)
678 .Case("usd.256", Intrinsic::x86_avx2_vpdpwusd_256)
679 .Case("usds.128", Intrinsic::x86_avx2_vpdpwusds_128)
680 .Case("usds.256", Intrinsic::x86_avx2_vpdpwusds_256)
681 .Case("uud.128", Intrinsic::x86_avx2_vpdpwuud_128)
682 .Case("uud.256", Intrinsic::x86_avx2_vpdpwuud_256)
683 .Case("uuds.128", Intrinsic::x86_avx2_vpdpwuuds_128)
684 .Case("uuds.256", Intrinsic::x86_avx2_vpdpwuuds_256)
685 .Default(Intrinsic::not_intrinsic);
686 if (ID != Intrinsic::not_intrinsic)
687 return upgradeX86MultiplyAddWords(F, ID, NewFn);
688 }
689 return false; // No other 'x86.avx2.*'
690 }
691
692 if (Name.consume_front("avx10.")) {
693 if (Name.consume_front("vpdpb")) {
694 // Added in 21.1
695 ID = StringSwitch<Intrinsic::ID>(Name)
696 .Case("ssd.512", Intrinsic::x86_avx10_vpdpbssd_512)
697 .Case("ssds.512", Intrinsic::x86_avx10_vpdpbssds_512)
698 .Case("sud.512", Intrinsic::x86_avx10_vpdpbsud_512)
699 .Case("suds.512", Intrinsic::x86_avx10_vpdpbsuds_512)
700 .Case("uud.512", Intrinsic::x86_avx10_vpdpbuud_512)
701 .Case("uuds.512", Intrinsic::x86_avx10_vpdpbuuds_512)
702 .Default(Intrinsic::not_intrinsic);
703 if (ID != Intrinsic::not_intrinsic)
704 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
705 } else if (Name.consume_front("vpdpw")) {
706 ID = StringSwitch<Intrinsic::ID>(Name)
707 .Case("sud.512", Intrinsic::x86_avx10_vpdpwsud_512)
708 .Case("suds.512", Intrinsic::x86_avx10_vpdpwsuds_512)
709 .Case("usd.512", Intrinsic::x86_avx10_vpdpwusd_512)
710 .Case("usds.512", Intrinsic::x86_avx10_vpdpwusds_512)
711 .Case("uud.512", Intrinsic::x86_avx10_vpdpwuud_512)
712 .Case("uuds.512", Intrinsic::x86_avx10_vpdpwuuds_512)
713 .Default(Intrinsic::not_intrinsic);
714 if (ID != Intrinsic::not_intrinsic)
715 return upgradeX86MultiplyAddWords(F, ID, NewFn);
716 }
717 return false; // No other 'x86.avx10.*'
718 }
719
720 if (Name.consume_front("avx512bf16.")) {
721 // Added in 9.0
722 ID = StringSwitch<Intrinsic::ID>(Name)
723 .Case("cvtne2ps2bf16.128",
724 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
725 .Case("cvtne2ps2bf16.256",
726 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
727 .Case("cvtne2ps2bf16.512",
728 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
729 .Case("mask.cvtneps2bf16.128",
730 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
731 .Case("cvtneps2bf16.256",
732 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
733 .Case("cvtneps2bf16.512",
734 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
735 .Default(Intrinsic::not_intrinsic);
736 if (ID != Intrinsic::not_intrinsic)
737 return upgradeX86BF16Intrinsic(F, ID, NewFn);
738
739 // Added in 9.0
740 ID = StringSwitch<Intrinsic::ID>(Name)
741 .Case("dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
742 .Case("dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
743 .Case("dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
744 .Default(Intrinsic::not_intrinsic);
745 if (ID != Intrinsic::not_intrinsic)
746 return upgradeX86BF16DPIntrinsic(F, ID, NewFn);
747 return false; // No other 'x86.avx512bf16.*'.
748 }
749
750 if (Name.consume_front("xop.")) {
751 Intrinsic::ID ID = Intrinsic::not_intrinsic;
752 if (Name.starts_with("vpermil2")) { // Added in 3.9
753 // Upgrade any XOP PERMIL2 index operand still using a float/double
754 // vector.
755 auto Idx = F->getFunctionType()->getParamType(2);
756 if (Idx->isFPOrFPVectorTy()) {
757 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
758 unsigned EltSize = Idx->getScalarSizeInBits();
759 if (EltSize == 64 && IdxSize == 128)
760 ID = Intrinsic::x86_xop_vpermil2pd;
761 else if (EltSize == 32 && IdxSize == 128)
762 ID = Intrinsic::x86_xop_vpermil2ps;
763 else if (EltSize == 64 && IdxSize == 256)
764 ID = Intrinsic::x86_xop_vpermil2pd_256;
765 else
766 ID = Intrinsic::x86_xop_vpermil2ps_256;
767 }
768 } else if (F->arg_size() == 2)
769 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
770 ID = StringSwitch<Intrinsic::ID>(Name)
771 .Case("vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
772 .Case("vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
773 .Default(Intrinsic::not_intrinsic);
774
775 if (ID != Intrinsic::not_intrinsic) {
776 rename(F);
777 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
778 return true;
779 }
780 return false; // No other 'x86.xop.*'
781 }
782
783 if (Name == "seh.recoverfp") {
784 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
785 Intrinsic::eh_recoverfp);
786 return true;
787 }
788
789 return false;
790}
791
792// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
793// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
794static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
795 StringRef Name,
796 Function *&NewFn) {
797 if (Name.starts_with("rbit")) {
798 // '(arm|aarch64).rbit'.
799 NewFn = Intrinsic::getOrInsertDeclaration(
800 F->getParent(), Intrinsic::bitreverse, F->arg_begin()->getType());
801 return true;
802 }
803
804 if (Name == "thread.pointer") {
805 // '(arm|aarch64).thread.pointer'.
806 NewFn = Intrinsic::getOrInsertDeclaration(
807 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
808 return true;
809 }
810
811 bool Neon = Name.consume_front("neon.");
812 if (Neon) {
813 // '(arm|aarch64).neon.*'.
814 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
815 // v16i8 respectively.
816 if (Name.consume_front("bfdot.")) {
817 // (arm|aarch64).neon.bfdot.*'.
818 Intrinsic::ID ID =
819 StringSwitch<Intrinsic::ID>(Name)
820 .Cases({"v2f32.v8i8", "v4f32.v16i8"},
821 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
822 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
823 .Default(Intrinsic::not_intrinsic);
824 if (ID != Intrinsic::not_intrinsic) {
825 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
826 assert((OperandWidth == 64 || OperandWidth == 128) &&
827 "Unexpected operand width");
828 LLVMContext &Ctx = F->getParent()->getContext();
829 std::array<Type *, 2> Tys{
830 {F->getReturnType(),
831 FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)}};
832 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
833 return true;
834 }
835 return false; // No other '(arm|aarch64).neon.bfdot.*'.
836 }
837
838 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
839 // anymore and accept v8bf16 instead of v16i8.
840 if (Name.consume_front("bfm")) {
841 // (arm|aarch64).neon.bfm*'.
842 if (Name.consume_back(".v4f32.v16i8")) {
843 // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
844 Intrinsic::ID ID =
845 StringSwitch<Intrinsic::ID>(Name)
846 .Case("mla",
847 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
848 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
849 .Case("lalb",
850 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
851 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
852 .Case("lalt",
853 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
854 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
855 .Default(Intrinsic::not_intrinsic);
856 if (ID != Intrinsic::not_intrinsic) {
857 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
858 return true;
859 }
860 return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
861 }
862 return false; // No other '(arm|aarch64).neon.bfm*.
863 }
864 // Continue on to Aarch64 Neon or Arm Neon.
865 }
866 // Continue on to Arm or Aarch64.
867
868 if (IsArm) {
869 // 'arm.*'.
870 if (Neon) {
871 // 'arm.neon.*'.
872 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
873 .StartsWith("vclz.", Intrinsic::ctlz)
874 .StartsWith("vcnt.", Intrinsic::ctpop)
875 .StartsWith("vqadds.", Intrinsic::sadd_sat)
876 .StartsWith("vqaddu.", Intrinsic::uadd_sat)
877 .StartsWith("vqsubs.", Intrinsic::ssub_sat)
878 .StartsWith("vqsubu.", Intrinsic::usub_sat)
879 .StartsWith("vrinta.", Intrinsic::round)
880 .StartsWith("vrintn.", Intrinsic::roundeven)
881 .StartsWith("vrintm.", Intrinsic::floor)
882 .StartsWith("vrintp.", Intrinsic::ceil)
883 .StartsWith("vrintx.", Intrinsic::rint)
884 .StartsWith("vrintz.", Intrinsic::trunc)
885 .Default(Intrinsic::not_intrinsic);
886 if (ID != Intrinsic::not_intrinsic) {
887 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
888 F->arg_begin()->getType());
889 return true;
890 }
891
892 if (Name.consume_front("vst")) {
893 // 'arm.neon.vst*'.
894 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
895 SmallVector<StringRef, 2> Groups;
896 if (vstRegex.match(Name, &Groups)) {
897 static const Intrinsic::ID StoreInts[] = {
898 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
899 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
900
901 static const Intrinsic::ID StoreLaneInts[] = {
902 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
903 Intrinsic::arm_neon_vst4lane};
904
905 auto fArgs = F->getFunctionType()->params();
906 Type *Tys[] = {fArgs[0], fArgs[1]};
907 if (Groups[1].size() == 1)
908 NewFn = Intrinsic::getOrInsertDeclaration(
909 F->getParent(), StoreInts[fArgs.size() - 3], Tys);
910 else
911 NewFn = Intrinsic::getOrInsertDeclaration(
912 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
913 return true;
914 }
915 return false; // No other 'arm.neon.vst*'.
916 }
917
918 return false; // No other 'arm.neon.*'.
919 }
920
921 if (Name.consume_front("mve.")) {
922 // 'arm.mve.*'.
923 if (Name == "vctp64") {
924 if (cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
925 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
926 // the function and deal with it below in UpgradeIntrinsicCall.
927 rename(F);
928 return true;
929 }
930 return false; // Not 'arm.mve.vctp64'.
931 }
932
933 if (Name.starts_with("vrintn.v")) {
934 NewFn = Intrinsic::getOrInsertDeclaration(
935 F->getParent(), Intrinsic::roundeven, F->arg_begin()->getType());
936 return true;
937 }
938
939 // These too are changed to accept a v2i1 instead of the old v4i1.
940 if (Name.consume_back(".v4i1")) {
941 // 'arm.mve.*.v4i1'.
942 if (Name.consume_back(".predicated.v2i64.v4i32"))
943 // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
944 return Name == "mull.int" || Name == "vqdmull";
945
946 if (Name.consume_back(".v2i64")) {
947 // 'arm.mve.*.v2i64.v4i1'
948 bool IsGather = Name.consume_front("vldr.gather.");
949 if (IsGather || Name.consume_front("vstr.scatter.")) {
950 if (Name.consume_front("base.")) {
951 // Optional 'wb.' prefix.
952 Name.consume_front("wb.");
953 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
954 // predicated.v2i64.v2i64.v4i1'.
955 return Name == "predicated.v2i64";
956 }
957
958 if (Name.consume_front("offset.predicated."))
959 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
960 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
961
962 // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
963 return false;
964 }
965
966 return false; // No other 'arm.mve.*.v2i64.v4i1'.
967 }
968 return false; // No other 'arm.mve.*.v4i1'.
969 }
970 return false; // No other 'arm.mve.*'.
971 }
972
973 if (Name.consume_front("cde.vcx")) {
974 // 'arm.cde.vcx*'.
975 if (Name.consume_back(".predicated.v2i64.v4i1"))
976 // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
977 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
978 Name == "3q" || Name == "3qa";
979
980 return false; // No other 'arm.cde.vcx*'.
981 }
982 } else {
983 // 'aarch64.*'.
984 if (Neon) {
985 // 'aarch64.neon.*'.
986 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
987 .StartsWith("frintn", Intrinsic::roundeven)
988 .StartsWith("rbit", Intrinsic::bitreverse)
989 .Default(Intrinsic::not_intrinsic);
990 if (ID != Intrinsic::not_intrinsic) {
991 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
992 F->arg_begin()->getType());
993 return true;
994 }
995
996 Intrinsic::ID MinMaxID =
997 StringSwitch<Intrinsic::ID>(Name.split('.').first)
998 .Case("smax", Intrinsic::smax)
999 .Case("smin", Intrinsic::smin)
1000 .Case("umax", Intrinsic::umax)
1001 .Case("umin", Intrinsic::umin)
1002 .Default(Intrinsic::not_intrinsic);
1003 if (MinMaxID != Intrinsic::not_intrinsic) {
1004 if (F->arg_size() != 2 || !F->getReturnType()->isIntOrIntVectorTy())
1005 return false; // Invalid IR.
1006 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), MinMaxID,
1007 F->getReturnType());
1008 return true;
1009 }
1010
1011 if (Name.starts_with("addp")) {
1012 // 'aarch64.neon.addp*'.
1013 if (F->arg_size() != 2)
1014 return false; // Invalid IR.
1015 VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
1016 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
1017 NewFn = Intrinsic::getOrInsertDeclaration(
1018 F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
1019 return true;
1020 }
1021 }
1022
1023 // Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
1024 if (Name.starts_with("bfcvt")) {
1025 NewFn = nullptr;
1026 return true;
1027 }
1028
1029 // vcvtfp2hf and vcvthf2fp -> fpext and fptrunc
1030 if (Name == "vcvtfp2hf" || Name == "vcvthf2fp") {
1031 NewFn = nullptr;
1032 return true;
1033 }
1034
1035 return false; // No other 'aarch64.neon.*'.
1036 }
1037 if (Name.consume_front("sve.")) {
1038 // 'aarch64.sve.*'.
1039 if (Name.consume_front("bf")) {
1040 if (Name == "mmla") {
1041 Type *Tys[] = {F->getReturnType(),
1042 std::next(F->arg_begin())->getType()};
1043 NewFn = Intrinsic::getOrInsertDeclaration(
1044 F->getParent(), Intrinsic::aarch64_sve_fmmla, Tys);
1045 return true;
1046 }
1047 if (Name.consume_back(".lane")) {
1048 // 'aarch64.sve.bf*.lane'.
1049 Intrinsic::ID ID =
1050 StringSwitch<Intrinsic::ID>(Name)
1051 .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
1052 .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1053 .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1054 .Default(Intrinsic::not_intrinsic);
1055 if (ID != Intrinsic::not_intrinsic) {
1056 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1057 return true;
1058 }
1059 return false; // No other 'aarch64.sve.bf*.lane'.
1060 }
1061 return false; // No other 'aarch64.sve.bf*'.
1062 }
1063
1064 // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
1065 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
1066 NewFn = nullptr;
1067 return true;
1068 }
1069
1070 if (Name.consume_front("convert.from.svbool")) {
1071 // 'aarch64.sve.convert.from.svbool'
1072 auto *TTy = dyn_cast<TargetExtType>(F->getReturnType());
1073 if (!TTy || TTy->getName() != "aarch64.svcount")
1074 return false;
1075
1076 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_to_svcount;
1077 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1078 return true;
1079 }
1080
1081 if (Name.consume_front("convert.to.svbool")) {
1082 // 'aarch64.sve.convert.to.svbool'
1083 auto *TTy = dyn_cast<TargetExtType>(F->arg_begin()->getType());
1084 if (!TTy || TTy->getName() != "aarch64.svcount")
1085 return false;
1086
1087 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_from_svcount;
1088 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1089 return true;
1090 }
1091
1092 if (Name.consume_front("addqv")) {
1093 // 'aarch64.sve.addqv'.
1094 if (!F->getReturnType()->isFPOrFPVectorTy())
1095 return false;
1096
1097 auto Args = F->getFunctionType()->params();
1098 Type *Tys[] = {F->getReturnType(), Args[1]};
1099 NewFn = Intrinsic::getOrInsertDeclaration(
1100 F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);
1101 return true;
1102 }
1103
1104 if (Name.consume_front("ld")) {
1105 // 'aarch64.sve.ld*'.
1106 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
1107 if (LdRegex.match(Name)) {
1108 Type *ScalarTy =
1109 cast<VectorType>(F->getReturnType())->getElementType();
1110 ElementCount EC =
1111 cast<VectorType>(F->arg_begin()->getType())->getElementCount();
1112 assert(F->arg_size() == 2 &&
1113 "Expected 2 arguments for ld* intrinsic.");
1114 Type *PtrTy = F->getArg(1)->getType();
1115 Type *Ty = VectorType::get(ScalarTy, EC);
1116 static const Intrinsic::ID LoadIDs[] = {
1117 Intrinsic::aarch64_sve_ld2_sret,
1118 Intrinsic::aarch64_sve_ld3_sret,
1119 Intrinsic::aarch64_sve_ld4_sret,
1120 };
1121 NewFn = Intrinsic::getOrInsertDeclaration(
1122 F->getParent(), LoadIDs[Name[0] - '2'], {Ty, PtrTy});
1123 return true;
1124 }
1125 return false; // No other 'aarch64.sve.ld*'.
1126 }
1127
1128 if (Name.consume_front("tuple.")) {
1129 // 'aarch64.sve.tuple.*'.
1130 if (Name.starts_with("get")) {
1131 // 'aarch64.sve.tuple.get*'.
1132 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1133 NewFn = Intrinsic::getOrInsertDeclaration(
1134 F->getParent(), Intrinsic::vector_extract, Tys);
1135 return true;
1136 }
1137
1138 if (Name.starts_with("set")) {
1139 // 'aarch64.sve.tuple.set*'.
1140 auto Args = F->getFunctionType()->params();
1141 Type *Tys[] = {Args[0], Args[2], Args[1]};
1142 NewFn = Intrinsic::getOrInsertDeclaration(
1143 F->getParent(), Intrinsic::vector_insert, Tys);
1144 return true;
1145 }
1146
1147 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
1148 if (CreateTupleRegex.match(Name)) {
1149 // 'aarch64.sve.tuple.create*'.
1150 auto Args = F->getFunctionType()->params();
1151 Type *Tys[] = {F->getReturnType(), Args[1]};
1152 NewFn = Intrinsic::getOrInsertDeclaration(
1153 F->getParent(), Intrinsic::vector_insert, Tys);
1154 return true;
1155 }
1156 return false; // No other 'aarch64.sve.tuple.*'.
1157 }
1158
1159 if (Name.starts_with("rev.nxv")) {
1160 // 'aarch64.sve.rev.<Ty>'
1161 NewFn = Intrinsic::getOrInsertDeclaration(
1162 F->getParent(), Intrinsic::vector_reverse, F->getReturnType());
1163 return true;
1164 }
1165
1166 return false; // No other 'aarch64.sve.*'.
1167 }
1168 if (Name.consume_front("sme.")) {
1169 // 'aarch64.sme.*'.
1170 if (Name.consume_front("ftmopa.")) {
1171 // The FP8 FTMOPA intrinsics were split out from the non-FP8 FTMOPA
1172 // intrinsics to model their FPMR dependency.
1173 Intrinsic::ID ID =
1174 StringSwitch<Intrinsic::ID>(Name)
1175 .Case("za16.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1176 .Case("za32.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1177 .Default(Intrinsic::not_intrinsic);
1178 if (ID != Intrinsic::not_intrinsic) {
1179 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1180 return true;
1181 }
1182 return false; // No other 'aarch64.sme.ftmopa.*'.
1183 }
1184
1185 return false; // No other 'aarch64.sme.*'.
1186 }
1187 }
1188 return false; // No other 'arm.*', 'aarch64.*'.
1189}
1190
1191// The TMA G2S (global-to-shared) tensor copy modes that have legacy
1192// declarations requiring an auto-upgrade. The same set applies to the
1193// cluster (g2s) and CTA (g2s_cta) variants.
1194#define NVVM_TMA_G2S_MODES(M) \
1195 M(tile_1d, "tile.1d") \
1196 M(tile_2d, "tile.2d") \
1197 M(tile_3d, "tile.3d") \
1198 M(tile_4d, "tile.4d") \
1199 M(tile_5d, "tile.5d") \
1200 M(tile_gather4_2d, "tile.gather4.2d") \
1201 M(im2col_3d, "im2col.3d") \
1202 M(im2col_4d, "im2col.4d") \
1203 M(im2col_5d, "im2col.5d") \
1204 M(im2col_w_3d, "im2col.w.3d") \
1205 M(im2col_w_4d, "im2col.w.4d") \
1206 M(im2col_w_5d, "im2col.w.5d") \
1207 M(im2col_w_128_3d, "im2col.w.128.3d") \
1208 M(im2col_w_128_4d, "im2col.w.128.4d") \
1209 M(im2col_w_128_5d, "im2col.w.128.5d")
1210
1211// Two legacy tails are:
1212//
1213// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch
1214// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group
1215//
1216// The current tail appends a trailing i32 %flag_valid_pattern, so both
1217// legacy tails are recognized by an i1 at parameter N-2.
1218static Intrinsic::ID
1219shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name,
1220 SmallVectorImpl<Type *> &OvlTys) {
1221 if (!Name.consume_front("cp.async.bulk.tensor.g2s."))
1222 return Intrinsic::not_intrinsic;
1223
1224#define G2S_ID(ID_SUFFIX, NAME) \
1225 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX)
1226 // clang-format off
1227 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1228 NVVM_TMA_G2S_MODES(G2S_ID)
1229 .Default(Intrinsic::not_intrinsic);
1230#undef G2S_ID
1231 // clang-format on
1232 if (ID == Intrinsic::not_intrinsic)
1233 return ID;
1234
1235 size_t NumParams = F->getFunctionType()->getNumParams();
1236
1237 // Parameter N-2 is i1 for both legacy tails; the current tail ends
1238 // with i32 %cta_group, i32 %flag_valid_pattern, for which N-2 is i32.
1239 if (!F->getFunctionType()->getParamType(NumParams - 2)->isIntegerTy(1))
1240 return Intrinsic::not_intrinsic;
1241
1242 // The multicast mask is the parameter immediately before the i64
1243 // cache-hint: N-4 for the 2-flag tail, N-5 for the 3-flag tail.
1244 ArrayRef<Type *> Params = F->getFunctionType()->params();
1245 size_t MaskIdx =
1246 Params[NumParams - 1]->isIntegerTy(1) ? NumParams - 4 : NumParams - 5;
1247 assert(Params[MaskIdx + 1]->isIntegerTy(64) &&
1248 "expected the i64 cache-hint after the multicast mask");
1249 Type *MaskTy = Params[MaskIdx];
1250 assert(MaskTy->isIntegerTy(16) && "unexpected multicast mask type");
1251 OvlTys.push_back(MaskTy);
1252
1253 return ID;
1254}
1255
1256// The legacy tail is:
1257//
1258// arg1, arg2, .. i64 %ch, i1 %flag_ch
1259//
1260// The current tail appends a trailing i32 %flag_valid_pattern, so the
1261// legacy tail is recognized by an i1 at parameter N-1.
1262static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F,
1263 StringRef Name) {
1264 if (!Name.consume_front("cp.async.bulk.tensor.g2s.cta."))
1265 return Intrinsic::not_intrinsic;
1266
1267#define G2S_CTA_ID(ID_SUFFIX, NAME) \
1268 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX)
1269 // clang-format off
1270 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1271 NVVM_TMA_G2S_MODES(G2S_CTA_ID)
1272 .Default(Intrinsic::not_intrinsic);
1273#undef G2S_CTA_ID
1274 // clang-format on
1275 if (ID == Intrinsic::not_intrinsic)
1276 return ID;
1277
1278 // Parameter N-1 is i1 for the legacy tail; the current tail ends
1279 // with i32 %flag_valid_pattern, for which N-1 is i32.
1280 if (!F->getFunctionType()
1281 ->getParamType(F->getFunctionType()->getNumParams() - 1)
1282 ->isIntegerTy(1))
1283 return Intrinsic::not_intrinsic;
1284
1285 return ID;
1286}
1287// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cluster is:
1288//
1289// ..., i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch
1290//
1291// The current intrinsic is overloaded on the multicast-mask type and takes a
1292// trailing i32 %flag_valid_pattern; the legacy tail is recognized by an i1 at
1293// parameter N-1.
1294static Intrinsic::ID
1295shouldUpgradeNVPTXBulkG2SClusterIntrinsic(Function *F, StringRef Name,
1296 SmallVectorImpl<Type *> &OvlTys) {
1297 if (!Name.consume_front("cp.async.bulk.global.to.shared.cluster"))
1298 return Intrinsic::not_intrinsic;
1299
1300 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1301 // i32 %flag_valid_pattern, for which N-1 is i32.
1302 size_t NumParams = F->getFunctionType()->getNumParams();
1303 if (!F->getFunctionType()->getParamType(NumParams - 1)->isIntegerTy(1))
1304 return Intrinsic::not_intrinsic;
1305
1306 // The multicast mask is parameter 4; legacy IR only uses i16.
1307 Type *MaskTy = F->getFunctionType()->getParamType(NumParams - 4);
1308 if (!MaskTy->isIntegerTy(16))
1309 return Intrinsic::not_intrinsic;
1310 OvlTys.push_back(MaskTy);
1311
1312 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster;
1313}
1314
1315// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cta is:
1316//
1317// ..., i64 %ch, i1 %flag_ch
1318//
1319// The current intrinsic adds %ignore_bytes_left/%ignore_bytes_right before
1320// %ch and trailing %flag_oob/%flag_valid_pattern; the legacy tail is
1321// recognized by an i1 at parameter N-1, whereas the current tail ends
1322// with an i32.
1323static Intrinsic::ID shouldUpgradeNVPTXBulkG2SCTAIntrinsic(Function *F,
1324 StringRef Name) {
1325 if (!Name.consume_front("cp.async.bulk.global.to.shared.cta"))
1326 return Intrinsic::not_intrinsic;
1327
1328 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1329 // i32 %flag_valid_pattern, for which N-1 is i32.
1330 if (!F->getFunctionType()->getParamType(5)->isIntegerTy(1))
1331 return Intrinsic::not_intrinsic;
1332
1333 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta;
1334}
1335
1336// The legacy TMA reduction intrinsics encode the reduction operator in their
1337// name, while the current ones take it as an immediate argument. Map the
1338// operator part of a legacy name to the corresponding immediate value.
1339static std::optional<unsigned> getNVPTXTMAReductionOp(StringRef Name) {
1340 return StringSwitch<std::optional<unsigned>>(Name)
1341 .Case("add", static_cast<unsigned>(nvvm::TMAReductionOp::ADD))
1342 .Case("min", static_cast<unsigned>(nvvm::TMAReductionOp::MIN))
1343 .Case("max", static_cast<unsigned>(nvvm::TMAReductionOp::MAX))
1344 .Case("inc", static_cast<unsigned>(nvvm::TMAReductionOp::INC))
1345 .Case("dec", static_cast<unsigned>(nvvm::TMAReductionOp::DEC))
1346 .Case("and", static_cast<unsigned>(nvvm::TMAReductionOp::AND))
1347 .Case("or", static_cast<unsigned>(nvvm::TMAReductionOp::OR))
1348 .Case("xor", static_cast<unsigned>(nvvm::TMAReductionOp::XOR))
1349 .Default(std::nullopt);
1350}
1351
1352static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name) {
1353 if (!Name.consume_front("cp.async.bulk.tensor.reduce."))
1354 return Intrinsic::not_intrinsic;
1355
1356 auto [RedOpName, ShapeName] = Name.split('.');
1357 if (!getNVPTXTMAReductionOp(RedOpName))
1358 return Intrinsic::not_intrinsic;
1359
1360 return StringSwitch<Intrinsic::ID>(ShapeName)
1361 .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d)
1362 .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d)
1363 .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d)
1364 .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d)
1365 .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d)
1366 .Case("im2col.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d)
1367 .Case("im2col.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d)
1368 .Case("im2col.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d)
1369 .Default(Intrinsic::not_intrinsic);
1370}
1371
1372static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
1373 StringRef Name) {
1374 if (Name.consume_front("mapa.shared.cluster"))
1375 if (F->getReturnType()->getPointerAddressSpace() ==
1376 NVPTXAS::ADDRESS_SPACE_SHARED)
1377 return Intrinsic::nvvm_mapa_shared_cluster;
1378
1379 if (Name.consume_front("cp.async.bulk.")) {
1380 Intrinsic::ID ID =
1381 StringSwitch<Intrinsic::ID>(Name)
1382 .Case("shared.cta.to.cluster",
1383 Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1384 .Default(Intrinsic::not_intrinsic);
1385
1386 if (ID != Intrinsic::not_intrinsic)
1387 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1388 NVPTXAS::ADDRESS_SPACE_SHARED)
1389 return ID;
1390 }
1391
1392 return Intrinsic::not_intrinsic;
1393}
1394
1395static Intrinsic::ID
1396shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name) {
1397 if (!Name.consume_front("tcgen05.commit."))
1398 return Intrinsic::not_intrinsic;
1399
1400 if (Name.consume_front("shared."))
1401 return StringSwitch<Intrinsic::ID>(Name)
1402 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_cg1)
1403 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_cg2)
1404 .Default(Intrinsic::not_intrinsic);
1405
1406 if (Name.consume_front("mc.shared.")) {
1407 // Only upgrade older i16 mc variants.
1408 if (!F->getArg(1)->getType()->isIntegerTy(16))
1409 return Intrinsic::not_intrinsic;
1410
1411 return StringSwitch<Intrinsic::ID>(Name)
1412 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_mc_cg1)
1413 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_mc_cg2)
1414 .Default(Intrinsic::not_intrinsic);
1415 }
1416
1417 return Intrinsic::not_intrinsic;
1418}
1419
1420static Intrinsic::ID
1421shouldUpgradeNVPTXTcgen05AllocSharedIntrinsic(Function *F, StringRef Name) {
1422 if (F->arg_size() != 2)
1423 return Intrinsic::not_intrinsic;
1424
1425 return StringSwitch<Intrinsic::ID>(Name)
1426 .Case("tcgen05.alloc.shared.cg1", Intrinsic::nvvm_tcgen05_alloc_cg1)
1427 .Case("tcgen05.alloc.shared.cg2", Intrinsic::nvvm_tcgen05_alloc_cg2)
1428 .Default(Intrinsic::not_intrinsic);
1429}
1430
1431static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
1432 if (Name.consume_front("fmax."))
1433 return StringSwitch<Intrinsic::ID>(Name)
1434 .Case("bf16", Intrinsic::nvvm_fmax_bf16)
1435 .Case("bf16x2", Intrinsic::nvvm_fmax_bf16x2)
1436 .Case("ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
1437 .Case("ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
1438 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
1439 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1440 .Case("ftz.nan.xorsign.abs.bf16",
1441 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1442 .Case("ftz.nan.xorsign.abs.bf16x2",
1443 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1444 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1445 .Case("ftz.xorsign.abs.bf16x2",
1446 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1447 .Case("nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
1448 .Case("nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
1449 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1450 .Case("nan.xorsign.abs.bf16x2",
1451 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1452 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1453 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1454 .Default(Intrinsic::not_intrinsic);
1455
1456 if (Name.consume_front("fmin."))
1457 return StringSwitch<Intrinsic::ID>(Name)
1458 .Case("bf16", Intrinsic::nvvm_fmin_bf16)
1459 .Case("bf16x2", Intrinsic::nvvm_fmin_bf16x2)
1460 .Case("ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
1461 .Case("ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
1462 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
1463 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1464 .Case("ftz.nan.xorsign.abs.bf16",
1465 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1466 .Case("ftz.nan.xorsign.abs.bf16x2",
1467 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1468 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1469 .Case("ftz.xorsign.abs.bf16x2",
1470 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1471 .Case("nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
1472 .Case("nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
1473 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1474 .Case("nan.xorsign.abs.bf16x2",
1475 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1476 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1477 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1478 .Default(Intrinsic::not_intrinsic);
1479
1480 if (Name.consume_front("neg."))
1481 return StringSwitch<Intrinsic::ID>(Name)
1482 .Case("bf16", Intrinsic::nvvm_neg_bf16)
1483 .Case("bf16x2", Intrinsic::nvvm_neg_bf16x2)
1484 .Default(Intrinsic::not_intrinsic);
1485
1486 return Intrinsic::not_intrinsic;
1487}
1488
1489static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID) {
1490 FunctionType *NewFnTy = Intrinsic::getType(F->getContext(), IID);
1491 FunctionType *OldFnTy = F->getFunctionType();
1492 auto IsOldBF16StorageTy = [](Type *OldTy, Type *NewTy) {
1493 return OldTy->getScalarType()->isIntegerTy() &&
1494 OldTy->getPrimitiveSizeInBits() == NewTy->getPrimitiveSizeInBits();
1495 };
1496
1497 if (!IsOldBF16StorageTy(OldFnTy->getReturnType(), NewFnTy->getReturnType()))
1498 return false;
1499
1500 if (OldFnTy->getNumParams() != NewFnTy->getNumParams())
1501 return false;
1502
1503 for (unsigned I = 0, E = OldFnTy->getNumParams(); I != E; ++I)
1504 if (!IsOldBF16StorageTy(OldFnTy->getParamType(I), NewFnTy->getParamType(I)))
1505 return false;
1506
1507 return true;
1508}
1509
1510// Overloaded fadd/fmul/fma intrinsic IDs, indexed by
1511// [`.ftz`][none|`.sat`|`.relu`].
1512static constexpr Intrinsic::ID NVVMFAddIIDs[2][3] = {
1513 {Intrinsic::nvvm_fadd, Intrinsic::nvvm_fadd_sat, Intrinsic::not_intrinsic},
1514 {Intrinsic::nvvm_fadd_ftz, Intrinsic::nvvm_fadd_ftz_sat,
1515 Intrinsic::not_intrinsic}};
1516static constexpr Intrinsic::ID NVVMFMulIIDs[2][3] = {
1517 {Intrinsic::nvvm_fmul, Intrinsic::nvvm_fmul_sat, Intrinsic::not_intrinsic},
1518 {Intrinsic::nvvm_fmul_ftz, Intrinsic::nvvm_fmul_ftz_sat,
1519 Intrinsic::not_intrinsic}};
1520static constexpr Intrinsic::ID NVVMFMAIIDs[2][3] = {
1521 {Intrinsic::nvvm_fma, Intrinsic::nvvm_fma_sat, Intrinsic::nvvm_fma_relu},
1522 {Intrinsic::nvvm_fma_ftz, Intrinsic::nvvm_fma_ftz_sat,
1523 Intrinsic::nvvm_fma_ftz_relu}};
1524
1525// Returns true if the legacy nvvm.{add,mul,fma}.<rnd>{.ftz}{.sat|.relu}.<Type>
1526// intrinsic with these modifiers existed.
1527static bool isValidLegacyNVVMFPArithIntrinsic(const Intrinsic::ID (&IIDs)[2][3],
1528 StringRef Type, bool IsRN,
1529 bool IsFTZ, bool IsSat,
1530 bool IsRelu) {
1531 const bool IsFMA = &IIDs == &NVVMFMAIIDs;
1532 const bool IsFMul = &IIDs == &NVVMFMulIIDs;
1533 return StringSwitch<bool>(Type)
1534 .Case("f", !IsRelu && !(IsSat && IsFMul))
1535 .Case("d", !IsFTZ && !IsSat && !IsRelu)
1536 .Case("f16", IsRN && (IsFMA || IsSat))
1537 .Case("v2f16", IsRN && !IsFMA && IsSat)
1538 .Case("f16x2", IsRN && IsFMA)
1539 .Cases({"bf16", "bf16x2"}, IsRN && IsFMA && !IsFTZ && !IsSat)
1540 .Default(false);
1541}
1542
1543static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
1544getNVVMFPArithUpgrade(StringRef Name, const Intrinsic::ID (&IIDs)[2][3]) {
1545 auto [Modifiers, Type] = Name.rsplit('.');
1546
1547 std::optional<llvm::RoundingMode> RoundingMode =
1548 StringSwitch<std::optional<llvm::RoundingMode>>(Modifiers.take_front(2))
1549 .Case("rn", llvm::RoundingMode::NearestTiesToEven)
1550 .Case("rz", llvm::RoundingMode::TowardZero)
1551 .Case("rm", llvm::RoundingMode::TowardNegative)
1552 .Case("rp", llvm::RoundingMode::TowardPositive)
1553 .Default(std::nullopt);
1554 if (!RoundingMode)
1555 return std::nullopt;
1556
1557 StringRef Rest = Modifiers.drop_front(2);
1558 const bool IsFTZ = Rest.consume_front(".ftz");
1559 const bool IsSat = Rest.consume_front(".sat");
1560 const bool IsRelu = !IsSat && Rest.consume_front(".relu");
1561 if (!Rest.empty())
1562 return std::nullopt;
1563
1564 const Intrinsic::ID IID = IIDs[IsFTZ][IsSat ? 1 : IsRelu ? 2 : 0];
1565 if (IID == Intrinsic::not_intrinsic)
1566 return std::nullopt;
1567
1568 // Reject combinations that never existed as legacy intrinsics.
1569 const bool IsRN = *RoundingMode == llvm::RoundingMode::NearestTiesToEven;
1570 if (!isValidLegacyNVVMFPArithIntrinsic(IIDs, Type, IsRN, IsFTZ, IsSat,
1571 IsRelu))
1572 return std::nullopt;
1573
1574 return std::make_pair(IID, *RoundingMode);
1575}
1576
1577static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name) {
1578 if (Name != "mbarrier.init" && Name != "mbarrier.init.shared")
1579 return Intrinsic::not_intrinsic;
1580
1581 return Intrinsic::nvvm_mbarrier_init;
1582}
1583
1584static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
1585 return Name.consume_front("local") || Name.consume_front("shared") ||
1586 Name.consume_front("global") || Name.consume_front("constant") ||
1587 Name.consume_front("param");
1588}
1589
1590static unsigned getFunctionalOpcodeForVP(StringRef Name) {
1591 if (!Name.consume_front("vp."))
1592 return 0;
1593 return StringSwitch<unsigned>(Name)
1594 .StartsWith("select", Instruction::Select)
1595 .StartsWith("add", Instruction::Add)
1596 .StartsWith("sub", Instruction::Sub)
1597 .StartsWith("mul", Instruction::Mul)
1598 .StartsWith("ashr", Instruction::AShr)
1599 .StartsWith("lshr", Instruction::LShr)
1600 .StartsWith("shl", Instruction::Shl)
1601 .StartsWith("or", Instruction::Or)
1602 .StartsWith("and", Instruction::And)
1603 .StartsWith("xor", Instruction::Xor)
1604 .StartsWith("fadd", Instruction::FAdd)
1605 .StartsWith("fsub", Instruction::FSub)
1606 .StartsWith("fmuladd", 0)
1607 .StartsWith("fmul", Instruction::FMul)
1608 .StartsWith("fdiv", Instruction::FDiv)
1609 .StartsWith("frem", Instruction::FRem)
1610 .StartsWith("fneg", Instruction::FNeg)
1611 .StartsWith("trunc", Instruction::Trunc)
1612 .StartsWith("zext", Instruction::ZExt)
1613 .StartsWith("sext", Instruction::SExt)
1614 .StartsWith("fptrunc", Instruction::FPTrunc)
1615 .StartsWith("fpext", Instruction::FPExt)
1616 .StartsWith("fptoui", Instruction::FPToUI)
1617 .StartsWith("fptosi", Instruction::FPToSI)
1618 .StartsWith("uitofp", Instruction::UIToFP)
1619 .StartsWith("sitofp", Instruction::SIToFP)
1620 .StartsWith("ptrtoint", Instruction::PtrToInt)
1621 .StartsWith("inttoptr", Instruction::IntToPtr)
1622 .StartsWith("icmp", Instruction::ICmp)
1623 .StartsWith("fcmp", Instruction::FCmp)
1624 .Default(0);
1625}
1626
1627static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name) {
1628 if (!Name.consume_front("vp."))
1629 return 0;
1630 return StringSwitch<Intrinsic::ID>(Name)
1631 .StartsWith("abs", Intrinsic::abs)
1632 .StartsWith("smax", Intrinsic::smax)
1633 .StartsWith("smin", Intrinsic::smin)
1634 .StartsWith("umax", Intrinsic::umax)
1635 .StartsWith("umin", Intrinsic::umin)
1636 .StartsWith("copysign", Intrinsic::copysign)
1637 .StartsWith("minnum", Intrinsic::minnum)
1638 .StartsWith("maxnum", Intrinsic::maxnum)
1639 .StartsWith("minimum", Intrinsic::minimum)
1640 .StartsWith("maximum", Intrinsic::maximum)
1641 .StartsWith("fabs", Intrinsic::fabs)
1642 .StartsWith("sqrt", Intrinsic::sqrt)
1643 .StartsWith("fma", Intrinsic::fma)
1644 .StartsWith("fmuladd", Intrinsic::fmuladd)
1645 .StartsWith("ceil", Intrinsic::ceil)
1646 .StartsWith("floor", Intrinsic::floor)
1647 .StartsWith("rint", Intrinsic::rint)
1648 .StartsWith("nearbyint", Intrinsic::nearbyint)
1649 .StartsWith("roundeven", Intrinsic::roundeven)
1650 .StartsWith("roundtozero", Intrinsic::trunc)
1651 .StartsWith("round", Intrinsic::round)
1652 .StartsWith("lrint", Intrinsic::lrint)
1653 .StartsWith("llrint", Intrinsic::llrint)
1654 .StartsWith("bitreverse", Intrinsic::bitreverse)
1655 .StartsWith("bswap", Intrinsic::bswap)
1656 .StartsWith("ctpop", Intrinsic::ctpop)
1657 .StartsWith("ctlz", Intrinsic::ctlz)
1658 .StartsWith("cttz.elts", 0)
1659 .StartsWith("cttz", Intrinsic::cttz)
1660 .StartsWith("sadd.sat", Intrinsic::sadd_sat)
1661 .StartsWith("uadd.sat", Intrinsic::uadd_sat)
1662 .StartsWith("ssub.sat", Intrinsic::ssub_sat)
1663 .StartsWith("usub.sat", Intrinsic::usub_sat)
1664 .StartsWith("fshl", Intrinsic::fshl)
1665 .StartsWith("fshr", Intrinsic::fshr)
1666 .StartsWith("is.fpclass", Intrinsic::is_fpclass)
1667 .Default(0);
1668}
1669
1673
1674static bool convertIntrinsicValidType(StringRef Name,
1675 const FunctionType *FuncTy) {
1676 Type *HalfTy = Type::getHalfTy(FuncTy->getContext());
1677 if (Name.starts_with("to.fp16")) {
1678 return CastInst::castIsValid(Instruction::FPTrunc, FuncTy->getParamType(0),
1679 HalfTy) &&
1680 CastInst::castIsValid(Instruction::BitCast, HalfTy,
1681 FuncTy->getReturnType());
1682 }
1683
1684 if (Name.starts_with("from.fp16")) {
1685 return CastInst::castIsValid(Instruction::BitCast, FuncTy->getParamType(0),
1686 HalfTy) &&
1687 CastInst::castIsValid(Instruction::FPExt, HalfTy,
1688 FuncTy->getReturnType());
1689 }
1690
1691 return false;
1692}
1693
1694static unsigned
1695getFullArgCountForDefaultArgUpgrade(Function *F, Intrinsic::ID IID,
1696 SmallVectorImpl<Type *> &OverloadTys) {
1697 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
1698 if (Defaults.empty())
1699 return 0;
1700
1701 unsigned FullArgCount = FirstDefault + Defaults.size();
1702
1703 // Only trailing default arguments can be missing.
1704 if (F->arg_size() < FirstDefault || F->arg_size() >= FullArgCount)
1705 return 0;
1706
1707 unsigned NumMissingTrailingParams = FullArgCount - F->arg_size();
1708 if (!Intrinsic::isSignatureValid(IID, F->getFunctionType(), OverloadTys,
1709 NumMissingTrailingParams))
1710 return 0;
1711
1712 return FullArgCount;
1713}
1714
1715static bool upgradeIntrinsicWithDefaultArgs(Function *F, Function *&NewFn) {
1716 Intrinsic::ID IID = F->getIntrinsicID();
1717 SmallVector<Type *, 4> OverloadTys;
1718
1719 unsigned FullArgCount =
1720 getFullArgCountForDefaultArgUpgrade(F, IID, OverloadTys);
1721 if (FullArgCount == 0)
1722 return false;
1723
1724 rename(F);
1725 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID, OverloadTys);
1726 assert(NewFn->arg_size() == FullArgCount &&
1727 "total number of default args does not match intrinsic signature");
1728 return true;
1729}
1730
1731static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
1732 bool CanUpgradeDebugIntrinsicsToRecords) {
1733 assert(F && "Illegal to upgrade a non-existent Function.");
1734
1735 StringRef Name = F->getName();
1736
1737 // Quickly eliminate it, if it's not a candidate.
1738 if (!Name.consume_front("llvm.") || Name.empty())
1739 return false;
1740
1741 switch (Name[0]) {
1742 default: break;
1743 case 'a': {
1744 bool IsArm = Name.consume_front("arm.");
1745 if (IsArm || Name.consume_front("aarch64.")) {
1746 if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1747 return true;
1748 break;
1749 }
1750
1751 if (Name.consume_front("amdgcn.")) {
1752 if (Name == "alignbit") {
1753 // Target specific intrinsic became redundant
1754 NewFn = Intrinsic::getOrInsertDeclaration(
1755 F->getParent(), Intrinsic::fshr, {F->getReturnType()});
1756 return true;
1757 }
1758
1759 if (Name.consume_front("atomic.")) {
1760 if (Name.starts_with("inc") || Name.starts_with("dec") ||
1761 Name.starts_with("cond.sub") || Name.starts_with("csub")) {
1762 // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1763 // and usub_sat so there's no new declaration.
1764 NewFn = nullptr;
1765 return true;
1766 }
1767 break; // No other 'amdgcn.atomic.*'
1768 }
1769
1770 if (Name.starts_with("addrspacecast.nonnull")) {
1771 // Replaced with an addrspacecast instruction carrying the nonnull flag,
1772 // so there's no new declaration.
1773 NewFn = nullptr;
1774 return true;
1775 }
1776
1777 switch (F->getIntrinsicID()) {
1778 default:
1779 break;
1780 // Legacy asyncmark intrinsics without the stage operand.
1781 case Intrinsic::amdgcn_asyncmark:
1782 case Intrinsic::amdgcn_wait_asyncmark: {
1783 Intrinsic::ID ID = F->getIntrinsicID();
1784 if (F->arg_size() == (ID == Intrinsic::amdgcn_asyncmark ? 0u : 1u)) {
1785 rename(F);
1786 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1787 return true;
1788 }
1789 break;
1790 }
1791 // Legacy wmma iu intrinsics without the optional clamp operand.
1792 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1793 if (F->arg_size() == 7) {
1794 NewFn = nullptr;
1795 return true;
1796 }
1797 break;
1798 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1799 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1800 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1801 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1802 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1803 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1804 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1805 if (F->arg_size() == 8) {
1806 NewFn = nullptr;
1807 return true;
1808 }
1809 break;
1810 }
1811
1812 if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||
1813 Name.consume_front("flat.atomic.")) {
1814 if (Name.starts_with("fadd") ||
1815 // FIXME: We should also remove fmin.num and fmax.num intrinsics.
1816 (Name.starts_with("fmin") && !Name.starts_with("fmin.num")) ||
1817 (Name.starts_with("fmax") && !Name.starts_with("fmax.num"))) {
1818 // Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1819 // declaration.
1820 NewFn = nullptr;
1821 return true;
1822 }
1823 }
1824
1825 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
1826 NewFn = nullptr;
1827 return true;
1828 }
1829
1830 if (Name.starts_with("ldexp.")) {
1831 // Target specific intrinsic became redundant
1832 NewFn = Intrinsic::getOrInsertDeclaration(
1833 F->getParent(), Intrinsic::ldexp,
1834 {F->getReturnType(), F->getArg(1)->getType()});
1835 return true;
1836 }
1837 break; // No other 'amdgcn.*'
1838 }
1839
1840 break;
1841 }
1842 case 'c': {
1843 if (F->arg_size() == 1) {
1844 if (Name.consume_front("convert.")) {
1845 if (convertIntrinsicValidType(Name, F->getFunctionType())) {
1846 NewFn = nullptr;
1847 return true;
1848 }
1849 }
1850
1851 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1852 .StartsWith("ctlz.", Intrinsic::ctlz)
1853 .StartsWith("cttz.", Intrinsic::cttz)
1854 .Default(Intrinsic::not_intrinsic);
1855 if (ID != Intrinsic::not_intrinsic) {
1856 rename(F);
1857 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1858 F->arg_begin()->getType());
1859 return true;
1860 }
1861 }
1862
1863 Intrinsic::ID CoroEndID = Intrinsic::not_intrinsic;
1864 if (Name == "coro.end" &&
1865 (F->arg_size() == 2 || F->getReturnType()->isIntegerTy(1)))
1866 CoroEndID = Intrinsic::coro_end;
1867 else if (Name == "coro.end.async" && F->getReturnType()->isIntegerTy(1))
1868 CoroEndID = Intrinsic::coro_end_async;
1869
1870 if (CoroEndID != Intrinsic::not_intrinsic) {
1871 rename(F);
1872 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), CoroEndID);
1873 return true;
1874 }
1875
1876 break;
1877 }
1878 case 'd':
1879 if (Name.consume_front("dbg.")) {
1880 // Mark debug intrinsics for upgrade to new debug format.
1881 if (CanUpgradeDebugIntrinsicsToRecords) {
1882 if (Name == "addr" || Name == "value" || Name == "assign" ||
1883 Name == "declare" || Name == "label") {
1884 // There's no function to replace these with.
1885 NewFn = nullptr;
1886 // But we do want these to get upgraded.
1887 return true;
1888 }
1889 }
1890 // Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1891 // converted to DbgVariableRecords later.
1892 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1893 rename(F);
1894 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1895 Intrinsic::dbg_value);
1896 return true;
1897 }
1898 break; // No other 'dbg.*'.
1899 }
1900 break;
1901 case 'e':
1902 if (Name.consume_front("experimental.vector.")) {
1903 Intrinsic::ID ID =
1904 StringSwitch<Intrinsic::ID>(Name)
1905 // Skip over extract.last.active, otherwise it will be 'upgraded'
1906 // to a regular vector extract which is a different operation.
1907 .StartsWith("extract.last.active.", Intrinsic::not_intrinsic)
1908 .StartsWith("extract.", Intrinsic::vector_extract)
1909 .StartsWith("insert.", Intrinsic::vector_insert)
1910 .StartsWith("reverse.", Intrinsic::vector_reverse)
1911 .StartsWith("interleave2.", Intrinsic::vector_interleave2)
1912 .StartsWith("deinterleave2.", Intrinsic::vector_deinterleave2)
1913 .StartsWith("partial.reduce.add",
1914 Intrinsic::vector_partial_reduce_add)
1915 .Default(Intrinsic::not_intrinsic);
1916 if (ID != Intrinsic::not_intrinsic) {
1917 const auto *FT = F->getFunctionType();
1918 SmallVector<Type *, 2> Tys;
1919 if (ID == Intrinsic::vector_extract ||
1920 ID == Intrinsic::vector_interleave2)
1921 // Extracting overloads the return type.
1922 Tys.push_back(FT->getReturnType());
1923 if (ID != Intrinsic::vector_interleave2)
1924 Tys.push_back(FT->getParamType(0));
1925 if (ID == Intrinsic::vector_insert ||
1926 ID == Intrinsic::vector_partial_reduce_add)
1927 // Inserting overloads the inserted type.
1928 Tys.push_back(FT->getParamType(1));
1929 rename(F);
1930 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
1931 return true;
1932 }
1933
1934 if (Name.consume_front("reduce.")) {
1935 SmallVector<StringRef, 2> Groups;
1936 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1937 if (R.match(Name, &Groups))
1938 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1939 .Case("add", Intrinsic::vector_reduce_add)
1940 .Case("mul", Intrinsic::vector_reduce_mul)
1941 .Case("and", Intrinsic::vector_reduce_and)
1942 .Case("or", Intrinsic::vector_reduce_or)
1943 .Case("xor", Intrinsic::vector_reduce_xor)
1944 .Case("smax", Intrinsic::vector_reduce_smax)
1945 .Case("smin", Intrinsic::vector_reduce_smin)
1946 .Case("umax", Intrinsic::vector_reduce_umax)
1947 .Case("umin", Intrinsic::vector_reduce_umin)
1948 .Case("fmax", Intrinsic::vector_reduce_fmax)
1949 .Case("fmin", Intrinsic::vector_reduce_fmin)
1950 .Default(Intrinsic::not_intrinsic);
1951
1952 bool V2 = false;
1953 if (ID == Intrinsic::not_intrinsic) {
1954 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1955 Groups.clear();
1956 V2 = true;
1957 if (R2.match(Name, &Groups))
1958 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1959 .Case("fadd", Intrinsic::vector_reduce_fadd)
1960 .Case("fmul", Intrinsic::vector_reduce_fmul)
1961 .Default(Intrinsic::not_intrinsic);
1962 }
1963 if (ID != Intrinsic::not_intrinsic) {
1964 rename(F);
1965 auto Args = F->getFunctionType()->params();
1966 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1967 {Args[V2 ? 1 : 0]});
1968 return true;
1969 }
1970 break; // No other 'expermental.vector.reduce.*'.
1971 }
1972
1973 if (Name.consume_front("splice"))
1974 return true;
1975 break; // No other 'experimental.vector.*'.
1976 }
1977 if (Name.consume_front("experimental.stepvector.")) {
1978 Intrinsic::ID ID = Intrinsic::stepvector;
1979 rename(F);
1980 NewFn = Intrinsic::getOrInsertDeclaration(
1981 F->getParent(), ID, F->getFunctionType()->getReturnType());
1982 return true;
1983 }
1984 break; // No other 'e*'.
1985 case 'f':
1986 if (Name.starts_with("flt.rounds")) {
1987 rename(F);
1988 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1989 Intrinsic::get_rounding);
1990 return true;
1991 }
1992 break;
1993 case 'i':
1994 if (Name.starts_with("invariant.group.barrier")) {
1995 // Rename invariant.group.barrier to launder.invariant.group
1996 auto Args = F->getFunctionType()->params();
1997 Type* ObjectPtr[1] = {Args[0]};
1998 rename(F);
1999 NewFn = Intrinsic::getOrInsertDeclaration(
2000 F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);
2001 return true;
2002 }
2003 break;
2004 case 'l': {
2005 bool IsLifetimeStart = Name.consume_front("lifetime.start");
2006 bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front("lifetime.end");
2007 if (IsLifetimeStart || IsLifetimeEnd) {
2008 if (F->arg_size() == 2) {
2009 Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
2010 : Intrinsic::lifetime_end;
2011 rename(F);
2012 // Old 2 argument form of these intrinsics have [Size, Ptr] as
2013 // arguments. Use the Ptr argument to create new declaration.
2014 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2015 F->getArg(1)->getType());
2016 return true;
2017 } else if (F->arg_size() == 1 && Name == ".i64") {
2018 // Matches @llvm.lifetime.{start/end}.i64 which used to be created by
2019 // Autoupgrade prior to
2020 // https://github.com/llvm/llvm-project/pull/204601. This is an invalid
2021 // intrinsic with no expected calls. To allow auto-upgrade process to
2022 // delete such invalid intrinsic declaration, set NewFn = nullptr
2023 // and return true here. If there are actual calls to this intrinsic
2024 // (which is not expected), they will be deleted in
2025 // UpgradeIntrinsicCall.
2026 NewFn = nullptr;
2027 return true;
2028 }
2029 }
2030 break;
2031 }
2032 case 'm': {
2033 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
2034 // alignment parameter to embedding the alignment as an attribute of
2035 // the pointer args.
2036 if (unsigned ID = StringSwitch<unsigned>(Name)
2037 .StartsWith("memcpy.", Intrinsic::memcpy)
2038 .StartsWith("memmove.", Intrinsic::memmove)
2039 .Default(0)) {
2040 if (F->arg_size() == 5) {
2041 rename(F);
2042 // Get the types of dest, src, and len
2043 ArrayRef<Type *> ParamTypes =
2044 F->getFunctionType()->params().slice(0, 3);
2045 NewFn =
2046 Intrinsic::getOrInsertDeclaration(F->getParent(), ID, ParamTypes);
2047 return true;
2048 }
2049 }
2050 if (Name.starts_with("memset.") && F->arg_size() == 5) {
2051 rename(F);
2052 // Get the types of dest, and len
2053 const auto *FT = F->getFunctionType();
2054 Type *ParamTypes[2] = {
2055 FT->getParamType(0), // Dest
2056 FT->getParamType(2) // len
2057 };
2058 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2059 Intrinsic::memset, ParamTypes);
2060 return true;
2061 }
2062
2063 unsigned MaskedID =
2064 StringSwitch<unsigned>(Name)
2065 .StartsWith("masked.load", Intrinsic::masked_load)
2066 .StartsWith("masked.gather", Intrinsic::masked_gather)
2067 .StartsWith("masked.store", Intrinsic::masked_store)
2068 .StartsWith("masked.scatter", Intrinsic::masked_scatter)
2069 .Default(0);
2070 if (MaskedID && F->arg_size() == 4) {
2071 rename(F);
2072 if (MaskedID == Intrinsic::masked_load ||
2073 MaskedID == Intrinsic::masked_gather) {
2074 NewFn = Intrinsic::getOrInsertDeclaration(
2075 F->getParent(), MaskedID,
2076 {F->getReturnType(), F->getArg(0)->getType()});
2077 return true;
2078 }
2079 NewFn = Intrinsic::getOrInsertDeclaration(
2080 F->getParent(), MaskedID,
2081 {F->getArg(0)->getType(), F->getArg(1)->getType()});
2082 return true;
2083 }
2084 break;
2085 }
2086 case 'n': {
2087 if (Name.consume_front("nvvm.")) {
2088 // Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
2089 if (F->arg_size() == 1) {
2090 Intrinsic::ID IID =
2091 StringSwitch<Intrinsic::ID>(Name)
2092 .Cases({"brev32", "brev64"}, Intrinsic::bitreverse)
2093 .Case("clz.i", Intrinsic::ctlz)
2094 .Case("popc.i", Intrinsic::ctpop)
2095 .Default(Intrinsic::not_intrinsic);
2096 if (IID != Intrinsic::not_intrinsic) {
2097 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2098 {F->getReturnType()});
2099 return true;
2100 }
2101 } else if (F->arg_size() == 2) {
2102 Intrinsic::ID IID =
2103 StringSwitch<Intrinsic::ID>(Name)
2104 .Cases({"max.s", "max.i", "max.ll"}, Intrinsic::smax)
2105 .Cases({"min.s", "min.i", "min.ll"}, Intrinsic::smin)
2106 .Cases({"max.us", "max.ui", "max.ull"}, Intrinsic::umax)
2107 .Cases({"min.us", "min.ui", "min.ull"}, Intrinsic::umin)
2108 .Cases({"mulhi.s", "mulhi.i", "mulhi.ll"}, Intrinsic::smulh)
2109 .Cases({"mulhi.us", "mulhi.ui", "mulhi.ull"}, Intrinsic::umulh)
2110 .Default(Intrinsic::not_intrinsic);
2111 if (IID != Intrinsic::not_intrinsic) {
2112 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2113 {F->getReturnType()});
2114 return true;
2115 }
2116 }
2117
2118 // Check for nvvm intrinsics that need a return type adjustment.
2119 {
2120 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
2121 if (IID != Intrinsic::not_intrinsic &&
2122 isLegacyNVPTXBF16IntSignature(F, IID)) {
2123 NewFn = nullptr;
2124 return true;
2125 }
2126 }
2127
2128 // Upgrade Distributed Shared Memory Intrinsics
2129 Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
2130 if (IID != Intrinsic::not_intrinsic) {
2131 rename(F);
2132 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2133 return true;
2134 }
2135
2136 // Upgrade TMA reduction intrinsics
2137 // llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>* =>
2138 // llvm.nvvm.cp.async.bulk.tensor.reduce.<shape>*
2139 IID = shouldUpgradeNVPTXTMAReductionIntrinsics(Name);
2140 if (IID != Intrinsic::not_intrinsic) {
2141 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2142 return true;
2143 }
2144
2145 // Upgrade tcgen05.commit shared variants to anyptr intrinsics.
2146 IID = shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(F, Name);
2147 if (IID != Intrinsic::not_intrinsic) {
2148 rename(F);
2149 NewFn = Intrinsic::getOrInsertDeclaration(
2150 F->getParent(), IID, F->getReturnType(),
2151 F->getFunctionType()->params());
2152 return true;
2153 }
2154
2155 // Upgrade tcgen05.alloc.shared.* to anyptr alloc intrinsic.
2156 IID = shouldUpgradeNVPTXTcgen05AllocSharedIntrinsic(F, Name);
2157 if (IID != Intrinsic::not_intrinsic) {
2158 rename(F);
2159 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2160 {F->getArg(0)->getType()});
2161 return true;
2162 }
2163
2164 // Upgrade TMA copy G2S CTA intrinsics.
2165 IID = shouldUpgradeNVPTXTMAG2SCTAIntrinsics(F, Name);
2166 if (IID != Intrinsic::not_intrinsic) {
2167 rename(F);
2168 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2169 return true;
2170 }
2171
2172 // Upgrade TMA copy G2S (cluster) intrinsics.
2173 SmallVector<Type *, 1> OvlTys;
2174 IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name, OvlTys);
2175 if (IID != Intrinsic::not_intrinsic) {
2176 rename(F);
2177 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID, OvlTys);
2178 return true;
2179 }
2180
2181 // Upgrade the legacy cp.async.bulk.global.to.shared.cluster signature
2182 // (multicast-mask overloading + trailing flag_valid_pattern).
2183 SmallVector<Type *, 1> BulkG2SOvlTys;
2184 IID = shouldUpgradeNVPTXBulkG2SClusterIntrinsic(F, Name, BulkG2SOvlTys);
2185 if (IID != Intrinsic::not_intrinsic) {
2186 rename(F);
2187 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2188 BulkG2SOvlTys);
2189 return true;
2190 }
2191
2192 // Upgrade the legacy cp.async.bulk.global.to.shared.cta signature
2193 // (no ignore_bytes_left/right + trailing flag_valid_pattern).
2194 IID = shouldUpgradeNVPTXBulkG2SCTAIntrinsic(F, Name);
2195 if (IID != Intrinsic::not_intrinsic) {
2196 rename(F);
2197 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2198 return true;
2199 }
2200
2201 // Upgrade mbarrier.init intrinsics missing the layout operand.
2202 IID = shouldUpgradeNVPTXMBarrierInitIntrinsic(Name);
2203 if (IID != Intrinsic::not_intrinsic) {
2204 rename(F);
2205 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2206 F->getArg(0)->getType());
2207 return true;
2208 }
2209
2210 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
2211 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
2212 //
2213 // TODO: We could add lohi.i2d.
2214 bool Expand = false;
2215 if (Name.consume_front("abs."))
2216 // nvvm.abs.{i,ii}
2217 Expand =
2218 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
2219 else if (Name.consume_front("fabs."))
2220 // nvvm.fabs.{f,ftz.f,d}
2221 Expand = Name == "f" || Name == "ftz.f" || Name == "d";
2222 else if (Name.consume_front("add."))
2223 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
2224 Expand = getNVVMFPArithUpgrade(Name, NVVMFAddIIDs).has_value();
2225 else if (Name.consume_front("mul."))
2226 // nvvm.mul.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
2227 Expand = getNVVMFPArithUpgrade(Name, NVVMFMulIIDs).has_value();
2228 else if (Name.consume_front("ex2.approx."))
2229 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
2230 Expand =
2231 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
2232 else if (Name.consume_front("fma."))
2233 // nvvm.fma.<rnd>{.ftz}{.sat|.relu}.{f,d,f16,f16x2,bf16,bf16x2}
2234 Expand = getNVVMFPArithUpgrade(Name, NVVMFMAIIDs).has_value();
2235 else if (Name.consume_front("atomic.load."))
2236 // nvvm.atomic.load.add.{f32,f64}.p
2237 // nvvm.atomic.load.{inc,dec}.32.p
2238 Expand = StringSwitch<bool>(Name)
2239 .StartsWith("add.f32.p", true)
2240 .StartsWith("add.f64.p", true)
2241 .StartsWith("inc.32.p", true)
2242 .StartsWith("dec.32.p", true)
2243 .Default(false);
2244 else if (Name.consume_front("atomic."))
2245 // nvvm.atomic.{add,exch,max,min,inc,dec,and,or,xor}.gen.{i,f}.{cta,sys}
2246 // nvvm.atomic.cas.gen.i.{cta,sys}
2247 Expand = StringSwitch<bool>(Name)
2248 .StartsWith("add.gen.", true)
2249 .StartsWith("exch.gen.", true)
2250 .StartsWith("max.gen.", true)
2251 .StartsWith("min.gen.", true)
2252 .StartsWith("inc.gen.", true)
2253 .StartsWith("dec.gen.", true)
2254 .StartsWith("and.gen.", true)
2255 .StartsWith("or.gen.", true)
2256 .StartsWith("xor.gen.", true)
2257 .StartsWith("cas.gen.", true)
2258 .Default(false);
2259 else if (Name.consume_front("bitcast."))
2260 // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
2261 Expand =
2262 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
2263 else if (Name.consume_front("rotate."))
2264 // nvvm.rotate.{b32,b64,right.b64}
2265 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
2266 else if (Name.consume_front("ptr.gen.to."))
2267 // nvvm.ptr.gen.to.{local,shared,global,constant,param}
2268 Expand = consumeNVVMPtrAddrSpace(Name);
2269 else if (Name.consume_front("ptr."))
2270 // nvvm.ptr.{local,shared,global,constant,param}.to.gen
2271 Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(".to.gen");
2272 else if (Name.consume_front("ldg.global."))
2273 // nvvm.ldg.global.{i,p,f}
2274 Expand = (Name.starts_with("i.") || Name.starts_with("f.") ||
2275 Name.starts_with("p."));
2276 else
2277 Expand = StringSwitch<bool>(Name)
2278 .Case("barrier0", true)
2279 .Case("barrier.n", true)
2280 .Case("barrier.sync.cnt", true)
2281 .Case("barrier.sync", true)
2282 .Case("barrier", true)
2283 .Case("bar.sync", true)
2284 .Case("barrier0.popc", true)
2285 .Case("barrier0.and", true)
2286 .Case("barrier0.or", true)
2287 .Case("clz.ll", true)
2288 .Case("popc.ll", true)
2289 .Case("h2f", true)
2290 .Case("swap.lo.hi.b64", true)
2291 .Case("tanh.approx.f32", true)
2292 .Default(false);
2293
2294 if (Expand) {
2295 NewFn = nullptr;
2296 return true;
2297 }
2298 break; // No other 'nvvm.*'.
2299 }
2300 break;
2301 }
2302 case 'o':
2303 if (Name.starts_with("objectsize.")) {
2304 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
2305 if (F->arg_size() == 2 || F->arg_size() == 3) {
2306 rename(F);
2307 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2308 Intrinsic::objectsize, Tys);
2309 return true;
2310 }
2311 }
2312 break;
2313
2314 case 'p':
2315 if (Name.starts_with("ptr.annotation.") && F->arg_size() == 4) {
2316 rename(F);
2317 NewFn = Intrinsic::getOrInsertDeclaration(
2318 F->getParent(), Intrinsic::ptr_annotation,
2319 {F->arg_begin()->getType(), F->getArg(1)->getType()});
2320 return true;
2321 }
2322 break;
2323
2324 case 'r': {
2325 if (Name.consume_front("riscv.")) {
2326 Intrinsic::ID ID;
2327 ID = StringSwitch<Intrinsic::ID>(Name)
2328 .Case("aes32dsi", Intrinsic::riscv_aes32dsi)
2329 .Case("aes32dsmi", Intrinsic::riscv_aes32dsmi)
2330 .Case("aes32esi", Intrinsic::riscv_aes32esi)
2331 .Case("aes32esmi", Intrinsic::riscv_aes32esmi)
2332 .Default(Intrinsic::not_intrinsic);
2333 if (ID != Intrinsic::not_intrinsic) {
2334 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
2335 rename(F);
2336 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2337 return true;
2338 }
2339 break; // No other applicable upgrades.
2340 }
2341
2342 ID = StringSwitch<Intrinsic::ID>(Name)
2343 .StartsWith("sm4ks", Intrinsic::riscv_sm4ks)
2344 .StartsWith("sm4ed", Intrinsic::riscv_sm4ed)
2345 .Default(Intrinsic::not_intrinsic);
2346 if (ID != Intrinsic::not_intrinsic) {
2347 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
2348 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2349 rename(F);
2350 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2351 return true;
2352 }
2353 break; // No other applicable upgrades.
2354 }
2355
2356 ID = StringSwitch<Intrinsic::ID>(Name)
2357 .StartsWith("sha256sig0", Intrinsic::riscv_sha256sig0)
2358 .StartsWith("sha256sig1", Intrinsic::riscv_sha256sig1)
2359 .StartsWith("sha256sum0", Intrinsic::riscv_sha256sum0)
2360 .StartsWith("sha256sum1", Intrinsic::riscv_sha256sum1)
2361 .StartsWith("sm3p0", Intrinsic::riscv_sm3p0)
2362 .StartsWith("sm3p1", Intrinsic::riscv_sm3p1)
2363 .Default(Intrinsic::not_intrinsic);
2364 if (ID != Intrinsic::not_intrinsic) {
2365 if (F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2366 rename(F);
2367 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2368 return true;
2369 }
2370 break; // No other applicable upgrades.
2371 }
2372
2373 // Replace llvm.riscv.clmul with llvm.clmul.
2374 if (Name == "clmul.i32" || Name == "clmul.i64") {
2375 NewFn = Intrinsic::getOrInsertDeclaration(
2376 F->getParent(), Intrinsic::clmul, {F->getReturnType()});
2377 return true;
2378 }
2379
2380 break; // No other 'riscv.*' intrinsics
2381 }
2382 } break;
2383
2384 case 's':
2385 if (Name == "stackprotectorcheck") {
2386 NewFn = nullptr;
2387 return true;
2388 }
2389 if (Name.starts_with("strip.invariant.group")) {
2390 // For clang's usage it would be safe to just drop the
2391 // strip.invariant.group, but to be conservative replace with the
2392 // stronger launder.invariant.group instead.
2393 NewFn = Intrinsic::getOrInsertDeclaration(
2394 F->getParent(), Intrinsic::launder_invariant_group,
2395 F->getReturnType());
2396 return true;
2397 }
2398 break;
2399
2400 case 't':
2401 if (Name == "thread.pointer") {
2402 NewFn = Intrinsic::getOrInsertDeclaration(
2403 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
2404 return true;
2405 }
2406 break;
2407
2408 case 'v': {
2409 if (Name == "var.annotation" && F->arg_size() == 4) {
2410 rename(F);
2411 NewFn = Intrinsic::getOrInsertDeclaration(
2412 F->getParent(), Intrinsic::var_annotation,
2413 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
2414 return true;
2415 }
2416 if (Name.consume_front("vector.splice")) {
2417 if (Name.starts_with(".left") || Name.starts_with(".right"))
2418 break;
2419 return true;
2420 }
2421 if (shouldUpgradeVPIntrinsic(Name))
2422 return true;
2423 break;
2424 }
2425
2426 case 'w':
2427 if (Name.consume_front("wasm.")) {
2428 Intrinsic::ID ID =
2429 StringSwitch<Intrinsic::ID>(Name)
2430 .StartsWith("fma.", Intrinsic::wasm_relaxed_madd)
2431 .StartsWith("fms.", Intrinsic::wasm_relaxed_nmadd)
2432 .StartsWith("laneselect.", Intrinsic::wasm_relaxed_laneselect)
2433 .Default(Intrinsic::not_intrinsic);
2434 if (ID != Intrinsic::not_intrinsic) {
2435 rename(F);
2436 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
2437 F->getReturnType());
2438 return true;
2439 }
2440
2441 if (Name.consume_front("dot.i8x16.i7x16.")) {
2442 ID = StringSwitch<Intrinsic::ID>(Name)
2443 .Case("signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
2444 .Case("add.signed",
2445 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
2446 .Default(Intrinsic::not_intrinsic);
2447 if (ID != Intrinsic::not_intrinsic) {
2448 rename(F);
2449 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2450 return true;
2451 }
2452 break; // No other 'wasm.dot.i8x16.i7x16.*'.
2453 }
2454 break; // No other 'wasm.*'.
2455 }
2456 break;
2457
2458 case 'x':
2459 if (upgradeX86IntrinsicFunction(F, Name, NewFn))
2460 return true;
2461 }
2462
2463 auto *ST = dyn_cast<StructType>(F->getReturnType());
2464 if (ST && (!ST->isLiteral() || ST->isPacked()) &&
2465 F->getIntrinsicID() != Intrinsic::not_intrinsic) {
2466 // Replace return type with literal non-packed struct. Only do this for
2467 // intrinsics declared to return a struct, not for intrinsics with
2468 // overloaded return type, in which case the exact struct type will be
2469 // mangled into the name.
2470 if (Intrinsic::hasStructReturnType(F->getIntrinsicID())) {
2471 FunctionType *FT = F->getFunctionType();
2472 auto *NewST = StructType::get(ST->getContext(), ST->elements());
2473 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
2474 std::string Name = F->getName().str();
2475 rename(F);
2476 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(),
2477 Name, F->getParent());
2478
2479 // The new function may also need remangling.
2480 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn))
2481 NewFn = *Result;
2482 return true;
2483 }
2484 }
2485
2486 // Remangle our intrinsic since we upgrade the mangling
2487 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
2488 if (Result != std::nullopt) {
2489 NewFn = *Result;
2490 return true;
2491 }
2492
2493 if (upgradeIntrinsicWithDefaultArgs(F, NewFn))
2494 return true;
2495
2496 // This may not belong here. This function is effectively being overloaded
2497 // to both detect an intrinsic which needs upgrading, and to provide the
2498 // upgraded form of the intrinsic. We should perhaps have two separate
2499 // functions for this.
2500
2501 return false;
2502}
2503
2504bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
2505 bool CanUpgradeDebugIntrinsicsToRecords) {
2506 NewFn = nullptr;
2507 bool Upgraded =
2508 upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
2509
2510 // Upgrade intrinsic attributes. This does not change the function.
2511 if (NewFn)
2512 F = NewFn;
2513 if (Intrinsic::ID id = F->getIntrinsicID()) {
2514 // Only do this if the intrinsic signature is valid.
2515 SmallVector<Type *> OverloadTys;
2516 if (Intrinsic::isSignatureValid(id, F->getFunctionType(), OverloadTys))
2517 F->setAttributes(
2518 Intrinsic::getAttributes(F->getContext(), id, F->getFunctionType()));
2519 }
2520 return Upgraded;
2521}
2522
2523GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
2524 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
2525 GV->getName() == "llvm.global_dtors")) ||
2526 !GV->hasInitializer())
2527 return nullptr;
2528 ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
2529 if (!ATy)
2530 return nullptr;
2531 StructType *STy = dyn_cast<StructType>(ATy->getElementType());
2532 if (!STy || STy->getNumElements() != 2)
2533 return nullptr;
2534
2535 IRBuilder<> IRB(*GV->getParent());
2536 auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
2537 IRB.getPtrTy());
2538 Constant *Init = GV->getInitializer();
2539 unsigned N = Init->getNumOperands();
2540 std::vector<Constant *> NewCtors(N);
2541 for (unsigned i = 0; i != N; ++i) {
2542 auto Ctor = cast<Constant>(Init->getOperand(i));
2543 NewCtors[i] = ConstantStruct::get(EltTy, Ctor->getAggregateElement(0u),
2544 Ctor->getAggregateElement(1),
2545 ConstantPointerNull::get(IRB.getPtrTy()));
2546 }
2547 Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
2548
2549 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
2550 NewInit, GV->getName());
2551}
2552
2553// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
2554// to byte shuffles.
2555static Value *upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2556 unsigned Shift) {
2557 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2558 unsigned NumElts = ResultTy->getNumElements() * 8;
2559
2560 // Bitcast from a 64-bit element type to a byte element type.
2561 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2562 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2563
2564 // We'll be shuffling in zeroes.
2565 Value *Res = Constant::getNullValue(VecTy);
2566
2567 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2568 // we'll just return the zero vector.
2569 if (Shift < 16) {
2570 int Idxs[64];
2571 // 256/512-bit version is split into 2/4 16-byte lanes.
2572 for (unsigned l = 0; l != NumElts; l += 16)
2573 for (unsigned i = 0; i != 16; ++i) {
2574 unsigned Idx = NumElts + i - Shift;
2575 if (Idx < NumElts)
2576 Idx -= NumElts - 16; // end of lane, switch operand.
2577 Idxs[l + i] = Idx + l;
2578 }
2579
2580 Res = Builder.CreateShuffleVector(Res, Op, ArrayRef(Idxs, NumElts));
2581 }
2582
2583 // Bitcast back to a 64-bit element type.
2584 return Builder.CreateBitCast(Res, ResultTy, "cast");
2585}
2586
2587// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
2588// to byte shuffles.
2589static Value *upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2590 unsigned Shift) {
2591 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2592 unsigned NumElts = ResultTy->getNumElements() * 8;
2593
2594 // Bitcast from a 64-bit element type to a byte element type.
2595 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2596 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2597
2598 // We'll be shuffling in zeroes.
2599 Value *Res = Constant::getNullValue(VecTy);
2600
2601 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2602 // we'll just return the zero vector.
2603 if (Shift < 16) {
2604 int Idxs[64];
2605 // 256/512-bit version is split into 2/4 16-byte lanes.
2606 for (unsigned l = 0; l != NumElts; l += 16)
2607 for (unsigned i = 0; i != 16; ++i) {
2608 unsigned Idx = i + Shift;
2609 if (Idx >= 16)
2610 Idx += NumElts - 16; // end of lane, switch operand.
2611 Idxs[l + i] = Idx + l;
2612 }
2613
2614 Res = Builder.CreateShuffleVector(Op, Res, ArrayRef(Idxs, NumElts));
2615 }
2616
2617 // Bitcast back to a 64-bit element type.
2618 return Builder.CreateBitCast(Res, ResultTy, "cast");
2619}
2620
2621static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
2622 unsigned NumElts) {
2623 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2624 llvm::VectorType *MaskTy = FixedVectorType::get(
2625 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
2626 Mask = Builder.CreateBitCast(Mask, MaskTy);
2627
2628 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2629 // i8 and we need to extract down to the right number of elements.
2630 if (NumElts <= 4) {
2631 int Indices[4];
2632 for (unsigned i = 0; i != NumElts; ++i)
2633 Indices[i] = i;
2634 Mask = Builder.CreateShuffleVector(Mask, Mask, ArrayRef(Indices, NumElts),
2635 "extract");
2636 }
2637
2638 return Mask;
2639}
2640
2641static Value *emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2642 Value *Op1) {
2643 // If the mask is all ones just emit the first operation.
2644 if (const auto *C = dyn_cast<Constant>(Mask))
2645 if (C->isAllOnesValue())
2646 return Op0;
2647
2648 Mask = getX86MaskVec(Builder, Mask,
2649 cast<FixedVectorType>(Op0->getType())->getNumElements());
2650 return Builder.CreateSelect(Mask, Op0, Op1);
2651}
2652
2653static Value *emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2654 Value *Op1) {
2655 // If the mask is all ones just emit the first operation.
2656 if (const auto *C = dyn_cast<Constant>(Mask))
2657 if (C->isAllOnesValue())
2658 return Op0;
2659
2660 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
2661 Mask->getType()->getIntegerBitWidth());
2662 Mask = Builder.CreateBitCast(Mask, MaskTy);
2663 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
2664 return Builder.CreateSelect(Mask, Op0, Op1);
2665}
2666
2667// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2668// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2669// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2670static Value *upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
2671 Value *Op1, Value *Shift,
2672 Value *Passthru, Value *Mask,
2673 bool IsVALIGN) {
2674 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
2675
2676 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2677 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
2678 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
2679 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2680
2681 // Mask the immediate for VALIGN.
2682 if (IsVALIGN)
2683 ShiftVal &= (NumElts - 1);
2684
2685 // If palignr is shifting the pair of vectors more than the size of two
2686 // lanes, emit zero.
2687 if (ShiftVal >= 32)
2688 return llvm::Constant::getNullValue(Op0->getType());
2689
2690 // If palignr is shifting the pair of input vectors more than one lane,
2691 // but less than two lanes, convert to shifting in zeroes.
2692 if (ShiftVal > 16) {
2693 ShiftVal -= 16;
2694 Op1 = Op0;
2695 Op0 = llvm::Constant::getNullValue(Op0->getType());
2696 }
2697
2698 int Indices[64];
2699 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2700 for (unsigned l = 0; l < NumElts; l += 16) {
2701 for (unsigned i = 0; i != 16; ++i) {
2702 unsigned Idx = ShiftVal + i;
2703 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
2704 Idx += NumElts - 16; // End of lane, switch operand.
2705 Indices[l + i] = Idx + l;
2706 }
2707 }
2708
2709 Value *Align = Builder.CreateShuffleVector(
2710 Op1, Op0, ArrayRef(Indices, NumElts), "palignr");
2711
2712 return emitX86Select(Builder, Mask, Align, Passthru);
2713}
2714
2715static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
2716 bool ZeroMask, bool IndexForm) {
2717 Type *Ty = CI.getType();
2718 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2719 unsigned EltWidth = Ty->getScalarSizeInBits();
2720 bool IsFloat = Ty->isFPOrFPVectorTy();
2721 Intrinsic::ID IID;
2722 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2723 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2724 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2725 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2726 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2727 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2728 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2729 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2730 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2731 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2732 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2733 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2734 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2735 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2736 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2737 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2738 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2739 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2740 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2741 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2742 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2743 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2744 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2745 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2746 else if (VecWidth == 128 && EltWidth == 16)
2747 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2748 else if (VecWidth == 256 && EltWidth == 16)
2749 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2750 else if (VecWidth == 512 && EltWidth == 16)
2751 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2752 else if (VecWidth == 128 && EltWidth == 8)
2753 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2754 else if (VecWidth == 256 && EltWidth == 8)
2755 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2756 else if (VecWidth == 512 && EltWidth == 8)
2757 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2758 else
2759 llvm_unreachable("Unexpected intrinsic");
2760
2761 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
2762 CI.getArgOperand(2) };
2763
2764 // If this isn't index form we need to swap operand 0 and 1.
2765 if (!IndexForm)
2766 std::swap(Args[0], Args[1]);
2767
2768 Value *V = Builder.CreateIntrinsic(IID, Args);
2769 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2770 : Builder.CreateBitCast(CI.getArgOperand(1),
2771 Ty);
2772 return emitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
2773}
2774
2775static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
2776 Intrinsic::ID IID) {
2777 Type *Ty = CI.getType();
2778 Value *Op0 = CI.getOperand(0);
2779 Value *Op1 = CI.getOperand(1);
2780 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1});
2781
2782 if (CI.arg_size() == 4) { // For masked intrinsics.
2783 Value *VecSrc = CI.getOperand(2);
2784 Value *Mask = CI.getOperand(3);
2785 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2786 }
2787 return Res;
2788}
2789
2790static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
2791 bool IsRotateRight) {
2792 Type *Ty = CI.getType();
2793 Value *Src = CI.getArgOperand(0);
2794 Value *Amt = CI.getArgOperand(1);
2795
2796 // Amount may be scalar immediate, in which case create a splat vector.
2797 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2798 // we only care about the lowest log2 bits anyway.
2799 if (Amt->getType() != Ty) {
2800 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2801 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2802 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2803 }
2804
2805 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2806 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Src, Src, Amt});
2807
2808 if (CI.arg_size() == 4) { // For masked intrinsics.
2809 Value *VecSrc = CI.getOperand(2);
2810 Value *Mask = CI.getOperand(3);
2811 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2812 }
2813 return Res;
2814}
2815
2816static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
2817 bool IsSigned) {
2818 Type *Ty = CI.getType();
2819 Value *LHS = CI.getArgOperand(0);
2820 Value *RHS = CI.getArgOperand(1);
2821
2822 CmpInst::Predicate Pred;
2823 switch (Imm) {
2824 case 0x0:
2825 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
2826 break;
2827 case 0x1:
2828 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2829 break;
2830 case 0x2:
2831 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
2832 break;
2833 case 0x3:
2834 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
2835 break;
2836 case 0x4:
2837 Pred = ICmpInst::ICMP_EQ;
2838 break;
2839 case 0x5:
2840 Pred = ICmpInst::ICMP_NE;
2841 break;
2842 case 0x6:
2843 return Constant::getNullValue(Ty); // FALSE
2844 case 0x7:
2845 return Constant::getAllOnesValue(Ty); // TRUE
2846 default:
2847 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2848 }
2849
2850 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
2851 Value *Ext = Builder.CreateSExt(Cmp, Ty);
2852 return Ext;
2853}
2854
2855static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
2856 bool IsShiftRight, bool ZeroMask) {
2857 Type *Ty = CI.getType();
2858 Value *Op0 = CI.getArgOperand(0);
2859 Value *Op1 = CI.getArgOperand(1);
2860 Value *Amt = CI.getArgOperand(2);
2861
2862 if (IsShiftRight)
2863 std::swap(Op0, Op1);
2864
2865 // Amount may be scalar immediate, in which case create a splat vector.
2866 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2867 // we only care about the lowest log2 bits anyway.
2868 if (Amt->getType() != Ty) {
2869 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2870 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2871 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2872 }
2873
2874 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2875 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1, Amt});
2876
2877 unsigned NumArgs = CI.arg_size();
2878 if (NumArgs >= 4) { // For masked intrinsics.
2879 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
2880 ZeroMask ? ConstantAggregateZero::get(CI.getType()) :
2881 CI.getArgOperand(0);
2882 Value *Mask = CI.getOperand(NumArgs - 1);
2883 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2884 }
2885 return Res;
2886}
2887
2888static Value *upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data,
2889 Value *Mask, bool Aligned) {
2890 const Align Alignment =
2891 Aligned
2892 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2893 : Align(1);
2894
2895 // If the mask is all ones just emit a regular store.
2896 if (const auto *C = dyn_cast<Constant>(Mask))
2897 if (C->isAllOnesValue())
2898 return Builder.CreateAlignedStore(Data, Ptr, Alignment);
2899
2900 // Convert the mask from an integer type to a vector of i1.
2901 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
2902 Mask = getX86MaskVec(Builder, Mask, NumElts);
2903 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
2904}
2905
2906static Value *upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr,
2907 Value *Passthru, Value *Mask, bool Aligned) {
2908 Type *ValTy = Passthru->getType();
2909 const Align Alignment =
2910 Aligned
2911 ? Align(
2912 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
2913 8)
2914 : Align(1);
2915
2916 // If the mask is all ones just emit a regular store.
2917 if (const auto *C = dyn_cast<Constant>(Mask))
2918 if (C->isAllOnesValue())
2919 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
2920
2921 // Convert the mask from an integer type to a vector of i1.
2922 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
2923 Mask = getX86MaskVec(Builder, Mask, NumElts);
2924 return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
2925}
2926
2927static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2928 Type *Ty = CI.getType();
2929 Value *Op0 = CI.getArgOperand(0);
2930 Value *Res = Builder.CreateIntrinsic(Intrinsic::abs, Ty,
2931 {Op0, Builder.getInt1(false)});
2932 if (CI.arg_size() == 3)
2933 Res = emitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
2934 return Res;
2935}
2936
2937static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
2938 Type *Ty = CI.getType();
2939
2940 // Arguments have a vXi32 type so cast to vXi64.
2941 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
2942 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
2943
2944 if (IsSigned) {
2945 // Shift left then arithmetic shift right.
2946 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
2947 LHS = Builder.CreateShl(LHS, ShiftAmt);
2948 LHS = Builder.CreateAShr(LHS, ShiftAmt);
2949 RHS = Builder.CreateShl(RHS, ShiftAmt);
2950 RHS = Builder.CreateAShr(RHS, ShiftAmt);
2951 } else {
2952 // Clear the upper bits.
2953 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
2954 LHS = Builder.CreateAnd(LHS, Mask);
2955 RHS = Builder.CreateAnd(RHS, Mask);
2956 }
2957
2958 Value *Res = Builder.CreateMul(LHS, RHS);
2959
2960 if (CI.arg_size() == 4)
2961 Res = emitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
2962
2963 return Res;
2964}
2965
2966// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2967static Value *applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
2968 Value *Mask) {
2969 unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
2970 if (Mask) {
2971 const auto *C = dyn_cast<Constant>(Mask);
2972 if (!C || !C->isAllOnesValue())
2973 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
2974 }
2975
2976 if (NumElts < 8) {
2977 int Indices[8];
2978 for (unsigned i = 0; i != NumElts; ++i)
2979 Indices[i] = i;
2980 for (unsigned i = NumElts; i != 8; ++i)
2981 Indices[i] = NumElts + i % NumElts;
2982 Vec = Builder.CreateShuffleVector(Vec,
2983 Constant::getNullValue(Vec->getType()),
2984 Indices);
2985 }
2986 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
2987}
2988
2989static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
2990 unsigned CC, bool Signed) {
2991 Value *Op0 = CI.getArgOperand(0);
2992 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2993
2994 Value *Cmp;
2995 if (CC == 3) {
2996 Cmp = Constant::getNullValue(
2997 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2998 } else if (CC == 7) {
2999 Cmp = Constant::getAllOnesValue(
3000 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
3001 } else {
3002 ICmpInst::Predicate Pred;
3003 switch (CC) {
3004 default: llvm_unreachable("Unknown condition code");
3005 case 0: Pred = ICmpInst::ICMP_EQ; break;
3006 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
3007 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
3008 case 4: Pred = ICmpInst::ICMP_NE; break;
3009 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
3010 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
3011 }
3012 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
3013 }
3014
3015 Value *Mask = CI.getArgOperand(CI.arg_size() - 1);
3016
3017 return applyX86MaskOn1BitsVec(Builder, Cmp, Mask);
3018}
3019
3020// Replace a masked intrinsic with an older unmasked intrinsic.
3021static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
3022 Intrinsic::ID IID) {
3023 Value *Rep =
3024 Builder.CreateIntrinsic(IID, {CI.getArgOperand(0), CI.getArgOperand(1)});
3025 return emitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
3026}
3027
3028static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
3029 Value* A = CI.getArgOperand(0);
3030 Value* B = CI.getArgOperand(1);
3031 Value* Src = CI.getArgOperand(2);
3032 Value* Mask = CI.getArgOperand(3);
3033
3034 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
3035 Value* Cmp = Builder.CreateIsNotNull(AndNode);
3036 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
3037 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
3038 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
3039 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
3040}
3041
3042static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
3043 Value* Op = CI.getArgOperand(0);
3044 Type* ReturnOp = CI.getType();
3045 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
3046 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
3047 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
3048}
3049
3050// Replace intrinsic with unmasked version and a select.
3051static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
3052 CallBase &CI, Value *&Rep) {
3053 Name = Name.substr(12); // Remove avx512.mask.
3054
3055 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
3056 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
3057 Intrinsic::ID IID;
3058 if (Name.starts_with("max.p")) {
3059 if (VecWidth == 128 && EltWidth == 32)
3060 IID = Intrinsic::x86_sse_max_ps;
3061 else if (VecWidth == 128 && EltWidth == 64)
3062 IID = Intrinsic::x86_sse2_max_pd;
3063 else if (VecWidth == 256 && EltWidth == 32)
3064 IID = Intrinsic::x86_avx_max_ps_256;
3065 else if (VecWidth == 256 && EltWidth == 64)
3066 IID = Intrinsic::x86_avx_max_pd_256;
3067 else
3068 llvm_unreachable("Unexpected intrinsic");
3069 } else if (Name.starts_with("min.p")) {
3070 if (VecWidth == 128 && EltWidth == 32)
3071 IID = Intrinsic::x86_sse_min_ps;
3072 else if (VecWidth == 128 && EltWidth == 64)
3073 IID = Intrinsic::x86_sse2_min_pd;
3074 else if (VecWidth == 256 && EltWidth == 32)
3075 IID = Intrinsic::x86_avx_min_ps_256;
3076 else if (VecWidth == 256 && EltWidth == 64)
3077 IID = Intrinsic::x86_avx_min_pd_256;
3078 else
3079 llvm_unreachable("Unexpected intrinsic");
3080 } else if (Name.starts_with("pshuf.b.")) {
3081 if (VecWidth == 128)
3082 IID = Intrinsic::x86_ssse3_pshuf_b_128;
3083 else if (VecWidth == 256)
3084 IID = Intrinsic::x86_avx2_pshuf_b;
3085 else if (VecWidth == 512)
3086 IID = Intrinsic::x86_avx512_pshuf_b_512;
3087 else
3088 llvm_unreachable("Unexpected intrinsic");
3089 } else if (Name.starts_with("pmul.hr.sw.")) {
3090 if (VecWidth == 128)
3091 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
3092 else if (VecWidth == 256)
3093 IID = Intrinsic::x86_avx2_pmul_hr_sw;
3094 else if (VecWidth == 512)
3095 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
3096 else
3097 llvm_unreachable("Unexpected intrinsic");
3098 } else if (Name.starts_with("pmulh.w")) {
3099 assert((VecWidth == 128 || VecWidth == 256 || VecWidth == 512) &&
3100 "Unexpected intrinsic");
3101 Rep = upgradeX86BinaryIntrinsics(Builder, CI, Intrinsic::smulh);
3102 return true;
3103 } else if (Name.starts_with("pmulhu.w")) {
3104 assert((VecWidth == 128 || VecWidth == 256 || VecWidth == 512) &&
3105 "Unexpected intrinsic");
3106 Rep = upgradeX86BinaryIntrinsics(Builder, CI, Intrinsic::umulh);
3107 return true;
3108 } else if (Name.starts_with("pmaddw.d.")) {
3109 if (VecWidth == 128)
3110 IID = Intrinsic::x86_sse2_pmadd_wd;
3111 else if (VecWidth == 256)
3112 IID = Intrinsic::x86_avx2_pmadd_wd;
3113 else if (VecWidth == 512)
3114 IID = Intrinsic::x86_avx512_pmaddw_d_512;
3115 else
3116 llvm_unreachable("Unexpected intrinsic");
3117 } else if (Name.starts_with("pmaddubs.w.")) {
3118 if (VecWidth == 128)
3119 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
3120 else if (VecWidth == 256)
3121 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
3122 else if (VecWidth == 512)
3123 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
3124 else
3125 llvm_unreachable("Unexpected intrinsic");
3126 } else if (Name.starts_with("packsswb.")) {
3127 if (VecWidth == 128)
3128 IID = Intrinsic::x86_sse2_packsswb_128;
3129 else if (VecWidth == 256)
3130 IID = Intrinsic::x86_avx2_packsswb;
3131 else if (VecWidth == 512)
3132 IID = Intrinsic::x86_avx512_packsswb_512;
3133 else
3134 llvm_unreachable("Unexpected intrinsic");
3135 } else if (Name.starts_with("packssdw.")) {
3136 if (VecWidth == 128)
3137 IID = Intrinsic::x86_sse2_packssdw_128;
3138 else if (VecWidth == 256)
3139 IID = Intrinsic::x86_avx2_packssdw;
3140 else if (VecWidth == 512)
3141 IID = Intrinsic::x86_avx512_packssdw_512;
3142 else
3143 llvm_unreachable("Unexpected intrinsic");
3144 } else if (Name.starts_with("packuswb.")) {
3145 if (VecWidth == 128)
3146 IID = Intrinsic::x86_sse2_packuswb_128;
3147 else if (VecWidth == 256)
3148 IID = Intrinsic::x86_avx2_packuswb;
3149 else if (VecWidth == 512)
3150 IID = Intrinsic::x86_avx512_packuswb_512;
3151 else
3152 llvm_unreachable("Unexpected intrinsic");
3153 } else if (Name.starts_with("packusdw.")) {
3154 if (VecWidth == 128)
3155 IID = Intrinsic::x86_sse41_packusdw;
3156 else if (VecWidth == 256)
3157 IID = Intrinsic::x86_avx2_packusdw;
3158 else if (VecWidth == 512)
3159 IID = Intrinsic::x86_avx512_packusdw_512;
3160 else
3161 llvm_unreachable("Unexpected intrinsic");
3162 } else if (Name.starts_with("vpermilvar.")) {
3163 if (VecWidth == 128 && EltWidth == 32)
3164 IID = Intrinsic::x86_avx_vpermilvar_ps;
3165 else if (VecWidth == 128 && EltWidth == 64)
3166 IID = Intrinsic::x86_avx_vpermilvar_pd;
3167 else if (VecWidth == 256 && EltWidth == 32)
3168 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
3169 else if (VecWidth == 256 && EltWidth == 64)
3170 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
3171 else if (VecWidth == 512 && EltWidth == 32)
3172 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
3173 else if (VecWidth == 512 && EltWidth == 64)
3174 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
3175 else
3176 llvm_unreachable("Unexpected intrinsic");
3177 } else if (Name == "cvtpd2dq.256") {
3178 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
3179 } else if (Name == "cvtpd2ps.256") {
3180 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
3181 } else if (Name == "cvttpd2dq.256") {
3182 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
3183 } else if (Name == "cvttps2dq.128") {
3184 IID = Intrinsic::x86_sse2_cvttps2dq;
3185 } else if (Name == "cvttps2dq.256") {
3186 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
3187 } else if (Name.starts_with("permvar.")) {
3188 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
3189 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
3190 IID = Intrinsic::x86_avx2_permps;
3191 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
3192 IID = Intrinsic::x86_avx2_permd;
3193 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
3194 IID = Intrinsic::x86_avx512_permvar_df_256;
3195 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
3196 IID = Intrinsic::x86_avx512_permvar_di_256;
3197 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
3198 IID = Intrinsic::x86_avx512_permvar_sf_512;
3199 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
3200 IID = Intrinsic::x86_avx512_permvar_si_512;
3201 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
3202 IID = Intrinsic::x86_avx512_permvar_df_512;
3203 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
3204 IID = Intrinsic::x86_avx512_permvar_di_512;
3205 else if (VecWidth == 128 && EltWidth == 16)
3206 IID = Intrinsic::x86_avx512_permvar_hi_128;
3207 else if (VecWidth == 256 && EltWidth == 16)
3208 IID = Intrinsic::x86_avx512_permvar_hi_256;
3209 else if (VecWidth == 512 && EltWidth == 16)
3210 IID = Intrinsic::x86_avx512_permvar_hi_512;
3211 else if (VecWidth == 128 && EltWidth == 8)
3212 IID = Intrinsic::x86_avx512_permvar_qi_128;
3213 else if (VecWidth == 256 && EltWidth == 8)
3214 IID = Intrinsic::x86_avx512_permvar_qi_256;
3215 else if (VecWidth == 512 && EltWidth == 8)
3216 IID = Intrinsic::x86_avx512_permvar_qi_512;
3217 else
3218 llvm_unreachable("Unexpected intrinsic");
3219 } else if (Name.starts_with("dbpsadbw.")) {
3220 if (VecWidth == 128)
3221 IID = Intrinsic::x86_avx512_dbpsadbw_128;
3222 else if (VecWidth == 256)
3223 IID = Intrinsic::x86_avx512_dbpsadbw_256;
3224 else if (VecWidth == 512)
3225 IID = Intrinsic::x86_avx512_dbpsadbw_512;
3226 else
3227 llvm_unreachable("Unexpected intrinsic");
3228 } else if (Name.starts_with("pmultishift.qb.")) {
3229 if (VecWidth == 128)
3230 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
3231 else if (VecWidth == 256)
3232 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
3233 else if (VecWidth == 512)
3234 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
3235 else
3236 llvm_unreachable("Unexpected intrinsic");
3237 } else if (Name.starts_with("conflict.")) {
3238 if (Name[9] == 'd' && VecWidth == 128)
3239 IID = Intrinsic::x86_avx512_conflict_d_128;
3240 else if (Name[9] == 'd' && VecWidth == 256)
3241 IID = Intrinsic::x86_avx512_conflict_d_256;
3242 else if (Name[9] == 'd' && VecWidth == 512)
3243 IID = Intrinsic::x86_avx512_conflict_d_512;
3244 else if (Name[9] == 'q' && VecWidth == 128)
3245 IID = Intrinsic::x86_avx512_conflict_q_128;
3246 else if (Name[9] == 'q' && VecWidth == 256)
3247 IID = Intrinsic::x86_avx512_conflict_q_256;
3248 else if (Name[9] == 'q' && VecWidth == 512)
3249 IID = Intrinsic::x86_avx512_conflict_q_512;
3250 else
3251 llvm_unreachable("Unexpected intrinsic");
3252 } else if (Name.starts_with("pavg.")) {
3253 if (Name[5] == 'b' && VecWidth == 128)
3254 IID = Intrinsic::x86_sse2_pavg_b;
3255 else if (Name[5] == 'b' && VecWidth == 256)
3256 IID = Intrinsic::x86_avx2_pavg_b;
3257 else if (Name[5] == 'b' && VecWidth == 512)
3258 IID = Intrinsic::x86_avx512_pavg_b_512;
3259 else if (Name[5] == 'w' && VecWidth == 128)
3260 IID = Intrinsic::x86_sse2_pavg_w;
3261 else if (Name[5] == 'w' && VecWidth == 256)
3262 IID = Intrinsic::x86_avx2_pavg_w;
3263 else if (Name[5] == 'w' && VecWidth == 512)
3264 IID = Intrinsic::x86_avx512_pavg_w_512;
3265 else
3266 llvm_unreachable("Unexpected intrinsic");
3267 } else
3268 return false;
3269
3270 SmallVector<Value *, 4> Args(CI.args());
3271 Args.pop_back();
3272 Args.pop_back();
3273 Rep = Builder.CreateIntrinsic(IID, Args);
3274 unsigned NumArgs = CI.arg_size();
3275 Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
3276 CI.getArgOperand(NumArgs - 2));
3277 return true;
3278}
3279
3280/// Upgrade comment in call to inline asm that represents an objc retain release
3281/// marker.
3282void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
3283 size_t Pos;
3284 if (AsmStr->find("mov\tfp") == 0 &&
3285 AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
3286 (Pos = AsmStr->find("# marker")) != std::string::npos) {
3287 AsmStr->replace(Pos, 1, ";");
3288 }
3289}
3290
3291static Value *upgradeNVVMFPArithCall(IRBuilder<> &Builder, CallBase *CI,
3292 StringRef Name,
3293 const Intrinsic::ID (&IIDs)[2][3]) {
3294 auto Result = getNVVMFPArithUpgrade(Name, IIDs);
3295 assert(Result && "unsupported nvvm.{add,mul,fma}.* intrinsic");
3296 auto [IID, RoundingMode] = *Result;
3297
3298 // The legacy bf16 variants used integer storage types for the bf16
3299 // operands; bitcast them to the floating-point types of the new
3300 // overloaded intrinsic.
3301 Type *Ty = CI->getType();
3302 const bool IsLegacyBF16Storage = Ty->getScalarType()->isIntegerTy();
3303 if (IsLegacyBF16Storage)
3304 Ty = Name.ends_with("bf16x2")
3305 ? FixedVectorType::get(Builder.getBFloatTy(), 2)
3306 : static_cast<Type *>(Builder.getBFloatTy());
3307
3308 SmallVector<Value *, 4> Args;
3309 for (Value *Arg : CI->args())
3310 Args.push_back(IsLegacyBF16Storage ? Builder.CreateBitCast(Arg, Ty) : Arg);
3311 Args.push_back(Builder.getInt32(static_cast<int>(RoundingMode)));
3312
3313 Value *Rep = Builder.CreateIntrinsic(Ty, IID, Args);
3314 return IsLegacyBF16Storage ? Builder.CreateBitCast(Rep, CI->getType()) : Rep;
3315}
3316
3317static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
3318 Function *F, IRBuilder<> &Builder) {
3319 Value *Rep = nullptr;
3320
3321 if (Name == "abs.i" || Name == "abs.ll") {
3322 Value *Arg = CI->getArgOperand(0);
3323 Rep = Builder.CreateIntrinsic(Intrinsic::abs, {Arg->getType()},
3324 {Arg, Builder.getTrue()},
3325 /*FMFSource=*/nullptr, "abs");
3326 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
3327 Type *Ty = (Name == "abs.bf16")
3328 ? Builder.getBFloatTy()
3329 : FixedVectorType::get(Builder.getBFloatTy(), 2);
3330 Value *Arg = Builder.CreateBitCast(CI->getArgOperand(0), Ty);
3331 Value *Abs = Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs, Arg);
3332 Rep = Builder.CreateBitCast(Abs, CI->getType());
3333 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
3334 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
3335 : Intrinsic::nvvm_fabs;
3336 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3337 } else if (Name.consume_front("add.")) {
3338 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
3339 Rep = upgradeNVVMFPArithCall(Builder, CI, Name, NVVMFAddIIDs);
3340 } else if (Name.consume_front("mul.")) {
3341 // nvvm.mul.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
3342 Rep = upgradeNVVMFPArithCall(Builder, CI, Name, NVVMFMulIIDs);
3343 } else if (Name.consume_front("ex2.approx.")) {
3344 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
3345 Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
3346 : Intrinsic::nvvm_ex2_approx;
3347 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3348 } else if (Name.consume_front("fma.")) {
3349 // nvvm.fma.<rnd>{.ftz}{.sat|.relu}.{f,d,f16,f16x2,bf16,bf16x2}
3350 Rep = upgradeNVVMFPArithCall(Builder, CI, Name, NVVMFMAIIDs);
3351 } else if (Name.starts_with("atomic.load.add.f32.p") ||
3352 Name.starts_with("atomic.load.add.f64.p")) {
3353 Value *Ptr = CI->getArgOperand(0);
3354 Value *Val = CI->getArgOperand(1);
3355 Rep = Builder.CreateAtomicRMW(
3356 AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3357 CI->getContext().getOrInsertSyncScopeID("device"));
3358 cast<Instruction>(Rep)->setMetadata(
3359 LLVMContext::MD_atomic_ignore_denormal_mode,
3360 MDNode::get(CI->getContext(), {}));
3361 // The default scope for atomic.load.* intrinsics is device
3362 // (= gpu scope in ptx), but the default LLVM atomic scope is
3363 // "system"
3364 } else if (Name.starts_with("atomic.load.inc.32.p") ||
3365 Name.starts_with("atomic.load.dec.32.p")) {
3366 Value *Ptr = CI->getArgOperand(0);
3367 Value *Val = CI->getArgOperand(1);
3368 auto Op = Name.starts_with("atomic.load.inc") ? AtomicRMWInst::UIncWrap
3369 : AtomicRMWInst::UDecWrap;
3370 Rep = Builder.CreateAtomicRMW(
3371 Op, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3372 CI->getContext().getOrInsertSyncScopeID("device"));
3373 // See comment above.
3374 } else if (Name.starts_with("atomic.") && Name.contains(".gen.")) {
3375 // nvvm.atomic.{op}.gen.{i,f}.{cta,sys} -> atomicrmw / cmpxchg.
3376 StringRef Op = Name.substr(StringRef("atomic.").size());
3377 Value *Ptr = CI->getArgOperand(0);
3378 Value *Val = CI->getArgOperand(1);
3379 SyncScope::ID SSID = CI->getContext().getOrInsertSyncScopeID(
3380 Op.contains(".cta.") ? "block" : "");
3381 if (Op.starts_with("cas.")) {
3382 Value *New = CI->getArgOperand(2);
3383 Value *Pair = Builder.CreateAtomicCmpXchg(
3384 Ptr, Val, New, MaybeAlign(), AtomicOrdering::Monotonic,
3385 AtomicOrdering::Monotonic, SSID);
3386 Rep = Builder.CreateExtractValue(Pair, 0);
3387 } else {
3388 // Note we don't upgrade anything to AtomicRMWInst::UMin/UMax. This is
3389 // because we were actually missing those intrinsics!
3390 AtomicRMWInst::BinOp BinOp =
3391 StringSwitch<AtomicRMWInst::BinOp>(Op)
3392 .StartsWith("add.gen.f", AtomicRMWInst::FAdd)
3393 .StartsWith("add.gen.i", AtomicRMWInst::Add)
3394 .StartsWith("exch.", AtomicRMWInst::Xchg)
3395 .StartsWith("max.", AtomicRMWInst::Max)
3396 .StartsWith("min.", AtomicRMWInst::Min)
3397 .StartsWith("inc.", AtomicRMWInst::UIncWrap)
3398 .StartsWith("dec.", AtomicRMWInst::UDecWrap)
3399 .StartsWith("and.", AtomicRMWInst::And)
3400 .StartsWith("or.", AtomicRMWInst::Or)
3401 .StartsWith("xor.", AtomicRMWInst::Xor)
3402 .Default(AtomicRMWInst::BAD_BINOP);
3403 assert(BinOp != AtomicRMWInst::BAD_BINOP &&
3404 "unexpected nvvm scoped atomic intrinsic");
3405 Rep = Builder.CreateAtomicRMW(BinOp, Ptr, Val, MaybeAlign(),
3406 AtomicOrdering::Monotonic, SSID);
3407 if (BinOp == AtomicRMWInst::FAdd) {
3408 cast<Instruction>(Rep)->setMetadata(
3409 LLVMContext::MD_atomic_ignore_denormal_mode,
3410 MDNode::get(CI->getContext(), {}));
3411 }
3412 }
3413 } else if (Name == "clz.ll") {
3414 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
3415 Value *Arg = CI->getArgOperand(0);
3416 Value *Ctlz = Builder.CreateIntrinsic(Intrinsic::ctlz, {Arg->getType()},
3417 {Arg, Builder.getFalse()},
3418 /*FMFSource=*/nullptr, "ctlz");
3419 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
3420 } else if (Name == "popc.ll") {
3421 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
3422 // i64.
3423 Value *Arg = CI->getArgOperand(0);
3424 Value *Popc = Builder.CreateIntrinsic(Intrinsic::ctpop, {Arg->getType()},
3425 Arg, /*FMFSource=*/nullptr, "ctpop");
3426 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
3427 } else if (Name == "h2f") {
3428 Value *Cast =
3429 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
3430 Rep = Builder.CreateFPExt(Cast, Builder.getFloatTy());
3431 } else if (Name.consume_front("bitcast.") &&
3432 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
3433 Name == "d2ll")) {
3434 Rep = Builder.CreateBitCast(CI->getArgOperand(0), CI->getType());
3435 } else if (Name == "rotate.b32") {
3436 Value *Arg = CI->getOperand(0);
3437 Value *ShiftAmt = CI->getOperand(1);
3438 Rep = Builder.CreateIntrinsic(Builder.getInt32Ty(), Intrinsic::fshl,
3439 {Arg, Arg, ShiftAmt});
3440 } else if (Name == "rotate.b64") {
3441 Type *Int64Ty = Builder.getInt64Ty();
3442 Value *Arg = CI->getOperand(0);
3443 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3444 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3445 {Arg, Arg, ZExtShiftAmt});
3446 } else if (Name == "rotate.right.b64") {
3447 Type *Int64Ty = Builder.getInt64Ty();
3448 Value *Arg = CI->getOperand(0);
3449 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3450 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshr,
3451 {Arg, Arg, ZExtShiftAmt});
3452 } else if (Name == "swap.lo.hi.b64") {
3453 Type *Int64Ty = Builder.getInt64Ty();
3454 Value *Arg = CI->getOperand(0);
3455 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3456 {Arg, Arg, Builder.getInt64(32)});
3457 } else if ((Name.consume_front("ptr.gen.to.") &&
3458 consumeNVVMPtrAddrSpace(Name)) ||
3459 (Name.consume_front("ptr.") && consumeNVVMPtrAddrSpace(Name) &&
3460 Name.starts_with(".to.gen"))) {
3461 Rep = Builder.CreateAddrSpaceCast(CI->getArgOperand(0), CI->getType());
3462 } else if (Name.consume_front("ldg.global")) {
3463 Value *Ptr = CI->getArgOperand(0);
3464 Align PtrAlign = cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue();
3465 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
3466 Value *ASC = Builder.CreateAddrSpaceCast(Ptr, Builder.getPtrTy(1));
3467 Instruction *LD = Builder.CreateAlignedLoad(CI->getType(), ASC, PtrAlign);
3468 MDNode *MD = MDNode::get(Builder.getContext(), {});
3469 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
3470 return LD;
3471 } else if (Name == "tanh.approx.f32") {
3472 // nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
3473 FastMathFlags FMF;
3474 FMF.setApproxFunc();
3475 Rep = Builder.CreateUnaryIntrinsic(Intrinsic::tanh, CI->getArgOperand(0),
3476 FMF);
3477 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {
3478 Value *Arg =
3479 Name.ends_with('0') ? Builder.getInt32(0) : CI->getArgOperand(0);
3480 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all,
3481 {}, {Arg});
3482 } else if (Name == "barrier") {
3483 Rep = Builder.CreateIntrinsic(
3484 Intrinsic::nvvm_barrier_cta_sync_aligned_count, {},
3485 {CI->getArgOperand(0), CI->getArgOperand(1)});
3486 } else if (Name == "barrier.sync") {
3487 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_all, {},
3488 {CI->getArgOperand(0)});
3489 } else if (Name == "barrier.sync.cnt") {
3490 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_count, {},
3491 {CI->getArgOperand(0), CI->getArgOperand(1)});
3492 } else if (Name == "barrier0.popc" || Name == "barrier0.and" ||
3493 Name == "barrier0.or") {
3494 Value *C = CI->getArgOperand(0);
3495 C = Builder.CreateICmpNE(C, Builder.getInt32(0));
3496
3497 Intrinsic::ID IID =
3498 StringSwitch<Intrinsic::ID>(Name)
3499 .Case("barrier0.popc",
3500 Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
3501 .Case("barrier0.and",
3502 Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
3503 .Case("barrier0.or",
3504 Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
3505 Value *Bar = Builder.CreateIntrinsic(IID, {}, {Builder.getInt32(0), C});
3506 Rep = Builder.CreateZExt(Bar, CI->getType());
3507 } else {
3508 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
3509 if (IID != Intrinsic::not_intrinsic &&
3510 isLegacyNVPTXBF16IntSignature(F, IID)) {
3511 rename(F);
3512 Function *NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
3513 SmallVector<Value *, 2> Args;
3514 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
3515 Value *Arg = CI->getArgOperand(I);
3516 Type *OldType = Arg->getType();
3517 Type *NewType = NewFn->getArg(I)->getType();
3518 Args.push_back(
3519 (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
3520 ? Builder.CreateBitCast(Arg, NewType)
3521 : Arg);
3522 }
3523 Rep = Builder.CreateCall(NewFn, Args);
3524 if (F->getReturnType()->isIntegerTy())
3525 Rep = Builder.CreateBitCast(Rep, F->getReturnType());
3526 }
3527 }
3528
3529 return Rep;
3530}
3531
3532static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
3533 IRBuilder<> &Builder) {
3534 LLVMContext &C = F->getContext();
3535 Value *Rep = nullptr;
3536
3537 if (Name.starts_with("sse4a.movnt.")) {
3538 SmallVector<Metadata *, 1> Elts;
3539 Elts.push_back(
3540 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3541 MDNode *Node = MDNode::get(C, Elts);
3542
3543 Value *Arg0 = CI->getArgOperand(0);
3544 Value *Arg1 = CI->getArgOperand(1);
3545
3546 // Nontemporal (unaligned) store of the 0'th element of the float/double
3547 // vector.
3548 Value *Extract =
3549 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
3550
3551 StoreInst *SI = Builder.CreateAlignedStore(Extract, Arg0, Align(1));
3552 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3553 } else if (Name.starts_with("avx.movnt.") ||
3554 Name.starts_with("avx512.storent.")) {
3555 SmallVector<Metadata *, 1> Elts;
3556 Elts.push_back(
3557 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3558 MDNode *Node = MDNode::get(C, Elts);
3559
3560 Value *Arg0 = CI->getArgOperand(0);
3561 Value *Arg1 = CI->getArgOperand(1);
3562
3563 StoreInst *SI = Builder.CreateAlignedStore(
3564 Arg1, Arg0,
3565 Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3566 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3567 } else if (Name == "sse2.storel.dq") {
3568 Value *Arg0 = CI->getArgOperand(0);
3569 Value *Arg1 = CI->getArgOperand(1);
3570
3571 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
3572 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
3573 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
3574 Builder.CreateAlignedStore(Elt, Arg0, Align(1));
3575 } else if (Name.starts_with("sse.storeu.") ||
3576 Name.starts_with("sse2.storeu.") ||
3577 Name.starts_with("avx.storeu.")) {
3578 Value *Arg0 = CI->getArgOperand(0);
3579 Value *Arg1 = CI->getArgOperand(1);
3580 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
3581 } else if (Name == "avx512.mask.store.ss") {
3582 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
3583 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3584 Mask, false);
3585 } else if (Name.starts_with("avx512.mask.store")) {
3586 // "avx512.mask.storeu." or "avx512.mask.store."
3587 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
3588 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3589 CI->getArgOperand(2), Aligned);
3590 } else if (Name.starts_with("sse2.pcmp") || Name.starts_with("avx2.pcmp")) {
3591 // Upgrade packed integer vector compare intrinsics to compare instructions.
3592 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
3593 bool CmpEq = Name[9] == 'e';
3594 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
3595 CI->getArgOperand(0), CI->getArgOperand(1));
3596 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
3597 } else if (Name.starts_with("avx512.broadcastm")) {
3598 Type *ExtTy = Type::getInt32Ty(C);
3599 if (CI->getOperand(0)->getType()->isIntegerTy(8))
3600 ExtTy = Type::getInt64Ty(C);
3601 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
3602 ExtTy->getPrimitiveSizeInBits();
3603 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
3604 Rep = Builder.CreateVectorSplat(NumElts, Rep);
3605 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
3606 Value *Vec = CI->getArgOperand(0);
3607 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
3608 Elt0 = Builder.CreateIntrinsic(Intrinsic::sqrt, Elt0->getType(), Elt0);
3609 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
3610 } else if (Name.starts_with("avx.sqrt.p") ||
3611 Name.starts_with("sse2.sqrt.p") ||
3612 Name.starts_with("sse.sqrt.p")) {
3613 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3614 {CI->getArgOperand(0)});
3615 } else if (Name.starts_with("avx512.mask.sqrt.p")) {
3616 if (CI->arg_size() == 4 &&
3617 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3618 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3619 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
3620 : Intrinsic::x86_avx512_sqrt_pd_512;
3621
3622 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(3)};
3623 Rep = Builder.CreateIntrinsic(IID, Args);
3624 } else {
3625 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3626 {CI->getArgOperand(0)});
3627 }
3628 Rep =
3629 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3630 } else if (Name.starts_with("avx512.ptestm") ||
3631 Name.starts_with("avx512.ptestnm")) {
3632 Value *Op0 = CI->getArgOperand(0);
3633 Value *Op1 = CI->getArgOperand(1);
3634 Value *Mask = CI->getArgOperand(2);
3635 Rep = Builder.CreateAnd(Op0, Op1);
3636 llvm::Type *Ty = Op0->getType();
3637 Value *Zero = llvm::Constant::getNullValue(Ty);
3638 ICmpInst::Predicate Pred = Name.starts_with("avx512.ptestm")
3639 ? ICmpInst::ICMP_NE
3640 : ICmpInst::ICMP_EQ;
3641 Rep = Builder.CreateICmp(Pred, Rep, Zero);
3642 Rep = applyX86MaskOn1BitsVec(Builder, Rep, Mask);
3643 } else if (Name.starts_with("avx512.mask.pbroadcast")) {
3644 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
3645 ->getNumElements();
3646 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
3647 Rep =
3648 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3649 } else if (Name.starts_with("avx512.kunpck")) {
3650 unsigned NumElts = CI->getType()->getScalarSizeInBits();
3651 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
3652 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
3653 int Indices[64];
3654 for (unsigned i = 0; i != NumElts; ++i)
3655 Indices[i] = i;
3656
3657 // First extract half of each vector. This gives better codegen than
3658 // doing it in a single shuffle.
3659 LHS = Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));
3660 RHS = Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));
3661 // Concat the vectors.
3662 // NOTE: Operands have to be swapped to match intrinsic definition.
3663 Rep = Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));
3664 Rep = Builder.CreateBitCast(Rep, CI->getType());
3665 } else if (Name == "avx512.kand.w") {
3666 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3667 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3668 Rep = Builder.CreateAnd(LHS, RHS);
3669 Rep = Builder.CreateBitCast(Rep, CI->getType());
3670 } else if (Name == "avx512.kandn.w") {
3671 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3672 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3673 LHS = Builder.CreateNot(LHS);
3674 Rep = Builder.CreateAnd(LHS, RHS);
3675 Rep = Builder.CreateBitCast(Rep, CI->getType());
3676 } else if (Name == "avx512.kor.w") {
3677 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3678 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3679 Rep = Builder.CreateOr(LHS, RHS);
3680 Rep = Builder.CreateBitCast(Rep, CI->getType());
3681 } else if (Name == "avx512.kxor.w") {
3682 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3683 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3684 Rep = Builder.CreateXor(LHS, RHS);
3685 Rep = Builder.CreateBitCast(Rep, CI->getType());
3686 } else if (Name == "avx512.kxnor.w") {
3687 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3688 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3689 LHS = Builder.CreateNot(LHS);
3690 Rep = Builder.CreateXor(LHS, RHS);
3691 Rep = Builder.CreateBitCast(Rep, CI->getType());
3692 } else if (Name == "avx512.knot.w") {
3693 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3694 Rep = Builder.CreateNot(Rep);
3695 Rep = Builder.CreateBitCast(Rep, CI->getType());
3696 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
3697 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3698 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3699 Rep = Builder.CreateOr(LHS, RHS);
3700 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
3701 Value *C;
3702 if (Name[14] == 'c')
3703 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
3704 else
3705 C = ConstantInt::getNullValue(Builder.getInt16Ty());
3706 Rep = Builder.CreateICmpEQ(Rep, C);
3707 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
3708 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
3709 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
3710 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
3711 Name == "sse.div.ss" || Name == "sse2.div.sd") {
3712 Type *I32Ty = Type::getInt32Ty(C);
3713 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
3714 ConstantInt::get(I32Ty, 0));
3715 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
3716 ConstantInt::get(I32Ty, 0));
3717 Value *EltOp;
3718 if (Name.contains(".add."))
3719 EltOp = Builder.CreateFAdd(Elt0, Elt1);
3720 else if (Name.contains(".sub."))
3721 EltOp = Builder.CreateFSub(Elt0, Elt1);
3722 else if (Name.contains(".mul."))
3723 EltOp = Builder.CreateFMul(Elt0, Elt1);
3724 else
3725 EltOp = Builder.CreateFDiv(Elt0, Elt1);
3726 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
3727 ConstantInt::get(I32Ty, 0));
3728 } else if (Name.starts_with("avx512.mask.pcmp")) {
3729 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3730 bool CmpEq = Name[16] == 'e';
3731 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
3732 } else if (Name.starts_with("avx512.mask.vpshufbitqmb.")) {
3733 Type *OpTy = CI->getArgOperand(0)->getType();
3734 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3735 Intrinsic::ID IID;
3736 switch (VecWidth) {
3737 default:
3738 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3739 break;
3740 case 128:
3741 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3742 break;
3743 case 256:
3744 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3745 break;
3746 case 512:
3747 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3748 break;
3749 }
3750
3751 Rep =
3752 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3753 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3754 } else if (Name.starts_with("avx512.mask.fpclass.p")) {
3755 Type *OpTy = CI->getArgOperand(0)->getType();
3756 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3757 unsigned EltWidth = OpTy->getScalarSizeInBits();
3758 Intrinsic::ID IID;
3759 if (VecWidth == 128 && EltWidth == 32)
3760 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3761 else if (VecWidth == 256 && EltWidth == 32)
3762 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3763 else if (VecWidth == 512 && EltWidth == 32)
3764 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3765 else if (VecWidth == 128 && EltWidth == 64)
3766 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3767 else if (VecWidth == 256 && EltWidth == 64)
3768 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3769 else if (VecWidth == 512 && EltWidth == 64)
3770 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3771 else
3772 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3773
3774 Rep =
3775 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3776 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3777 } else if (Name.starts_with("avx512.cmp.p")) {
3778 SmallVector<Value *, 4> Args(CI->args());
3779 Type *OpTy = Args[0]->getType();
3780 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3781 unsigned EltWidth = OpTy->getScalarSizeInBits();
3782 Intrinsic::ID IID;
3783 if (VecWidth == 128 && EltWidth == 32)
3784 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3785 else if (VecWidth == 256 && EltWidth == 32)
3786 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3787 else if (VecWidth == 512 && EltWidth == 32)
3788 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3789 else if (VecWidth == 128 && EltWidth == 64)
3790 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3791 else if (VecWidth == 256 && EltWidth == 64)
3792 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3793 else if (VecWidth == 512 && EltWidth == 64)
3794 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3795 else
3796 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3797
3798 Value *Mask = Constant::getAllOnesValue(CI->getType());
3799 if (VecWidth == 512)
3800 std::swap(Mask, Args.back());
3801 Args.push_back(Mask);
3802
3803 Rep = Builder.CreateIntrinsic(IID, Args);
3804 } else if (Name.starts_with("avx512.mask.cmp.")) {
3805 // Integer compare intrinsics.
3806 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3807 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
3808 } else if (Name.starts_with("avx512.mask.ucmp.")) {
3809 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3810 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
3811 } else if (Name.starts_with("avx512.cvtb2mask.") ||
3812 Name.starts_with("avx512.cvtw2mask.") ||
3813 Name.starts_with("avx512.cvtd2mask.") ||
3814 Name.starts_with("avx512.cvtq2mask.")) {
3815 Value *Op = CI->getArgOperand(0);
3816 Value *Zero = llvm::Constant::getNullValue(Op->getType());
3817 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
3818 Rep = applyX86MaskOn1BitsVec(Builder, Rep, nullptr);
3819 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
3820 Name == "ssse3.pabs.d.128" || Name.starts_with("avx2.pabs") ||
3821 Name.starts_with("avx512.mask.pabs")) {
3822 Rep = upgradeAbs(Builder, *CI);
3823 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
3824 Name == "sse41.pmaxsd" || Name.starts_with("avx2.pmaxs") ||
3825 Name.starts_with("avx512.mask.pmaxs")) {
3826 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
3827 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
3828 Name == "sse41.pmaxud" || Name.starts_with("avx2.pmaxu") ||
3829 Name.starts_with("avx512.mask.pmaxu")) {
3830 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
3831 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
3832 Name == "sse41.pminsd" || Name.starts_with("avx2.pmins") ||
3833 Name.starts_with("avx512.mask.pmins")) {
3834 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
3835 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
3836 Name == "sse41.pminud" || Name.starts_with("avx2.pminu") ||
3837 Name.starts_with("avx512.mask.pminu")) {
3838 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
3839 } else if (Name == "sse2.pmulh.w" || Name.starts_with("avx2.pmulh.w") ||
3840 Name.starts_with("avx512.pmulh.w")) {
3841 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smulh);
3842 } else if (Name == "sse2.pmulhu.w" || Name.starts_with("avx2.pmulhu.w") ||
3843 Name.starts_with("avx512.pmulhu.w")) {
3844 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umulh);
3845 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
3846 Name == "avx512.pmulu.dq.512" ||
3847 Name.starts_with("avx512.mask.pmulu.dq.")) {
3848 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ false);
3849 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
3850 Name == "avx512.pmul.dq.512" ||
3851 Name.starts_with("avx512.mask.pmul.dq.")) {
3852 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ true);
3853 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
3854 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
3855 Rep =
3856 Builder.CreateSIToFP(CI->getArgOperand(1),
3857 cast<VectorType>(CI->getType())->getElementType());
3858 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3859 } else if (Name == "avx512.cvtusi2sd") {
3860 Rep =
3861 Builder.CreateUIToFP(CI->getArgOperand(1),
3862 cast<VectorType>(CI->getType())->getElementType());
3863 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3864 } else if (Name == "sse2.cvtss2sd") {
3865 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
3866 Rep = Builder.CreateFPExt(
3867 Rep, cast<VectorType>(CI->getType())->getElementType());
3868 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3869 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
3870 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
3871 Name.starts_with("avx512.mask.cvtdq2pd.") ||
3872 Name.starts_with("avx512.mask.cvtudq2pd.") ||
3873 Name.starts_with("avx512.mask.cvtdq2ps.") ||
3874 Name.starts_with("avx512.mask.cvtudq2ps.") ||
3875 Name.starts_with("avx512.mask.cvtqq2pd.") ||
3876 Name.starts_with("avx512.mask.cvtuqq2pd.") ||
3877 Name == "avx512.mask.cvtqq2ps.256" ||
3878 Name == "avx512.mask.cvtqq2ps.512" ||
3879 Name == "avx512.mask.cvtuqq2ps.256" ||
3880 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
3881 Name == "avx.cvt.ps2.pd.256" ||
3882 Name == "avx512.mask.cvtps2pd.128" ||
3883 Name == "avx512.mask.cvtps2pd.256") {
3884 auto *DstTy = cast<FixedVectorType>(CI->getType());
3885 Rep = CI->getArgOperand(0);
3886 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3887
3888 unsigned NumDstElts = DstTy->getNumElements();
3889 if (NumDstElts < SrcTy->getNumElements()) {
3890 assert(NumDstElts == 2 && "Unexpected vector size");
3891 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
3892 }
3893
3894 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3895 bool IsUnsigned = Name.contains("cvtu");
3896 if (IsPS2PD)
3897 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
3898 else if (CI->arg_size() == 4 &&
3899 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3900 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3901 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3902 : Intrinsic::x86_avx512_sitofp_round;
3903 Rep = Builder.CreateIntrinsic(IID, {DstTy, SrcTy},
3904 {Rep, CI->getArgOperand(3)});
3905 } else {
3906 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
3907 : Builder.CreateSIToFP(Rep, DstTy, "cvt");
3908 }
3909
3910 if (CI->arg_size() >= 3)
3911 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3912 CI->getArgOperand(1));
3913 } else if (Name.starts_with("avx512.mask.vcvtph2ps.") ||
3914 Name.starts_with("vcvtph2ps.")) {
3915 auto *DstTy = cast<FixedVectorType>(CI->getType());
3916 Rep = CI->getArgOperand(0);
3917 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3918 unsigned NumDstElts = DstTy->getNumElements();
3919 if (NumDstElts != SrcTy->getNumElements()) {
3920 assert(NumDstElts == 4 && "Unexpected vector size");
3921 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
3922 }
3923 Rep = Builder.CreateBitCast(
3924 Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
3925 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
3926 if (CI->arg_size() >= 3)
3927 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3928 CI->getArgOperand(1));
3929 } else if (Name.starts_with("avx512.mask.load")) {
3930 // "avx512.mask.loadu." or "avx512.mask.load."
3931 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
3932 Rep = upgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3933 CI->getArgOperand(2), Aligned);
3934 } else if (Name.starts_with("avx512.mask.expand.load.")) {
3935 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3936 auto *PtrTy = CI->getOperand(0)->getType();
3937 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3938 ResultTy->getNumElements());
3939 Rep = Builder.CreateIntrinsic(
3940 Intrinsic::masked_expandload, {ResultTy, PtrTy},
3941 {CI->getOperand(0), MaskVec, CI->getOperand(1)});
3942 } else if (Name.starts_with("avx512.mask.compress.store.")) {
3943 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
3944 auto *PtrTy = CI->getArgOperand(0)->getType();
3945 Value *MaskVec =
3946 getX86MaskVec(Builder, CI->getArgOperand(2),
3947 cast<FixedVectorType>(ResultTy)->getNumElements());
3948 Rep = Builder.CreateIntrinsic(
3949 Intrinsic::masked_compressstore, {ResultTy, PtrTy},
3950 {CI->getArgOperand(1), CI->getArgOperand(0), MaskVec});
3951 } else if (Name.starts_with("avx512.mask.compress.") ||
3952 Name.starts_with("avx512.mask.expand.")) {
3953 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3954
3955 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3956 ResultTy->getNumElements());
3957
3958 bool IsCompress = Name[12] == 'c';
3959 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3960 : Intrinsic::x86_avx512_mask_expand;
3961 Rep = Builder.CreateIntrinsic(
3962 IID, ResultTy, {CI->getOperand(0), CI->getOperand(1), MaskVec});
3963 } else if (Name.starts_with("xop.vpcom")) {
3964 bool IsSigned;
3965 if (Name.ends_with("ub") || Name.ends_with("uw") || Name.ends_with("ud") ||
3966 Name.ends_with("uq"))
3967 IsSigned = false;
3968 else if (Name.ends_with("b") || Name.ends_with("w") ||
3969 Name.ends_with("d") || Name.ends_with("q"))
3970 IsSigned = true;
3971 else
3972 reportFatalUsageErrorWithCI("Intrinsic has unknown suffix", CI);
3973
3974 unsigned Imm;
3975 if (CI->arg_size() == 3) {
3976 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3977 } else {
3978 Name = Name.substr(9); // strip off "xop.vpcom"
3979 if (Name.starts_with("lt"))
3980 Imm = 0;
3981 else if (Name.starts_with("le"))
3982 Imm = 1;
3983 else if (Name.starts_with("gt"))
3984 Imm = 2;
3985 else if (Name.starts_with("ge"))
3986 Imm = 3;
3987 else if (Name.starts_with("eq"))
3988 Imm = 4;
3989 else if (Name.starts_with("ne"))
3990 Imm = 5;
3991 else if (Name.starts_with("false"))
3992 Imm = 6;
3993 else if (Name.starts_with("true"))
3994 Imm = 7;
3995 else
3996 llvm_unreachable("Unknown condition");
3997 }
3998
3999 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
4000 } else if (Name.starts_with("xop.vpcmov")) {
4001 Value *Sel = CI->getArgOperand(2);
4002 Value *NotSel = Builder.CreateNot(Sel);
4003 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
4004 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
4005 Rep = Builder.CreateOr(Sel0, Sel1);
4006 } else if (Name.starts_with("xop.vprot") || Name.starts_with("avx512.prol") ||
4007 Name.starts_with("avx512.mask.prol")) {
4008 Rep = upgradeX86Rotate(Builder, *CI, false);
4009 } else if (Name.starts_with("avx512.pror") ||
4010 Name.starts_with("avx512.mask.pror")) {
4011 Rep = upgradeX86Rotate(Builder, *CI, true);
4012 } else if (Name.starts_with("avx512.vpshld.") ||
4013 Name.starts_with("avx512.mask.vpshld") ||
4014 Name.starts_with("avx512.maskz.vpshld")) {
4015 bool ZeroMask = Name[11] == 'z';
4016 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
4017 } else if (Name.starts_with("avx512.vpshrd.") ||
4018 Name.starts_with("avx512.mask.vpshrd") ||
4019 Name.starts_with("avx512.maskz.vpshrd")) {
4020 bool ZeroMask = Name[11] == 'z';
4021 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
4022 } else if (Name == "sse42.crc32.64.8") {
4023 Value *Trunc0 =
4024 Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
4025 Rep = Builder.CreateIntrinsic(Intrinsic::x86_sse42_crc32_32_8,
4026 {Trunc0, CI->getArgOperand(1)});
4027 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
4028 } else if (Name.starts_with("avx.vbroadcast.s") ||
4029 Name.starts_with("avx512.vbroadcast.s")) {
4030 // Replace broadcasts with a series of insertelements.
4031 auto *VecTy = cast<FixedVectorType>(CI->getType());
4032 Type *EltTy = VecTy->getElementType();
4033 unsigned EltNum = VecTy->getNumElements();
4034 Value *Load = Builder.CreateLoad(EltTy, CI->getArgOperand(0));
4035 Type *I32Ty = Type::getInt32Ty(C);
4036 Rep = PoisonValue::get(VecTy);
4037 for (unsigned I = 0; I < EltNum; ++I)
4038 Rep = Builder.CreateInsertElement(Rep, Load, ConstantInt::get(I32Ty, I));
4039 } else if (Name.starts_with("sse41.pmovsx") ||
4040 Name.starts_with("sse41.pmovzx") ||
4041 Name.starts_with("avx2.pmovsx") ||
4042 Name.starts_with("avx2.pmovzx") ||
4043 Name.starts_with("avx512.mask.pmovsx") ||
4044 Name.starts_with("avx512.mask.pmovzx")) {
4045 auto *DstTy = cast<FixedVectorType>(CI->getType());
4046 unsigned NumDstElts = DstTy->getNumElements();
4047
4048 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
4049 SmallVector<int, 8> ShuffleMask(NumDstElts);
4050 for (unsigned i = 0; i != NumDstElts; ++i)
4051 ShuffleMask[i] = i;
4052
4053 Value *SV = Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
4054
4055 bool DoSext = Name.contains("pmovsx");
4056 Rep =
4057 DoSext ? Builder.CreateSExt(SV, DstTy) : Builder.CreateZExt(SV, DstTy);
4058 // If there are 3 arguments, it's a masked intrinsic so we need a select.
4059 if (CI->arg_size() == 3)
4060 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
4061 CI->getArgOperand(1));
4062 } else if (Name == "avx512.mask.pmov.qd.256" ||
4063 Name == "avx512.mask.pmov.qd.512" ||
4064 Name == "avx512.mask.pmov.wb.256" ||
4065 Name == "avx512.mask.pmov.wb.512") {
4066 Type *Ty = CI->getArgOperand(1)->getType();
4067 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
4068 Rep =
4069 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4070 } else if (Name.starts_with("avx.vbroadcastf128") ||
4071 Name == "avx2.vbroadcasti128") {
4072 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
4073 Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
4074 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
4075 auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
4076 Value *Load = Builder.CreateAlignedLoad(VT, CI->getArgOperand(0), Align(1));
4077 if (NumSrcElts == 2)
4078 Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
4079 else
4080 Rep = Builder.CreateShuffleVector(Load,
4081 ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
4082 } else if (Name.starts_with("avx512.mask.shuf.i") ||
4083 Name.starts_with("avx512.mask.shuf.f")) {
4084 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4085 Type *VT = CI->getType();
4086 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
4087 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
4088 unsigned ControlBitsMask = NumLanes - 1;
4089 unsigned NumControlBits = NumLanes / 2;
4090 SmallVector<int, 8> ShuffleMask(0);
4091
4092 for (unsigned l = 0; l != NumLanes; ++l) {
4093 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
4094 // We actually need the other source.
4095 if (l >= NumLanes / 2)
4096 LaneMask += NumLanes;
4097 for (unsigned i = 0; i != NumElementsInLane; ++i)
4098 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
4099 }
4100 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
4101 CI->getArgOperand(1), ShuffleMask);
4102 Rep =
4103 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
4104 } else if (Name.starts_with("avx512.mask.broadcastf") ||
4105 Name.starts_with("avx512.mask.broadcasti")) {
4106 unsigned NumSrcElts = cast<FixedVectorType>(CI->getArgOperand(0)->getType())
4107 ->getNumElements();
4108 unsigned NumDstElts =
4109 cast<FixedVectorType>(CI->getType())->getNumElements();
4110
4111 SmallVector<int, 8> ShuffleMask(NumDstElts);
4112 for (unsigned i = 0; i != NumDstElts; ++i)
4113 ShuffleMask[i] = i % NumSrcElts;
4114
4115 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
4116 CI->getArgOperand(0), ShuffleMask);
4117 Rep =
4118 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4119 } else if (Name.starts_with("avx2.pbroadcast") ||
4120 Name.starts_with("avx2.vbroadcast") ||
4121 Name.starts_with("avx512.pbroadcast") ||
4122 Name.starts_with("avx512.mask.broadcast.s")) {
4123 // Replace vp?broadcasts with a vector shuffle.
4124 Value *Op = CI->getArgOperand(0);
4125 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
4126 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
4127 SmallVector<int, 8> M;
4128 ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
4129 Rep = Builder.CreateShuffleVector(Op, M);
4130
4131 if (CI->arg_size() == 3)
4132 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
4133 CI->getArgOperand(1));
4134 } else if (Name.starts_with("sse2.padds.") ||
4135 Name.starts_with("avx2.padds.") ||
4136 Name.starts_with("avx512.padds.") ||
4137 Name.starts_with("avx512.mask.padds.")) {
4138 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
4139 } else if (Name.starts_with("sse2.psubs.") ||
4140 Name.starts_with("avx2.psubs.") ||
4141 Name.starts_with("avx512.psubs.") ||
4142 Name.starts_with("avx512.mask.psubs.")) {
4143 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
4144 } else if (Name.starts_with("sse2.paddus.") ||
4145 Name.starts_with("avx2.paddus.") ||
4146 Name.starts_with("avx512.mask.paddus.")) {
4147 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
4148 } else if (Name.starts_with("sse2.psubus.") ||
4149 Name.starts_with("avx2.psubus.") ||
4150 Name.starts_with("avx512.mask.psubus.")) {
4151 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
4152 } else if (Name.starts_with("avx512.mask.palignr.")) {
4153 Rep = upgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
4154 CI->getArgOperand(1), CI->getArgOperand(2),
4155 CI->getArgOperand(3), CI->getArgOperand(4),
4156 false);
4157 } else if (Name.starts_with("avx512.mask.valign.")) {
4158 Rep = upgradeX86ALIGNIntrinsics(
4159 Builder, CI->getArgOperand(0), CI->getArgOperand(1),
4160 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), true);
4161 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
4162 // 128/256-bit shift left specified in bits.
4163 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4164 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
4165 Shift / 8); // Shift is in bits.
4166 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
4167 // 128/256-bit shift right specified in bits.
4168 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4169 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
4170 Shift / 8); // Shift is in bits.
4171 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
4172 Name == "avx512.psll.dq.512") {
4173 // 128/256/512-bit shift left specified in bytes.
4174 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4175 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
4176 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
4177 Name == "avx512.psrl.dq.512") {
4178 // 128/256/512-bit shift right specified in bytes.
4179 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4180 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
4181 } else if (Name == "sse41.pblendw" || Name.starts_with("sse41.blendp") ||
4182 Name.starts_with("avx.blend.p") || Name == "avx2.pblendw" ||
4183 Name.starts_with("avx2.pblendd.")) {
4184 Value *Op0 = CI->getArgOperand(0);
4185 Value *Op1 = CI->getArgOperand(1);
4186 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4187 auto *VecTy = cast<FixedVectorType>(CI->getType());
4188 unsigned NumElts = VecTy->getNumElements();
4189
4190 SmallVector<int, 16> Idxs(NumElts);
4191 for (unsigned i = 0; i != NumElts; ++i)
4192 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
4193
4194 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4195 } else if (Name.starts_with("avx.vinsertf128.") ||
4196 Name == "avx2.vinserti128" ||
4197 Name.starts_with("avx512.mask.insert")) {
4198 Value *Op0 = CI->getArgOperand(0);
4199 Value *Op1 = CI->getArgOperand(1);
4200 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4201 unsigned DstNumElts =
4202 cast<FixedVectorType>(CI->getType())->getNumElements();
4203 unsigned SrcNumElts =
4204 cast<FixedVectorType>(Op1->getType())->getNumElements();
4205 unsigned Scale = DstNumElts / SrcNumElts;
4206
4207 // Mask off the high bits of the immediate value; hardware ignores those.
4208 Imm = Imm % Scale;
4209
4210 // Extend the second operand into a vector the size of the destination.
4211 SmallVector<int, 8> Idxs(DstNumElts);
4212 for (unsigned i = 0; i != SrcNumElts; ++i)
4213 Idxs[i] = i;
4214 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
4215 Idxs[i] = SrcNumElts;
4216 Rep = Builder.CreateShuffleVector(Op1, Idxs);
4217
4218 // Insert the second operand into the first operand.
4219
4220 // Note that there is no guarantee that instruction lowering will actually
4221 // produce a vinsertf128 instruction for the created shuffles. In
4222 // particular, the 0 immediate case involves no lane changes, so it can
4223 // be handled as a blend.
4224
4225 // Example of shuffle mask for 32-bit elements:
4226 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
4227 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
4228
4229 // First fill with identify mask.
4230 for (unsigned i = 0; i != DstNumElts; ++i)
4231 Idxs[i] = i;
4232 // Then replace the elements where we need to insert.
4233 for (unsigned i = 0; i != SrcNumElts; ++i)
4234 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
4235 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
4236
4237 // If the intrinsic has a mask operand, handle that.
4238 if (CI->arg_size() == 5)
4239 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep,
4240 CI->getArgOperand(3));
4241 } else if (Name.starts_with("avx.vextractf128.") ||
4242 Name == "avx2.vextracti128" ||
4243 Name.starts_with("avx512.mask.vextract")) {
4244 Value *Op0 = CI->getArgOperand(0);
4245 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4246 unsigned DstNumElts =
4247 cast<FixedVectorType>(CI->getType())->getNumElements();
4248 unsigned SrcNumElts =
4249 cast<FixedVectorType>(Op0->getType())->getNumElements();
4250 unsigned Scale = SrcNumElts / DstNumElts;
4251
4252 // Mask off the high bits of the immediate value; hardware ignores those.
4253 Imm = Imm % Scale;
4254
4255 // Get indexes for the subvector of the input vector.
4256 SmallVector<int, 8> Idxs(DstNumElts);
4257 for (unsigned i = 0; i != DstNumElts; ++i) {
4258 Idxs[i] = i + (Imm * DstNumElts);
4259 }
4260 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4261
4262 // If the intrinsic has a mask operand, handle that.
4263 if (CI->arg_size() == 4)
4264 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4265 CI->getArgOperand(2));
4266 } else if (Name.starts_with("avx512.mask.perm.df.") ||
4267 Name.starts_with("avx512.mask.perm.di.")) {
4268 Value *Op0 = CI->getArgOperand(0);
4269 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4270 auto *VecTy = cast<FixedVectorType>(CI->getType());
4271 unsigned NumElts = VecTy->getNumElements();
4272
4273 SmallVector<int, 8> Idxs(NumElts);
4274 for (unsigned i = 0; i != NumElts; ++i)
4275 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
4276
4277 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4278
4279 if (CI->arg_size() == 4)
4280 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4281 CI->getArgOperand(2));
4282 } else if (Name.starts_with("avx.vperm2f128.") || Name == "avx2.vperm2i128") {
4283 // The immediate permute control byte looks like this:
4284 // [1:0] - select 128 bits from sources for low half of destination
4285 // [2] - ignore
4286 // [3] - zero low half of destination
4287 // [5:4] - select 128 bits from sources for high half of destination
4288 // [6] - ignore
4289 // [7] - zero high half of destination
4290
4291 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4292
4293 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4294 unsigned HalfSize = NumElts / 2;
4295 SmallVector<int, 8> ShuffleMask(NumElts);
4296
4297 // Determine which operand(s) are actually in use for this instruction.
4298 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4299 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4300
4301 // If needed, replace operands based on zero mask.
4302 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
4303 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
4304
4305 // Permute low half of result.
4306 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
4307 for (unsigned i = 0; i < HalfSize; ++i)
4308 ShuffleMask[i] = StartIndex + i;
4309
4310 // Permute high half of result.
4311 StartIndex = (Imm & 0x10) ? HalfSize : 0;
4312 for (unsigned i = 0; i < HalfSize; ++i)
4313 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
4314
4315 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
4316
4317 } else if (Name.starts_with("avx.vpermil.") || Name == "sse2.pshuf.d" ||
4318 Name.starts_with("avx512.mask.vpermil.p") ||
4319 Name.starts_with("avx512.mask.pshuf.d.")) {
4320 Value *Op0 = CI->getArgOperand(0);
4321 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4322 auto *VecTy = cast<FixedVectorType>(CI->getType());
4323 unsigned NumElts = VecTy->getNumElements();
4324 // Calculate the size of each index in the immediate.
4325 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
4326 unsigned IdxMask = ((1 << IdxSize) - 1);
4327
4328 SmallVector<int, 8> Idxs(NumElts);
4329 // Lookup the bits for this element, wrapping around the immediate every
4330 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
4331 // to offset by the first index of each group.
4332 for (unsigned i = 0; i != NumElts; ++i)
4333 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
4334
4335 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4336
4337 if (CI->arg_size() == 4)
4338 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4339 CI->getArgOperand(2));
4340 } else if (Name == "sse2.pshufl.w" ||
4341 Name.starts_with("avx512.mask.pshufl.w.")) {
4342 Value *Op0 = CI->getArgOperand(0);
4343 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4344 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4345
4346 if (Name == "sse2.pshufl.w" && NumElts % 8 != 0)
4347 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4348
4349 SmallVector<int, 16> Idxs(NumElts);
4350 for (unsigned l = 0; l != NumElts; l += 8) {
4351 for (unsigned i = 0; i != 4; ++i)
4352 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
4353 for (unsigned i = 4; i != 8; ++i)
4354 Idxs[i + l] = i + l;
4355 }
4356
4357 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4358
4359 if (CI->arg_size() == 4)
4360 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4361 CI->getArgOperand(2));
4362 } else if (Name == "sse2.pshufh.w" ||
4363 Name.starts_with("avx512.mask.pshufh.w.")) {
4364 Value *Op0 = CI->getArgOperand(0);
4365 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4366 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4367
4368 if (Name == "sse2.pshufh.w" && NumElts % 8 != 0)
4369 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4370
4371 SmallVector<int, 16> Idxs(NumElts);
4372 for (unsigned l = 0; l != NumElts; l += 8) {
4373 for (unsigned i = 0; i != 4; ++i)
4374 Idxs[i + l] = i + l;
4375 for (unsigned i = 0; i != 4; ++i)
4376 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
4377 }
4378
4379 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4380
4381 if (CI->arg_size() == 4)
4382 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4383 CI->getArgOperand(2));
4384 } else if (Name.starts_with("avx512.mask.shuf.p")) {
4385 Value *Op0 = CI->getArgOperand(0);
4386 Value *Op1 = CI->getArgOperand(1);
4387 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4388 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4389
4390 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4391 unsigned HalfLaneElts = NumLaneElts / 2;
4392
4393 SmallVector<int, 16> Idxs(NumElts);
4394 for (unsigned i = 0; i != NumElts; ++i) {
4395 // Base index is the starting element of the lane.
4396 Idxs[i] = i - (i % NumLaneElts);
4397 // If we are half way through the lane switch to the other source.
4398 if ((i % NumLaneElts) >= HalfLaneElts)
4399 Idxs[i] += NumElts;
4400 // Now select the specific element. By adding HalfLaneElts bits from
4401 // the immediate. Wrapping around the immediate every 8-bits.
4402 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
4403 }
4404
4405 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4406
4407 Rep =
4408 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
4409 } else if (Name.starts_with("avx512.mask.movddup") ||
4410 Name.starts_with("avx512.mask.movshdup") ||
4411 Name.starts_with("avx512.mask.movsldup")) {
4412 Value *Op0 = CI->getArgOperand(0);
4413 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4414 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4415
4416 unsigned Offset = 0;
4417 if (Name.starts_with("avx512.mask.movshdup."))
4418 Offset = 1;
4419
4420 SmallVector<int, 16> Idxs(NumElts);
4421 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
4422 for (unsigned i = 0; i != NumLaneElts; i += 2) {
4423 Idxs[i + l + 0] = i + l + Offset;
4424 Idxs[i + l + 1] = i + l + Offset;
4425 }
4426
4427 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4428
4429 Rep =
4430 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4431 } else if (Name.starts_with("avx512.mask.punpckl") ||
4432 Name.starts_with("avx512.mask.unpckl.")) {
4433 Value *Op0 = CI->getArgOperand(0);
4434 Value *Op1 = CI->getArgOperand(1);
4435 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4436 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4437
4438 SmallVector<int, 64> Idxs(NumElts);
4439 for (int l = 0; l != NumElts; l += NumLaneElts)
4440 for (int i = 0; i != NumLaneElts; ++i)
4441 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
4442
4443 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4444
4445 Rep =
4446 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4447 } else if (Name.starts_with("avx512.mask.punpckh") ||
4448 Name.starts_with("avx512.mask.unpckh.")) {
4449 Value *Op0 = CI->getArgOperand(0);
4450 Value *Op1 = CI->getArgOperand(1);
4451 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4452 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4453
4454 SmallVector<int, 64> Idxs(NumElts);
4455 for (int l = 0; l != NumElts; l += NumLaneElts)
4456 for (int i = 0; i != NumLaneElts; ++i)
4457 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
4458
4459 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4460
4461 Rep =
4462 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4463 } else if (Name.starts_with("avx512.mask.and.") ||
4464 Name.starts_with("avx512.mask.pand.")) {
4465 VectorType *FTy = cast<VectorType>(CI->getType());
4466 VectorType *ITy = VectorType::getInteger(FTy);
4467 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4468 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4469 Rep = Builder.CreateBitCast(Rep, FTy);
4470 Rep =
4471 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4472 } else if (Name.starts_with("avx512.mask.andn.") ||
4473 Name.starts_with("avx512.mask.pandn.")) {
4474 VectorType *FTy = cast<VectorType>(CI->getType());
4475 VectorType *ITy = VectorType::getInteger(FTy);
4476 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
4477 Rep = Builder.CreateAnd(Rep,
4478 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4479 Rep = Builder.CreateBitCast(Rep, FTy);
4480 Rep =
4481 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4482 } else if (Name.starts_with("avx512.mask.or.") ||
4483 Name.starts_with("avx512.mask.por.")) {
4484 VectorType *FTy = cast<VectorType>(CI->getType());
4485 VectorType *ITy = VectorType::getInteger(FTy);
4486 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4487 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4488 Rep = Builder.CreateBitCast(Rep, FTy);
4489 Rep =
4490 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4491 } else if (Name.starts_with("avx512.mask.xor.") ||
4492 Name.starts_with("avx512.mask.pxor.")) {
4493 VectorType *FTy = cast<VectorType>(CI->getType());
4494 VectorType *ITy = VectorType::getInteger(FTy);
4495 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4496 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4497 Rep = Builder.CreateBitCast(Rep, FTy);
4498 Rep =
4499 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4500 } else if (Name.starts_with("avx512.mask.padd.")) {
4501 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4502 Rep =
4503 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4504 } else if (Name.starts_with("avx512.mask.psub.")) {
4505 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
4506 Rep =
4507 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4508 } else if (Name.starts_with("avx512.mask.pmull.")) {
4509 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
4510 Rep =
4511 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4512 } else if (Name.starts_with("avx512.mask.add.p")) {
4513 if (Name.ends_with(".512")) {
4514 Intrinsic::ID IID;
4515 if (Name[17] == 's')
4516 IID = Intrinsic::x86_avx512_add_ps_512;
4517 else
4518 IID = Intrinsic::x86_avx512_add_pd_512;
4519
4520 Rep = Builder.CreateIntrinsic(
4521 IID,
4522 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4523 } else {
4524 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4525 }
4526 Rep =
4527 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4528 } else if (Name.starts_with("avx512.mask.div.p")) {
4529 if (Name.ends_with(".512")) {
4530 Intrinsic::ID IID;
4531 if (Name[17] == 's')
4532 IID = Intrinsic::x86_avx512_div_ps_512;
4533 else
4534 IID = Intrinsic::x86_avx512_div_pd_512;
4535
4536 Rep = Builder.CreateIntrinsic(
4537 IID,
4538 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4539 } else {
4540 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
4541 }
4542 Rep =
4543 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4544 } else if (Name.starts_with("avx512.mask.mul.p")) {
4545 if (Name.ends_with(".512")) {
4546 Intrinsic::ID IID;
4547 if (Name[17] == 's')
4548 IID = Intrinsic::x86_avx512_mul_ps_512;
4549 else
4550 IID = Intrinsic::x86_avx512_mul_pd_512;
4551
4552 Rep = Builder.CreateIntrinsic(
4553 IID,
4554 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4555 } else {
4556 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
4557 }
4558 Rep =
4559 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4560 } else if (Name.starts_with("avx512.mask.sub.p")) {
4561 if (Name.ends_with(".512")) {
4562 Intrinsic::ID IID;
4563 if (Name[17] == 's')
4564 IID = Intrinsic::x86_avx512_sub_ps_512;
4565 else
4566 IID = Intrinsic::x86_avx512_sub_pd_512;
4567
4568 Rep = Builder.CreateIntrinsic(
4569 IID,
4570 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4571 } else {
4572 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
4573 }
4574 Rep =
4575 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4576 } else if ((Name.starts_with("avx512.mask.max.p") ||
4577 Name.starts_with("avx512.mask.min.p")) &&
4578 Name.drop_front(18) == ".512") {
4579 bool IsDouble = Name[17] == 'd';
4580 bool IsMin = Name[13] == 'i';
4581 static const Intrinsic::ID MinMaxTbl[2][2] = {
4582 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4583 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4584 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
4585
4586 Rep = Builder.CreateIntrinsic(
4587 IID,
4588 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4589 Rep =
4590 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4591 } else if (Name.starts_with("avx512.mask.lzcnt.")) {
4592 Rep =
4593 Builder.CreateIntrinsic(Intrinsic::ctlz, CI->getType(),
4594 {CI->getArgOperand(0), Builder.getInt1(false)});
4595 Rep =
4596 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4597 } else if (Name.starts_with("avx512.mask.psll")) {
4598 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4599 bool IsVariable = Name[16] == 'v';
4600 char Size = Name[16] == '.' ? Name[17]
4601 : Name[17] == '.' ? Name[18]
4602 : Name[18] == '.' ? Name[19]
4603 : Name[20];
4604
4605 Intrinsic::ID IID;
4606 if (IsVariable && Name[17] != '.') {
4607 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
4608 IID = Intrinsic::x86_avx2_psllv_q;
4609 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
4610 IID = Intrinsic::x86_avx2_psllv_q_256;
4611 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
4612 IID = Intrinsic::x86_avx2_psllv_d;
4613 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
4614 IID = Intrinsic::x86_avx2_psllv_d_256;
4615 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
4616 IID = Intrinsic::x86_avx512_psllv_w_128;
4617 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
4618 IID = Intrinsic::x86_avx512_psllv_w_256;
4619 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
4620 IID = Intrinsic::x86_avx512_psllv_w_512;
4621 else
4622 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4623 } else if (Name.ends_with(".128")) {
4624 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
4625 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4626 : Intrinsic::x86_sse2_psll_d;
4627 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
4628 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4629 : Intrinsic::x86_sse2_psll_q;
4630 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
4631 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4632 : Intrinsic::x86_sse2_psll_w;
4633 else
4634 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4635 } else if (Name.ends_with(".256")) {
4636 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
4637 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4638 : Intrinsic::x86_avx2_psll_d;
4639 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
4640 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4641 : Intrinsic::x86_avx2_psll_q;
4642 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
4643 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4644 : Intrinsic::x86_avx2_psll_w;
4645 else
4646 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4647 } else {
4648 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
4649 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4650 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4651 : Intrinsic::x86_avx512_psll_d_512;
4652 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
4653 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4654 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4655 : Intrinsic::x86_avx512_psll_q_512;
4656 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
4657 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4658 : Intrinsic::x86_avx512_psll_w_512;
4659 else
4660 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4661 }
4662
4663 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4664 } else if (Name.starts_with("avx512.mask.psrl")) {
4665 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4666 bool IsVariable = Name[16] == 'v';
4667 char Size = Name[16] == '.' ? Name[17]
4668 : Name[17] == '.' ? Name[18]
4669 : Name[18] == '.' ? Name[19]
4670 : Name[20];
4671
4672 Intrinsic::ID IID;
4673 if (IsVariable && Name[17] != '.') {
4674 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
4675 IID = Intrinsic::x86_avx2_psrlv_q;
4676 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
4677 IID = Intrinsic::x86_avx2_psrlv_q_256;
4678 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
4679 IID = Intrinsic::x86_avx2_psrlv_d;
4680 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
4681 IID = Intrinsic::x86_avx2_psrlv_d_256;
4682 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
4683 IID = Intrinsic::x86_avx512_psrlv_w_128;
4684 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
4685 IID = Intrinsic::x86_avx512_psrlv_w_256;
4686 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
4687 IID = Intrinsic::x86_avx512_psrlv_w_512;
4688 else
4689 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4690 } else if (Name.ends_with(".128")) {
4691 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4692 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4693 : Intrinsic::x86_sse2_psrl_d;
4694 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4695 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4696 : Intrinsic::x86_sse2_psrl_q;
4697 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4698 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4699 : Intrinsic::x86_sse2_psrl_w;
4700 else
4701 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4702 } else if (Name.ends_with(".256")) {
4703 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4704 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4705 : Intrinsic::x86_avx2_psrl_d;
4706 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4707 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4708 : Intrinsic::x86_avx2_psrl_q;
4709 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4710 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4711 : Intrinsic::x86_avx2_psrl_w;
4712 else
4713 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4714 } else {
4715 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4716 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4717 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4718 : Intrinsic::x86_avx512_psrl_d_512;
4719 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4720 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4721 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4722 : Intrinsic::x86_avx512_psrl_q_512;
4723 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
4724 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4725 : Intrinsic::x86_avx512_psrl_w_512;
4726 else
4727 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4728 }
4729
4730 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4731 } else if (Name.starts_with("avx512.mask.psra")) {
4732 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4733 bool IsVariable = Name[16] == 'v';
4734 char Size = Name[16] == '.' ? Name[17]
4735 : Name[17] == '.' ? Name[18]
4736 : Name[18] == '.' ? Name[19]
4737 : Name[20];
4738
4739 Intrinsic::ID IID;
4740 if (IsVariable && Name[17] != '.') {
4741 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
4742 IID = Intrinsic::x86_avx2_psrav_d;
4743 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
4744 IID = Intrinsic::x86_avx2_psrav_d_256;
4745 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
4746 IID = Intrinsic::x86_avx512_psrav_w_128;
4747 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
4748 IID = Intrinsic::x86_avx512_psrav_w_256;
4749 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
4750 IID = Intrinsic::x86_avx512_psrav_w_512;
4751 else
4752 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4753 } else if (Name.ends_with(".128")) {
4754 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4755 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4756 : Intrinsic::x86_sse2_psra_d;
4757 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4758 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4759 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4760 : Intrinsic::x86_avx512_psra_q_128;
4761 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4762 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4763 : Intrinsic::x86_sse2_psra_w;
4764 else
4765 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4766 } else if (Name.ends_with(".256")) {
4767 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4768 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4769 : Intrinsic::x86_avx2_psra_d;
4770 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4771 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4772 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4773 : Intrinsic::x86_avx512_psra_q_256;
4774 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4775 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4776 : Intrinsic::x86_avx2_psra_w;
4777 else
4778 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4779 } else {
4780 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
4781 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4782 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4783 : Intrinsic::x86_avx512_psra_d_512;
4784 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
4785 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4786 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4787 : Intrinsic::x86_avx512_psra_q_512;
4788 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
4789 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4790 : Intrinsic::x86_avx512_psra_w_512;
4791 else
4792 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4793 }
4794
4795 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4796 } else if (Name.starts_with("avx512.mask.move.s")) {
4797 Rep = upgradeMaskedMove(Builder, *CI);
4798 } else if (Name.starts_with("avx512.cvtmask2")) {
4799 Rep = upgradeMaskToInt(Builder, *CI);
4800 } else if (Name.ends_with(".movntdqa")) {
4801 MDNode *Node = MDNode::get(
4802 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
4803
4804 LoadInst *LI = Builder.CreateAlignedLoad(
4805 CI->getType(), CI->getArgOperand(0),
4806 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
4807 LI->setMetadata(LLVMContext::MD_nontemporal, Node);
4808 Rep = LI;
4809 } else if (Name.starts_with("fma.vfmadd.") ||
4810 Name.starts_with("fma.vfmsub.") ||
4811 Name.starts_with("fma.vfnmadd.") ||
4812 Name.starts_with("fma.vfnmsub.")) {
4813 bool NegMul = Name[6] == 'n';
4814 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
4815 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
4816
4817 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4818 CI->getArgOperand(2)};
4819
4820 if (IsScalar) {
4821 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4822 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4823 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4824 }
4825
4826 if (NegMul && !IsScalar)
4827 Ops[0] = Builder.CreateFNeg(Ops[0]);
4828 if (NegMul && IsScalar)
4829 Ops[1] = Builder.CreateFNeg(Ops[1]);
4830 if (NegAcc)
4831 Ops[2] = Builder.CreateFNeg(Ops[2]);
4832
4833 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4834
4835 if (IsScalar)
4836 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
4837 } else if (Name.starts_with("fma4.vfmadd.s")) {
4838 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4839 CI->getArgOperand(2)};
4840
4841 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4842 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4843 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4844
4845 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4846
4847 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
4848 Rep, (uint64_t)0);
4849 } else if (Name.starts_with("avx512.mask.vfmadd.s") ||
4850 Name.starts_with("avx512.maskz.vfmadd.s") ||
4851 Name.starts_with("avx512.mask3.vfmadd.s") ||
4852 Name.starts_with("avx512.mask3.vfmsub.s") ||
4853 Name.starts_with("avx512.mask3.vfnmsub.s")) {
4854 bool IsMask3 = Name[11] == '3';
4855 bool IsMaskZ = Name[11] == 'z';
4856 // Drop the "avx512.mask." to make it easier.
4857 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4858 bool NegMul = Name[2] == 'n';
4859 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4860
4861 Value *A = CI->getArgOperand(0);
4862 Value *B = CI->getArgOperand(1);
4863 Value *C = CI->getArgOperand(2);
4864
4865 if (NegMul && (IsMask3 || IsMaskZ))
4866 A = Builder.CreateFNeg(A);
4867 if (NegMul && !(IsMask3 || IsMaskZ))
4868 B = Builder.CreateFNeg(B);
4869 if (NegAcc)
4870 C = Builder.CreateFNeg(C);
4871
4872 A = Builder.CreateExtractElement(A, (uint64_t)0);
4873 B = Builder.CreateExtractElement(B, (uint64_t)0);
4874 C = Builder.CreateExtractElement(C, (uint64_t)0);
4875
4876 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4877 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
4878 Value *Ops[] = {A, B, C, CI->getArgOperand(4)};
4879
4880 Intrinsic::ID IID;
4881 if (Name.back() == 'd')
4882 IID = Intrinsic::x86_avx512_vfmadd_f64;
4883 else
4884 IID = Intrinsic::x86_avx512_vfmadd_f32;
4885 Rep = Builder.CreateIntrinsic(IID, Ops);
4886 } else {
4887 Rep = Builder.CreateFMA(A, B, C);
4888 }
4889
4890 Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType())
4891 : IsMask3 ? C
4892 : A;
4893
4894 // For Mask3 with NegAcc, we need to create a new extractelement that
4895 // avoids the negation above.
4896 if (NegAcc && IsMask3)
4897 PassThru =
4898 Builder.CreateExtractElement(CI->getArgOperand(2), (uint64_t)0);
4899
4900 Rep = emitX86ScalarSelect(Builder, CI->getArgOperand(3), Rep, PassThru);
4901 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), Rep,
4902 (uint64_t)0);
4903 } else if (Name.starts_with("avx512.mask.vfmadd.p") ||
4904 Name.starts_with("avx512.mask.vfnmadd.p") ||
4905 Name.starts_with("avx512.mask.vfnmsub.p") ||
4906 Name.starts_with("avx512.mask3.vfmadd.p") ||
4907 Name.starts_with("avx512.mask3.vfmsub.p") ||
4908 Name.starts_with("avx512.mask3.vfnmsub.p") ||
4909 Name.starts_with("avx512.maskz.vfmadd.p")) {
4910 bool IsMask3 = Name[11] == '3';
4911 bool IsMaskZ = Name[11] == 'z';
4912 // Drop the "avx512.mask." to make it easier.
4913 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4914 bool NegMul = Name[2] == 'n';
4915 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4916
4917 Value *A = CI->getArgOperand(0);
4918 Value *B = CI->getArgOperand(1);
4919 Value *C = CI->getArgOperand(2);
4920
4921 if (NegMul && (IsMask3 || IsMaskZ))
4922 A = Builder.CreateFNeg(A);
4923 if (NegMul && !(IsMask3 || IsMaskZ))
4924 B = Builder.CreateFNeg(B);
4925 if (NegAcc)
4926 C = Builder.CreateFNeg(C);
4927
4928 if (CI->arg_size() == 5 &&
4929 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4930 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
4931 Intrinsic::ID IID;
4932 // Check the character before ".512" in string.
4933 if (Name[Name.size() - 5] == 's')
4934 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4935 else
4936 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4937
4938 Rep = Builder.CreateIntrinsic(IID, {A, B, C, CI->getArgOperand(4)});
4939 } else {
4940 Rep = Builder.CreateFMA(A, B, C);
4941 }
4942
4943 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4944 : IsMask3 ? CI->getArgOperand(2)
4945 : CI->getArgOperand(0);
4946
4947 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4948 } else if (Name.starts_with("fma.vfmsubadd.p")) {
4949 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4950 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4951 Intrinsic::ID IID;
4952 if (VecWidth == 128 && EltWidth == 32)
4953 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4954 else if (VecWidth == 256 && EltWidth == 32)
4955 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4956 else if (VecWidth == 128 && EltWidth == 64)
4957 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4958 else if (VecWidth == 256 && EltWidth == 64)
4959 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4960 else
4961 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4962
4963 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4964 CI->getArgOperand(2)};
4965 Ops[2] = Builder.CreateFNeg(Ops[2]);
4966 Rep = Builder.CreateIntrinsic(IID, Ops);
4967 } else if (Name.starts_with("avx512.mask.vfmaddsub.p") ||
4968 Name.starts_with("avx512.mask3.vfmaddsub.p") ||
4969 Name.starts_with("avx512.maskz.vfmaddsub.p") ||
4970 Name.starts_with("avx512.mask3.vfmsubadd.p")) {
4971 bool IsMask3 = Name[11] == '3';
4972 bool IsMaskZ = Name[11] == 'z';
4973 // Drop the "avx512.mask." to make it easier.
4974 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4975 bool IsSubAdd = Name[3] == 's';
4976 if (CI->arg_size() == 5) {
4977 Intrinsic::ID IID;
4978 // Check the character before ".512" in string.
4979 if (Name[Name.size() - 5] == 's')
4980 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
4981 else
4982 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
4983
4984 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4985 CI->getArgOperand(2), CI->getArgOperand(4)};
4986 if (IsSubAdd)
4987 Ops[2] = Builder.CreateFNeg(Ops[2]);
4988
4989 Rep = Builder.CreateIntrinsic(IID, Ops);
4990 } else {
4991 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4992
4993 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4994 CI->getArgOperand(2)};
4995
4996 Function *FMA = Intrinsic::getOrInsertDeclaration(
4997 CI->getModule(), Intrinsic::fma, Ops[0]->getType());
4998 Value *Odd = Builder.CreateCall(FMA, Ops);
4999 Ops[2] = Builder.CreateFNeg(Ops[2]);
5000 Value *Even = Builder.CreateCall(FMA, Ops);
5001
5002 if (IsSubAdd)
5003 std::swap(Even, Odd);
5004
5005 SmallVector<int, 32> Idxs(NumElts);
5006 for (int i = 0; i != NumElts; ++i)
5007 Idxs[i] = i + (i % 2) * NumElts;
5008
5009 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
5010 }
5011
5012 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
5013 : IsMask3 ? CI->getArgOperand(2)
5014 : CI->getArgOperand(0);
5015
5016 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5017 } else if (Name.starts_with("avx512.mask.pternlog.") ||
5018 Name.starts_with("avx512.maskz.pternlog.")) {
5019 bool ZeroMask = Name[11] == 'z';
5020 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5021 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
5022 Intrinsic::ID IID;
5023 if (VecWidth == 128 && EltWidth == 32)
5024 IID = Intrinsic::x86_avx512_pternlog_d_128;
5025 else if (VecWidth == 256 && EltWidth == 32)
5026 IID = Intrinsic::x86_avx512_pternlog_d_256;
5027 else if (VecWidth == 512 && EltWidth == 32)
5028 IID = Intrinsic::x86_avx512_pternlog_d_512;
5029 else if (VecWidth == 128 && EltWidth == 64)
5030 IID = Intrinsic::x86_avx512_pternlog_q_128;
5031 else if (VecWidth == 256 && EltWidth == 64)
5032 IID = Intrinsic::x86_avx512_pternlog_q_256;
5033 else if (VecWidth == 512 && EltWidth == 64)
5034 IID = Intrinsic::x86_avx512_pternlog_q_512;
5035 else
5036 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5037
5038 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5039 CI->getArgOperand(2), CI->getArgOperand(3)};
5040 Rep = Builder.CreateIntrinsic(IID, Args);
5041 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5042 : CI->getArgOperand(0);
5043 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
5044 } else if (Name.starts_with("avx512.mask.vpmadd52") ||
5045 Name.starts_with("avx512.maskz.vpmadd52")) {
5046 bool ZeroMask = Name[11] == 'z';
5047 bool High = Name[20] == 'h' || Name[21] == 'h';
5048 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5049 Intrinsic::ID IID;
5050 if (VecWidth == 128 && !High)
5051 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
5052 else if (VecWidth == 256 && !High)
5053 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
5054 else if (VecWidth == 512 && !High)
5055 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
5056 else if (VecWidth == 128 && High)
5057 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
5058 else if (VecWidth == 256 && High)
5059 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
5060 else if (VecWidth == 512 && High)
5061 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
5062 else
5063 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5064
5065 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5066 CI->getArgOperand(2)};
5067 Rep = Builder.CreateIntrinsic(IID, Args);
5068 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5069 : CI->getArgOperand(0);
5070 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5071 } else if (Name.starts_with("avx512.mask.vpermi2var.") ||
5072 Name.starts_with("avx512.mask.vpermt2var.") ||
5073 Name.starts_with("avx512.maskz.vpermt2var.")) {
5074 bool ZeroMask = Name[11] == 'z';
5075 bool IndexForm = Name[17] == 'i';
5076 Rep = upgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
5077 } else if (Name.starts_with("avx512.mask.vpdpbusd.") ||
5078 Name.starts_with("avx512.maskz.vpdpbusd.") ||
5079 Name.starts_with("avx512.mask.vpdpbusds.") ||
5080 Name.starts_with("avx512.maskz.vpdpbusds.")) {
5081 bool ZeroMask = Name[11] == 'z';
5082 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
5083 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5084 Intrinsic::ID IID;
5085 if (VecWidth == 128 && !IsSaturating)
5086 IID = Intrinsic::x86_avx512_vpdpbusd_128;
5087 else if (VecWidth == 256 && !IsSaturating)
5088 IID = Intrinsic::x86_avx512_vpdpbusd_256;
5089 else if (VecWidth == 512 && !IsSaturating)
5090 IID = Intrinsic::x86_avx512_vpdpbusd_512;
5091 else if (VecWidth == 128 && IsSaturating)
5092 IID = Intrinsic::x86_avx512_vpdpbusds_128;
5093 else if (VecWidth == 256 && IsSaturating)
5094 IID = Intrinsic::x86_avx512_vpdpbusds_256;
5095 else if (VecWidth == 512 && IsSaturating)
5096 IID = Intrinsic::x86_avx512_vpdpbusds_512;
5097 else
5098 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5099
5100 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5101 CI->getArgOperand(2)};
5102
5103 // Input arguments types were incorrectly set to vectors of i32 before but
5104 // they should be vectors of i8. Insert bit cast when encountering the old
5105 // types
5106 if (Args[1]->getType()->isVectorTy() &&
5107 cast<VectorType>(Args[1]->getType())
5108 ->getElementType()
5109 ->isIntegerTy(32) &&
5110 Args[2]->getType()->isVectorTy() &&
5111 cast<VectorType>(Args[2]->getType())
5112 ->getElementType()
5113 ->isIntegerTy(32)) {
5114 Type *NewArgType = nullptr;
5115 if (VecWidth == 128)
5116 NewArgType = VectorType::get(Builder.getInt8Ty(), 16, false);
5117 else if (VecWidth == 256)
5118 NewArgType = VectorType::get(Builder.getInt8Ty(), 32, false);
5119 else if (VecWidth == 512)
5120 NewArgType = VectorType::get(Builder.getInt8Ty(), 64, false);
5121 else
5122 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
5123 CI);
5124
5125 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5126 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5127 }
5128
5129 Rep = Builder.CreateIntrinsic(IID, Args);
5130 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5131 : CI->getArgOperand(0);
5132 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5133 } else if (Name.starts_with("avx512.mask.vpdpwssd.") ||
5134 Name.starts_with("avx512.maskz.vpdpwssd.") ||
5135 Name.starts_with("avx512.mask.vpdpwssds.") ||
5136 Name.starts_with("avx512.maskz.vpdpwssds.")) {
5137 bool ZeroMask = Name[11] == 'z';
5138 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
5139 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5140 Intrinsic::ID IID;
5141 if (VecWidth == 128 && !IsSaturating)
5142 IID = Intrinsic::x86_avx512_vpdpwssd_128;
5143 else if (VecWidth == 256 && !IsSaturating)
5144 IID = Intrinsic::x86_avx512_vpdpwssd_256;
5145 else if (VecWidth == 512 && !IsSaturating)
5146 IID = Intrinsic::x86_avx512_vpdpwssd_512;
5147 else if (VecWidth == 128 && IsSaturating)
5148 IID = Intrinsic::x86_avx512_vpdpwssds_128;
5149 else if (VecWidth == 256 && IsSaturating)
5150 IID = Intrinsic::x86_avx512_vpdpwssds_256;
5151 else if (VecWidth == 512 && IsSaturating)
5152 IID = Intrinsic::x86_avx512_vpdpwssds_512;
5153 else
5154 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5155
5156 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5157 CI->getArgOperand(2)};
5158
5159 // Input arguments types were incorrectly set to vectors of i32 before but
5160 // they should be vectors of i16. Insert bit cast when encountering the old
5161 // types
5162 if (Args[1]->getType()->isVectorTy() &&
5163 cast<VectorType>(Args[1]->getType())
5164 ->getElementType()
5165 ->isIntegerTy(32) &&
5166 Args[2]->getType()->isVectorTy() &&
5167 cast<VectorType>(Args[2]->getType())
5168 ->getElementType()
5169 ->isIntegerTy(32)) {
5170 Type *NewArgType = nullptr;
5171 if (VecWidth == 128)
5172 NewArgType = VectorType::get(Builder.getInt16Ty(), 8, false);
5173 else if (VecWidth == 256)
5174 NewArgType = VectorType::get(Builder.getInt16Ty(), 16, false);
5175 else if (VecWidth == 512)
5176 NewArgType = VectorType::get(Builder.getInt16Ty(), 32, false);
5177 else
5178 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
5179 CI);
5180
5181 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5182 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5183 }
5184
5185 Rep = Builder.CreateIntrinsic(IID, Args);
5186 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5187 : CI->getArgOperand(0);
5188 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5189 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
5190 Name == "addcarry.u32" || Name == "addcarry.u64" ||
5191 Name == "subborrow.u32" || Name == "subborrow.u64") {
5192 Intrinsic::ID IID;
5193 if (Name[0] == 'a' && Name.back() == '2')
5194 IID = Intrinsic::x86_addcarry_32;
5195 else if (Name[0] == 'a' && Name.back() == '4')
5196 IID = Intrinsic::x86_addcarry_64;
5197 else if (Name[0] == 's' && Name.back() == '2')
5198 IID = Intrinsic::x86_subborrow_32;
5199 else if (Name[0] == 's' && Name.back() == '4')
5200 IID = Intrinsic::x86_subborrow_64;
5201 else
5202 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5203
5204 // Make a call with 3 operands.
5205 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5206 CI->getArgOperand(2)};
5207 Value *NewCall = Builder.CreateIntrinsic(IID, Args);
5208
5209 // Extract the second result and store it.
5210 Value *Data = Builder.CreateExtractValue(NewCall, 1);
5211 Builder.CreateAlignedStore(Data, CI->getArgOperand(3), Align(1));
5212 // Replace the original call result with the first result of the new call.
5213 Value *CF = Builder.CreateExtractValue(NewCall, 0);
5214
5215 CI->replaceAllUsesWith(CF);
5216 Rep = nullptr;
5217 } else if (Name.starts_with("avx512.mask.") &&
5218 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
5219 // Rep will be updated by the call in the condition.
5220 } else if (Name.starts_with("bmi.pdep.")) {
5221 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pdep);
5222 } else if (Name.starts_with("bmi.pext.")) {
5223 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pext);
5224 } else
5225 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5226
5227 return Rep;
5228}
5229
5230static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
5231 Function *F, IRBuilder<> &Builder) {
5232 if (Name.starts_with("neon.bfcvt")) {
5233 if (Name.starts_with("neon.bfcvtn2")) {
5234 SmallVector<int, 32> LoMask(4);
5235 std::iota(LoMask.begin(), LoMask.end(), 0);
5236 SmallVector<int, 32> ConcatMask(8);
5237 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
5238 Value *Inactive = Builder.CreateShuffleVector(CI->getOperand(0), LoMask);
5239 Value *Trunc =
5240 Builder.CreateFPTrunc(CI->getOperand(1), Inactive->getType());
5241 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
5242 } else if (Name.starts_with("neon.bfcvtn")) {
5243 SmallVector<int, 32> ConcatMask(8);
5244 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
5245 Type *V4BF16 =
5246 FixedVectorType::get(Type::getBFloatTy(F->getContext()), 4);
5247 Value *Trunc = Builder.CreateFPTrunc(CI->getOperand(0), V4BF16);
5248 dbgs() << "Trunc: " << *Trunc << "\n";
5249 return Builder.CreateShuffleVector(
5250 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
5251 } else {
5252 return Builder.CreateFPTrunc(CI->getOperand(0),
5253 Type::getBFloatTy(F->getContext()));
5254 }
5255 } else if (Name.starts_with("sve.fcvt")) {
5256 Intrinsic::ID NewID =
5257 StringSwitch<Intrinsic::ID>(Name)
5258 .Case("sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
5259 .Case("sve.fcvtnt.bf16f32",
5260 Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
5261 .Default(Intrinsic::not_intrinsic);
5262 if (NewID == Intrinsic::not_intrinsic)
5263 llvm_unreachable("Unhandled Intrinsic!");
5264
5265 SmallVector<Value *, 3> Args(CI->args());
5266
5267 // The original intrinsics incorrectly used a predicate based on the
5268 // smallest element type rather than the largest.
5269 Type *BadPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 8);
5270 Type *GoodPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 4);
5271
5272 if (Args[1]->getType() != BadPredTy)
5273 llvm_unreachable("Unexpected predicate type!");
5274
5275 Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
5276 BadPredTy, Args[1]);
5277 Args[1] = Builder.CreateIntrinsic(
5278 Intrinsic::aarch64_sve_convert_from_svbool, GoodPredTy, Args[1]);
5279
5280 return Builder.CreateIntrinsic(NewID, Args, /*FMFSource=*/nullptr,
5281 CI->getName());
5282 }
5283
5284 if (Name == "neon.vcvtfp2hf")
5285 return Builder.CreateBitCast(
5286 Builder.CreateFPTrunc(
5287 CI->getOperand(0),
5288 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5289 FixedVectorType::get(Type::getInt16Ty(F->getContext()), 4));
5290 if (Name == "neon.vcvthf2fp")
5291 return Builder.CreateFPExt(
5292 Builder.CreateBitCast(
5293 CI->getOperand(0),
5294 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5295 FixedVectorType::get(Type::getFloatTy(F->getContext()), 4));
5296
5297 llvm_unreachable("Unhandled Intrinsic!");
5298}
5299
5300static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
5301 IRBuilder<> &Builder) {
5302 if (Name == "mve.vctp64.old") {
5303 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
5304 // correct type.
5305 Value *VCTP = Builder.CreateIntrinsic(Intrinsic::arm_mve_vctp64, {},
5306 CI->getArgOperand(0),
5307 /*FMFSource=*/nullptr, CI->getName());
5308 Value *C1 = Builder.CreateIntrinsic(
5309 Intrinsic::arm_mve_pred_v2i,
5310 {VectorType::get(Builder.getInt1Ty(), 2, false)}, VCTP);
5311 return Builder.CreateIntrinsic(
5312 Intrinsic::arm_mve_pred_i2v,
5313 {VectorType::get(Builder.getInt1Ty(), 4, false)}, C1);
5314 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
5315 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
5316 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
5317 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
5318 Name ==
5319 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
5320 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
5321 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
5322 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
5323 Name ==
5324 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
5325 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
5326 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
5327 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
5328 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
5329 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
5330 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
5331 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
5332 std::vector<Type *> Tys;
5333 unsigned ID = CI->getIntrinsicID();
5334 Type *V2I1Ty = FixedVectorType::get(Builder.getInt1Ty(), 2);
5335 switch (ID) {
5336 case Intrinsic::arm_mve_mull_int_predicated:
5337 case Intrinsic::arm_mve_vqdmull_predicated:
5338 case Intrinsic::arm_mve_vldr_gather_base_predicated:
5339 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty};
5340 break;
5341 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
5342 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
5343 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
5344 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(),
5345 V2I1Ty};
5346 break;
5347 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
5348 Tys = {CI->getType(), CI->getOperand(0)->getType(),
5349 CI->getOperand(1)->getType(), V2I1Ty};
5350 break;
5351 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
5352 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(),
5353 CI->getOperand(2)->getType(), V2I1Ty};
5354 break;
5355 case Intrinsic::arm_cde_vcx1q_predicated:
5356 case Intrinsic::arm_cde_vcx1qa_predicated:
5357 case Intrinsic::arm_cde_vcx2q_predicated:
5358 case Intrinsic::arm_cde_vcx2qa_predicated:
5359 case Intrinsic::arm_cde_vcx3q_predicated:
5360 case Intrinsic::arm_cde_vcx3qa_predicated:
5361 Tys = {CI->getOperand(1)->getType(), V2I1Ty};
5362 break;
5363 default:
5364 llvm_unreachable("Unhandled Intrinsic!");
5365 }
5366
5367 std::vector<Value *> Ops;
5368 for (Value *Op : CI->args()) {
5369 Type *Ty = Op->getType();
5370 if (Ty->getScalarSizeInBits() == 1) {
5371 Value *C1 = Builder.CreateIntrinsic(
5372 Intrinsic::arm_mve_pred_v2i,
5373 {VectorType::get(Builder.getInt1Ty(), 4, false)}, Op);
5374 Op = Builder.CreateIntrinsic(Intrinsic::arm_mve_pred_i2v, {V2I1Ty}, C1);
5375 }
5376 Ops.push_back(Op);
5377 }
5378
5379 return Builder.CreateIntrinsic(ID, Tys, Ops, /*FMFSource=*/nullptr,
5380 CI->getName());
5381 }
5382 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
5383}
5384
5385// These are expected to have the arguments:
5386// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
5387//
5388// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
5389//
5390static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
5391 Function *F, IRBuilder<> &Builder) {
5392 // Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
5393 // for compatibility.
5394 auto UpgradeLegacyWMMAIUIntrinsicCall =
5395 [](Function *F, CallBase *CI, IRBuilder<> &Builder,
5396 ArrayRef<Type *> OverloadTys) -> Value * {
5397 // Prepare arguments, append clamp=0 for compatibility
5398 SmallVector<Value *, 10> Args(CI->args().begin(), CI->args().end());
5399 Args.push_back(Builder.getFalse());
5400
5401 // Insert the declaration for the right overload types
5402 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5403 F->getParent(), F->getIntrinsicID(), OverloadTys);
5404
5405 // Copy operand bundles if any
5406 SmallVector<OperandBundleDef, 1> Bundles;
5407 CI->getOperandBundlesAsDefs(Bundles);
5408
5409 // Create the new call and copy calling properties
5410 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5411 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5412 NewCall->setCallingConv(CI->getCallingConv());
5413 NewCall->setAttributes(CI->getAttributes());
5414 NewCall->copyMetadata(*CI);
5415 return NewCall;
5416 };
5417
5418 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
5419 assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
5420 "intrinsic should have 7 arguments");
5421 Type *T1 = CI->getArgOperand(4)->getType();
5422 Type *T2 = CI->getArgOperand(1)->getType();
5423 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2});
5424 }
5425 if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
5426 assert(CI->arg_size() == 8 && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
5427 "intrinsic should have 8 arguments");
5428 Type *T1 = CI->getArgOperand(4)->getType();
5429 Type *T2 = CI->getArgOperand(1)->getType();
5430 Type *T3 = CI->getArgOperand(3)->getType();
5431 Type *T4 = CI->getArgOperand(5)->getType();
5432 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
5433 }
5434
5435 switch (F->getIntrinsicID()) {
5436 default:
5437 break;
5438 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
5439 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
5440 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
5441 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
5442 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
5443 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
5444 // Drop src0 and src1 modifiers.
5445 const Value *Op0 = CI->getArgOperand(0);
5446 const Value *Op2 = CI->getArgOperand(2);
5447 assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
5448 const ConstantInt *ModA = dyn_cast<ConstantInt>(Op0);
5449 const ConstantInt *ModB = dyn_cast<ConstantInt>(Op2);
5450 if (!ModA->isZero() || !ModB->isZero())
5451 reportFatalUsageError(Name + " matrix A and B modifiers shall be zero");
5452
5453 SmallVector<Value *, 8> Args{CI->getArgOperand(1), CI->getArgOperand(3)};
5454 for (int I = 4, E = CI->arg_size(); I < E; ++I)
5455 Args.push_back(CI->getArgOperand(I));
5456
5457 SmallVector<Type *, 3> Overloads{F->getReturnType(), Args[0]->getType()};
5458 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
5459 Overloads.push_back(Args[3]->getType());
5460 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5461 F->getParent(), F->getIntrinsicID(), Overloads);
5462
5463 SmallVector<OperandBundleDef, 1> Bundles;
5464 CI->getOperandBundlesAsDefs(Bundles);
5465
5466 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5467 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5468 NewCall->setCallingConv(CI->getCallingConv());
5469 NewCall->setAttributes(CI->getAttributes());
5470 NewCall->copyMetadata(*CI);
5471 NewCall->takeName(CI);
5472 return NewCall;
5473 }
5474 }
5475
5476 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
5477 Value *LHS = CI->getArgOperand(0);
5478 Value *RHS = CI->getArgOperand(1);
5479 CmpInst::Predicate Pred = static_cast<CmpInst::Predicate>(
5480 cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue());
5481 Value *Cmp = Builder.CreateCmp(Pred, LHS, RHS);
5482 CallInst *NewCall = Builder.CreateIntrinsicWithoutFolding(
5483 CI->getType(), Intrinsic::amdgcn_ballot, Cmp);
5484 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5485 NewCall->setCallingConv(CI->getCallingConv());
5486 NewCall->copyMetadata(*CI);
5487 NewCall->takeName(CI);
5488 return NewCall;
5489 }
5490
5491 if (Name.starts_with("addrspacecast.nonnull")) {
5492 if (CI->getNumOperands() < 2) // Malformed bitcode.
5493 return nullptr;
5494 Value *ASC = Builder.CreateAddrSpaceCast(
5495 CI->getArgOperand(0), CI->getType(), "", /*IsNonNull=*/true);
5496 ASC->takeName(CI);
5497 return ASC;
5498 }
5499
5500 AtomicRMWInst::BinOp RMWOp =
5501 StringSwitch<AtomicRMWInst::BinOp>(Name)
5502 .StartsWith("ds.fadd", AtomicRMWInst::FAdd)
5503 .StartsWith("ds.fmin", AtomicRMWInst::FMin)
5504 .StartsWith("ds.fmax", AtomicRMWInst::FMax)
5505 .StartsWith("atomic.inc.", AtomicRMWInst::UIncWrap)
5506 .StartsWith("atomic.dec.", AtomicRMWInst::UDecWrap)
5507 .StartsWith("global.atomic.fadd", AtomicRMWInst::FAdd)
5508 .StartsWith("flat.atomic.fadd", AtomicRMWInst::FAdd)
5509 .StartsWith("global.atomic.fmin", AtomicRMWInst::FMin)
5510 .StartsWith("flat.atomic.fmin", AtomicRMWInst::FMin)
5511 .StartsWith("global.atomic.fmax", AtomicRMWInst::FMax)
5512 .StartsWith("flat.atomic.fmax", AtomicRMWInst::FMax)
5513 .StartsWith("atomic.cond.sub", AtomicRMWInst::USubCond)
5514 .StartsWith("atomic.csub", AtomicRMWInst::USubSat);
5515
5516 unsigned NumOperands = CI->getNumOperands();
5517 if (NumOperands < 3) // Malformed bitcode.
5518 return nullptr;
5519
5520 Value *Ptr = CI->getArgOperand(0);
5521 PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
5522 if (!PtrTy) // Malformed.
5523 return nullptr;
5524
5525 Value *Val = CI->getArgOperand(1);
5526 if (Val->getType() != CI->getType()) // Malformed.
5527 return nullptr;
5528
5529 ConstantInt *OrderArg = nullptr;
5530 bool IsVolatile = false;
5531
5532 // These should have 5 arguments (plus the callee). A separate version of the
5533 // ds_fadd intrinsic was defined for bf16 which was missing arguments.
5534 if (NumOperands > 3)
5535 OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5536
5537 // Ignore scope argument at 3
5538
5539 if (NumOperands > 5) {
5540 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4));
5541 IsVolatile = !VolatileArg || !VolatileArg->isZero();
5542 }
5543
5544 AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
5545 if (OrderArg && isValidAtomicOrdering(OrderArg->getZExtValue()))
5546 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
5547 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
5548 Order = AtomicOrdering::SequentiallyConsistent;
5549
5550 LLVMContext &Ctx = F->getContext();
5551
5552 // Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
5553 Type *RetTy = CI->getType();
5554 if (VectorType *VT = dyn_cast<VectorType>(RetTy)) {
5555 if (VT->getElementType()->isIntegerTy(16)) {
5556 VectorType *AsBF16 =
5557 VectorType::get(Type::getBFloatTy(Ctx), VT->getElementCount());
5558 Val = Builder.CreateBitCast(Val, AsBF16);
5559 }
5560 }
5561
5562 // The scope argument never really worked correctly. Use agent as the most
5563 // conservative option which should still always produce the instruction.
5564 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID("agent");
5565 AtomicRMWInst *RMW =
5566 Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);
5567
5568 unsigned AddrSpace = PtrTy->getAddressSpace();
5569 if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
5570 MDNode *EmptyMD = MDNode::get(F->getContext(), {});
5571 RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);
5572 if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
5573 RMW->setMetadata(LLVMContext::MD_atomic_ignore_denormal_mode, EmptyMD);
5574 }
5575
5576 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
5577 MDBuilder MDB(F->getContext());
5578 MDNode *RangeNotPrivate =
5579 MDB.createRange(APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
5580 APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
5581 RMW->setMetadata(LLVMContext::MD_noalias_addrspace, RangeNotPrivate);
5582 }
5583
5584 if (IsVolatile)
5585 RMW->setVolatile(true);
5586
5587 return Builder.CreateBitCast(RMW, RetTy);
5588}
5589
5590/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
5591/// plain MDNode, as it's the verifier's job to check these are the correct
5592/// types later.
5593static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
5594 if (Op < CI->arg_size()) {
5595 if (MetadataAsValue *MAV =
5596 dyn_cast<MetadataAsValue>(CI->getArgOperand(Op))) {
5597 Metadata *MD = MAV->getMetadata();
5598 return dyn_cast_if_present<MDNode>(MD);
5599 }
5600 }
5601 return nullptr;
5602}
5603
5604/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
5605static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
5606 if (Op < CI->arg_size())
5607 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(CI->getArgOperand(Op)))
5608 return MAV->getMetadata();
5609 return nullptr;
5610}
5611
5612/// Convert debug intrinsic calls to non-instruction debug records.
5613/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
5614/// \p CI - The debug intrinsic call.
5615static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
5616 DbgRecord *DR = nullptr;
5617 if (Name == "label") {
5618 DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(unwrapMAVOp(CI, 0));
5619 } else if (Name == "assign") {
5620 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5621 DbgVariableRecord::LocationType::Assign, unwrapMAVMetadataOp(CI, 0),
5622 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), unwrapMAVOp(CI, 3),
5623 unwrapMAVMetadataOp(CI, 4),
5624 /*The address is a Value ref, it will be stored as a Metadata */
5625 unwrapMAVOp(CI, 5));
5626 } else if (Name == "declare") {
5627 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5628 DbgVariableRecord::LocationType::Declare, unwrapMAVMetadataOp(CI, 0),
5629 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), nullptr, nullptr, nullptr);
5630 } else if (Name == "addr") {
5631 // Upgrade dbg.addr to dbg.value with DW_OP_deref.
5632 MDNode *ExprNode = unwrapMAVOp(CI, 2);
5633 // Don't try to add something to the expression if it's not an expression.
5634 // Instead, allow the verifier to fail later.
5635 if (DIExpression *Expr = dyn_cast<DIExpression>(ExprNode)) {
5636 ExprNode = DIExpression::append(Expr, dwarf::DW_OP_deref);
5637 }
5638 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5639 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5640 unwrapMAVOp(CI, 1), ExprNode, nullptr, nullptr, nullptr);
5641 } else if (Name == "value") {
5642 // An old version of dbg.value had an extra offset argument.
5643 unsigned VarOp = 1;
5644 unsigned ExprOp = 2;
5645 if (CI->arg_size() == 4) {
5646 auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1));
5647 // Nonzero offset dbg.values get dropped without a replacement.
5648 if (!Offset || !Offset->isNullValue())
5649 return;
5650 VarOp = 2;
5651 ExprOp = 3;
5652 }
5653 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5654 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5655 unwrapMAVOp(CI, VarOp), unwrapMAVOp(CI, ExprOp), nullptr, nullptr,
5656 nullptr);
5657 }
5658 DR->setDebugLoc(CI->getDebugLoc());
5659 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
5660 CI->getParent()->insertDbgRecordBefore(DR, CI->getIterator());
5661}
5662
5663static Value *upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder) {
5664 auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5665 if (!Offset)
5666 reportFatalUsageError("Invalid llvm.vector.splice offset argument");
5667 int64_t OffsetVal = Offset->getSExtValue();
5668 return Builder.CreateIntrinsic(OffsetVal >= 0
5669 ? Intrinsic::vector_splice_left
5670 : Intrinsic::vector_splice_right,
5671 CI->getType(),
5672 {CI->getArgOperand(0), CI->getArgOperand(1),
5673 Builder.getInt32(std::abs(OffsetVal))});
5674}
5675
5676static Value *upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI,
5677 Function *F, IRBuilder<> &Builder) {
5678 if (Name.starts_with("to.fp16")) {
5679 Value *Cast =
5680 Builder.CreateFPTrunc(CI->getArgOperand(0), Builder.getHalfTy());
5681 return Builder.CreateBitCast(Cast, CI->getType());
5682 }
5683
5684 if (Name.starts_with("from.fp16")) {
5685 Value *Cast =
5686 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
5687 return Builder.CreateFPExt(Cast, CI->getType());
5688 }
5689
5690 return nullptr;
5691}
5692
5693static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op) {
5694 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5695 if (!MD || !isa<MDString>(MD))
5696 return ICmpInst::BAD_ICMP_PREDICATE;
5697 return StringSwitch<ICmpInst::Predicate>(cast<MDString>(MD)->getString())
5698 .Case("eq", ICmpInst::ICMP_EQ)
5699 .Case("ne", ICmpInst::ICMP_NE)
5700 .Case("ugt", ICmpInst::ICMP_UGT)
5701 .Case("uge", ICmpInst::ICMP_UGE)
5702 .Case("ult", ICmpInst::ICMP_ULT)
5703 .Case("ule", ICmpInst::ICMP_ULE)
5704 .Case("sgt", ICmpInst::ICMP_SGT)
5705 .Case("sge", ICmpInst::ICMP_SGE)
5706 .Case("slt", ICmpInst::ICMP_SLT)
5707 .Case("sle", ICmpInst::ICMP_SLE)
5708 .Default(ICmpInst::BAD_ICMP_PREDICATE);
5709}
5710
5711static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op) {
5712 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5713 if (!MD || !isa<MDString>(MD))
5714 return FCmpInst::BAD_FCMP_PREDICATE;
5715 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(MD)->getString())
5716 .Case("oeq", FCmpInst::FCMP_OEQ)
5717 .Case("ogt", FCmpInst::FCMP_OGT)
5718 .Case("oge", FCmpInst::FCMP_OGE)
5719 .Case("olt", FCmpInst::FCMP_OLT)
5720 .Case("ole", FCmpInst::FCMP_OLE)
5721 .Case("one", FCmpInst::FCMP_ONE)
5722 .Case("ord", FCmpInst::FCMP_ORD)
5723 .Case("uno", FCmpInst::FCMP_UNO)
5724 .Case("ueq", FCmpInst::FCMP_UEQ)
5725 .Case("ugt", FCmpInst::FCMP_UGT)
5726 .Case("uge", FCmpInst::FCMP_UGE)
5727 .Case("ult", FCmpInst::FCMP_ULT)
5728 .Case("ule", FCmpInst::FCMP_ULE)
5729 .Case("une", FCmpInst::FCMP_UNE)
5730 .Default(FCmpInst::BAD_FCMP_PREDICATE);
5731}
5732
5733static Value *upgradeVPIntrinsicCall(StringRef Name, CallBase *CI,
5734 IRBuilder<> &Builder) {
5735 Value *Rep;
5736 unsigned Opcode = getFunctionalOpcodeForVP(Name);
5737 if (Opcode && Instruction::isUnaryOp(Opcode))
5738 Rep =
5739 Builder.CreateUnOp((Instruction::UnaryOps)Opcode, CI->getArgOperand(0));
5740 else if (Opcode && Instruction::isBinaryOp(Opcode))
5741 Rep = Builder.CreateBinOp((Instruction::BinaryOps)Opcode,
5742 CI->getArgOperand(0), CI->getArgOperand(1));
5743 else if (Opcode && Instruction::isCast(Opcode))
5744 Rep = Builder.CreateCast((Instruction::CastOps)Opcode, CI->getArgOperand(0),
5745 CI->getType());
5746 else if (Opcode == Instruction::ICmp)
5747 Rep = Builder.CreateICmp(getVPIntPredicateFromMD(CI->getArgOperand(2)),
5748 CI->getArgOperand(0), CI->getArgOperand(1));
5749 else if (Opcode == Instruction::FCmp)
5750 Rep = Builder.CreateFCmp(getVPFPPredicateFromMD(CI->getArgOperand(2)),
5751 CI->getArgOperand(0), CI->getArgOperand(1));
5752 else if (Opcode == Instruction::Select)
5753 Rep = Builder.CreateSelect(CI->getArgOperand(0), CI->getArgOperand(1),
5754 CI->getArgOperand(2));
5755 else if (auto IntrinsicID = getFunctionalIntrinsicIDForVP(Name)) {
5756 SmallVector<Value *, 2> Args(drop_end(CI->args(), 2));
5757 Rep = Builder.CreateIntrinsic(CI->getType(), IntrinsicID, Args, {});
5758 } else
5759 llvm_unreachable("Unexpected vp intrinsic");
5760 Rep->takeName(CI);
5761 return Rep;
5762}
5763
5764static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn,
5765 IRBuilder<> &Builder) {
5766 Intrinsic::ID IID = NewFn->getIntrinsicID();
5767
5768 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
5769 if (Defaults.empty())
5770 return false;
5771
5772 unsigned OldArgCount = CI->arg_size();
5773 unsigned NewArgCount = NewFn->arg_size();
5774
5775 if (OldArgCount < FirstDefault)
5776 return false;
5777
5778 // More arguments than the new intrinsic accepts, cannot upgrade.
5779 if (OldArgCount > NewArgCount)
5780 return false;
5781
5782 // The call already passes the defaulted arguments explicitly; only the
5783 // callee is still the old, shorter declaration, so retarget it.
5784 if (OldArgCount == NewArgCount) {
5785 if (CI->getFunctionType() != NewFn->getFunctionType())
5786 return false;
5787 CI->setCalledFunction(NewFn);
5788 return true;
5789 }
5790
5791 // OldArgCount < NewArgCount: Fill in each missing trailing default
5792 // argument from the table.
5793 SmallVector<Value *, 8> NewArgs(CI->args());
5794
5795 FunctionType *NewFT = NewFn->getFunctionType();
5796 for (unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5797 assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5798 "missing argument outside the default range");
5799 Type *ParamTy = NewFT->getParamType(Idx);
5800
5801 // Only integer types are supported (i1, i8, i16, i32, i64).
5802 if (!ParamTy->isIntegerTy())
5803 return false;
5804 NewArgs.push_back(ConstantInt::get(ParamTy, Defaults[Idx - FirstDefault]));
5805 }
5806
5807 // Preserve operand bundles by creating the call with them.
5808 SmallVector<OperandBundleDef, 1> OpBundles;
5809 CI->getOperandBundlesAsDefs(OpBundles);
5810 CallInst *NewCall = Builder.CreateCall(NewFn, NewArgs, OpBundles);
5811
5812 NewCall->takeName(CI);
5813 NewCall->setCallingConv(CI->getCallingConv());
5814 NewCall->copyMetadata(*CI);
5815 if (auto *OldCI = dyn_cast<CallInst>(CI))
5816 NewCall->setTailCallKind(OldCI->getTailCallKind());
5817
5818 CI->replaceAllUsesWith(NewCall);
5819 CI->eraseFromParent();
5820 return true;
5821}
5822
5823/// Upgrade a call to an old intrinsic. All argument and return casting must be
5824/// provided to seamlessly integrate with existing context.
5825void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
5826 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
5827 // checks the callee's function type matches. It's likely we need to handle
5828 // type changes here.
5829 Function *F = dyn_cast<Function>(CI->getCalledOperand());
5830 if (!F)
5831 return;
5832
5833 LLVMContext &C = CI->getContext();
5834 IRBuilder<> Builder(CI->getIterator());
5835 if (isa<FPMathOperator>(CI))
5836 Builder.setFastMathFlags(CI->getFastMathFlags());
5837
5838 if (!NewFn) {
5839 // Get the Function's name.
5840 StringRef Name = F->getName();
5841 if (!Name.consume_front("llvm."))
5842 llvm_unreachable("intrinsic doesn't start with 'llvm.'");
5843
5844 bool IsX86 = Name.consume_front("x86.");
5845 bool IsNVVM = Name.consume_front("nvvm.");
5846 bool IsAArch64 = Name.consume_front("aarch64.");
5847 bool IsARM = Name.consume_front("arm.");
5848 bool IsAMDGCN = Name.consume_front("amdgcn.");
5849 bool IsDbg = Name.consume_front("dbg.");
5850 bool IsOldSplice =
5851 (Name.consume_front("experimental.vector.splice") ||
5852 Name.consume_front("vector.splice")) &&
5853 !(Name.starts_with(".left") || Name.starts_with(".right"));
5854 Value *Rep = nullptr;
5855
5856 if (!IsX86 && Name == "stackprotectorcheck") {
5857 Rep = nullptr;
5858 } else if (IsNVVM) {
5859 Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5860 } else if (IsX86) {
5861 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5862 } else if (IsAArch64) {
5863 Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5864 } else if (IsARM) {
5865 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5866 } else if (IsAMDGCN) {
5867 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5868 } else if (IsDbg) {
5869 upgradeDbgIntrinsicToDbgRecord(Name, CI);
5870 } else if (IsOldSplice) {
5871 Rep = upgradeVectorSplice(CI, Builder);
5872 } else if (Name.consume_front("convert.")) {
5873 Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5874 } else if (Name == "lifetime.start.i64" || Name == "lifetime.end.i64") {
5875 // Delete calls to invalid @llvm.lifetime.{start,end}.i64 intrinsics.
5876 Rep = nullptr;
5877 } else if (shouldUpgradeVPIntrinsic(Name)) {
5878 Rep = upgradeVPIntrinsicCall(Name, CI, Builder);
5879 } else {
5880 llvm_unreachable("Unknown function for CallBase upgrade.");
5881 }
5882
5883 if (Rep)
5884 CI->replaceAllUsesWith(Rep);
5885 CI->eraseFromParent();
5886 return;
5887 }
5888
5889 const auto &DefaultCase = [&]() -> void {
5890 if (F == NewFn)
5891 return;
5892
5893 if (CI->getFunctionType() == NewFn->getFunctionType()) {
5894 // Handle generic mangling change.
5895 assert(
5896 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
5897 "Unknown function for CallBase upgrade and isn't just a name change");
5898 CI->setCalledFunction(NewFn);
5899 return;
5900 }
5901
5902 // This must be an upgrade from a named to a literal struct.
5903 if (auto *OldST = dyn_cast<StructType>(CI->getType())) {
5904 assert(OldST != NewFn->getReturnType() &&
5905 "Return type must have changed");
5906 assert(OldST->getNumElements() ==
5907 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5908 "Must have same number of elements");
5909
5910 SmallVector<Value *> Args(CI->args());
5911 CallInst *NewCI = Builder.CreateCall(NewFn, Args);
5912 NewCI->setAttributes(CI->getAttributes());
5913 Value *Res = PoisonValue::get(OldST);
5914 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5915 Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
5916 Res = Builder.CreateInsertValue(Res, Elem, Idx);
5917 }
5918 CI->replaceAllUsesWith(Res);
5919 CI->eraseFromParent();
5920 return;
5921 }
5922
5923 // We're probably about to produce something invalid. Let the verifier catch
5924 // it instead of dying here.
5925 CI->setCalledOperand(
5926 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
5927 return;
5928 };
5929 CallInst *NewCall = nullptr;
5930 switch (NewFn->getIntrinsicID()) {
5931 default: {
5932 if (upgradeIntrinsicCallWithDefaultArgs(CI, NewFn, Builder))
5933 return;
5934 DefaultCase();
5935 return;
5936 }
5937 case Intrinsic::arm_neon_vst1:
5938 case Intrinsic::arm_neon_vst2:
5939 case Intrinsic::arm_neon_vst3:
5940 case Intrinsic::arm_neon_vst4:
5941 case Intrinsic::arm_neon_vst2lane:
5942 case Intrinsic::arm_neon_vst3lane:
5943 case Intrinsic::arm_neon_vst4lane: {
5944 SmallVector<Value *, 4> Args(CI->args());
5945 NewCall = Builder.CreateCall(NewFn, Args);
5946 break;
5947 }
5948 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5949 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5950 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5951 LLVMContext &Ctx = F->getParent()->getContext();
5952 SmallVector<Value *, 4> Args(CI->args());
5953 Args[3] = ConstantInt::get(Type::getInt32Ty(Ctx),
5954 cast<ConstantInt>(Args[3])->getZExtValue());
5955 NewCall = Builder.CreateCall(NewFn, Args);
5956 break;
5957 }
5958 case Intrinsic::aarch64_sve_ld3_sret:
5959 case Intrinsic::aarch64_sve_ld4_sret:
5960 case Intrinsic::aarch64_sve_ld2_sret: {
5961 // Is this a trivial remangle of the name to support ptr address spaces?
5962 if (isa<StructType>(F->getReturnType())) {
5963 DefaultCase();
5964 return;
5965 }
5966
5967 StringRef Name = F->getName();
5968 Name = Name.substr(5);
5969 unsigned N = StringSwitch<unsigned>(Name)
5970 .StartsWith("aarch64.sve.ld2", 2)
5971 .StartsWith("aarch64.sve.ld3", 3)
5972 .StartsWith("aarch64.sve.ld4", 4)
5973 .Default(0);
5974 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5975 unsigned MinElts = RetTy->getMinNumElements() / N;
5976 SmallVector<Value *, 2> Args(CI->args());
5977 Value *NewLdCall = Builder.CreateCall(NewFn, Args);
5978 Value *Ret = llvm::PoisonValue::get(RetTy);
5979 for (unsigned I = 0; I < N; I++) {
5980 Value *SRet = Builder.CreateExtractValue(NewLdCall, I);
5981 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, I * MinElts);
5982 }
5983 NewCall = dyn_cast<CallInst>(Ret);
5984 break;
5985 }
5986
5987 case Intrinsic::coro_end_async:
5988 case Intrinsic::coro_end: {
5989 SmallVector<Value *, 3> Args(CI->args());
5990 if (NewFn->getIntrinsicID() == Intrinsic::coro_end && Args.size() == 2)
5991 Args.push_back(ConstantTokenNone::get(CI->getContext()));
5992 NewCall = Builder.CreateCall(NewFn, Args);
5993
5994 if (!CI->getType()->isVoidTy()) {
5995 if (!CI->use_empty()) {
5996 Function *IsInRamp = Intrinsic::getOrInsertDeclaration(
5997 CI->getModule(), Intrinsic::coro_is_in_ramp);
5998 Value *InRamp = Builder.CreateCall(IsInRamp);
5999 CI->replaceAllUsesWith(Builder.CreateNot(InRamp));
6000 }
6001 CI->eraseFromParent();
6002 return;
6003 }
6004
6005 break;
6006 }
6007
6008 case Intrinsic::vector_extract: {
6009 StringRef Name = F->getName();
6010 Name = Name.substr(5); // Strip llvm
6011 if (!Name.starts_with("aarch64.sve.tuple.get")) {
6012 DefaultCase();
6013 return;
6014 }
6015 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
6016 unsigned MinElts = RetTy->getMinNumElements();
6017 uint64_t I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
6018 Value *NewIdx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
6019 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});
6020 break;
6021 }
6022
6023 case Intrinsic::vector_insert: {
6024 StringRef Name = F->getName();
6025 Name = Name.substr(5);
6026 if (!Name.starts_with("aarch64.sve.tuple")) {
6027 DefaultCase();
6028 return;
6029 }
6030 if (Name.starts_with("aarch64.sve.tuple.set")) {
6031 uint64_t I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
6032 auto *Ty = cast<ScalableVectorType>(CI->getArgOperand(2)->getType());
6033 Value *NewIdx =
6034 ConstantInt::get(Type::getInt64Ty(C), I * Ty->getMinNumElements());
6035 NewCall = Builder.CreateCall(
6036 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx});
6037 break;
6038 }
6039 if (Name.starts_with("aarch64.sve.tuple.create")) {
6040 unsigned N = StringSwitch<unsigned>(Name)
6041 .StartsWith("aarch64.sve.tuple.create2", 2)
6042 .StartsWith("aarch64.sve.tuple.create3", 3)
6043 .StartsWith("aarch64.sve.tuple.create4", 4)
6044 .Default(0);
6045 assert(N > 1 && "Create is expected to be between 2-4");
6046 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
6047 Value *Ret = llvm::PoisonValue::get(RetTy);
6048 unsigned MinElts = RetTy->getMinNumElements() / N;
6049 for (unsigned I = 0; I < N; I++) {
6050 Value *V = CI->getArgOperand(I);
6051 Ret = Builder.CreateInsertVector(RetTy, Ret, V, I * MinElts);
6052 }
6053 NewCall = dyn_cast<CallInst>(Ret);
6054 }
6055 break;
6056 }
6057
6058 case Intrinsic::arm_neon_bfdot:
6059 case Intrinsic::arm_neon_bfmmla:
6060 case Intrinsic::arm_neon_bfmlalb:
6061 case Intrinsic::arm_neon_bfmlalt:
6062 case Intrinsic::aarch64_neon_bfdot:
6063 case Intrinsic::aarch64_neon_bfmmla:
6064 case Intrinsic::aarch64_neon_bfmlalb:
6065 case Intrinsic::aarch64_neon_bfmlalt: {
6066 SmallVector<Value *, 3> Args;
6067 assert(CI->arg_size() == 3 &&
6068 "Mismatch between function args and call args");
6069 size_t OperandWidth =
6070 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
6071 assert((OperandWidth == 64 || OperandWidth == 128) &&
6072 "Unexpected operand width");
6073 Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
6074 auto Iter = CI->args().begin();
6075 Args.push_back(*Iter++);
6076 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
6077 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
6078 NewCall = Builder.CreateCall(NewFn, Args);
6079 break;
6080 }
6081
6082 case Intrinsic::bitreverse:
6083 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
6084 break;
6085
6086 case Intrinsic::ctlz:
6087 case Intrinsic::cttz: {
6088 if (CI->arg_size() != 1) {
6089 DefaultCase();
6090 return;
6091 }
6092
6093 NewCall =
6094 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
6095 break;
6096 }
6097
6098 case Intrinsic::objectsize: {
6099 Value *NullIsUnknownSize =
6100 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2);
6101 Value *Dynamic =
6102 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
6103 NewCall = Builder.CreateCall(
6104 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
6105 break;
6106 }
6107
6108 case Intrinsic::ctpop:
6109 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
6110 break;
6111 case Intrinsic::dbg_value: {
6112 StringRef Name = F->getName();
6113 Name = Name.substr(5); // Strip llvm.
6114 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
6115 if (Name.starts_with("dbg.addr")) {
6116 DIExpression *Expr = cast<DIExpression>(
6117 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata());
6118 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
6119 NewCall =
6120 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
6121 MetadataAsValue::get(C, Expr)});
6122 break;
6123 }
6124
6125 // Upgrade from the old version that had an extra offset argument.
6126 assert(CI->arg_size() == 4);
6127 // Drop nonzero offsets instead of attempting to upgrade them.
6128 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
6129 if (Offset->isNullValue()) {
6130 NewCall = Builder.CreateCall(
6131 NewFn,
6132 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
6133 break;
6134 }
6135 CI->eraseFromParent();
6136 return;
6137 }
6138
6139 case Intrinsic::ptr_annotation:
6140 // Upgrade from versions that lacked the annotation attribute argument.
6141 if (CI->arg_size() != 4) {
6142 DefaultCase();
6143 return;
6144 }
6145
6146 // Create a new call with an added null annotation attribute argument.
6147 NewCall = Builder.CreateCall(
6148 NewFn,
6149 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
6150 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
6151 NewCall->takeName(CI);
6152 CI->replaceAllUsesWith(NewCall);
6153 CI->eraseFromParent();
6154 return;
6155
6156 case Intrinsic::var_annotation:
6157 // Upgrade from versions that lacked the annotation attribute argument.
6158 if (CI->arg_size() != 4) {
6159 DefaultCase();
6160 return;
6161 }
6162 // Create a new call with an added null annotation attribute argument.
6163 NewCall = Builder.CreateCall(
6164 NewFn,
6165 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
6166 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
6167 NewCall->takeName(CI);
6168 CI->replaceAllUsesWith(NewCall);
6169 CI->eraseFromParent();
6170 return;
6171
6172 case Intrinsic::riscv_aes32dsi:
6173 case Intrinsic::riscv_aes32dsmi:
6174 case Intrinsic::riscv_aes32esi:
6175 case Intrinsic::riscv_aes32esmi:
6176 case Intrinsic::riscv_sm4ks:
6177 case Intrinsic::riscv_sm4ed: {
6178 // The last argument to these intrinsics used to be i8 and changed to i32.
6179 // The type overload for sm4ks and sm4ed was removed.
6180 Value *Arg2 = CI->getArgOperand(2);
6181 if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64))
6182 return;
6183
6184 Value *Arg0 = CI->getArgOperand(0);
6185 Value *Arg1 = CI->getArgOperand(1);
6186 if (CI->getType()->isIntegerTy(64)) {
6187 Arg0 = Builder.CreateTrunc(Arg0, Builder.getInt32Ty());
6188 Arg1 = Builder.CreateTrunc(Arg1, Builder.getInt32Ty());
6189 }
6190
6191 Arg2 = ConstantInt::get(Type::getInt32Ty(C),
6192 cast<ConstantInt>(Arg2)->getZExtValue());
6193
6194 NewCall = Builder.CreateCall(NewFn, {Arg0, Arg1, Arg2});
6195 Value *Res = NewCall;
6196 if (Res->getType() != CI->getType())
6197 Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6198 NewCall->takeName(CI);
6199 CI->replaceAllUsesWith(Res);
6200 CI->eraseFromParent();
6201 return;
6202 }
6203 case Intrinsic::nvvm_mapa_shared_cluster: {
6204 // Create a new call with the correct address space.
6205 NewCall =
6206 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1)});
6207 Value *Res = NewCall;
6208 Res = Builder.CreateAddrSpaceCast(
6209 Res, Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED));
6210 NewCall->takeName(CI);
6211 CI->replaceAllUsesWith(Res);
6212 CI->eraseFromParent();
6213 return;
6214 }
6215 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster: {
6216 SmallVector<Value *, 4> Args(CI->args());
6217 unsigned AS = Args[0]->getType()->getPointerAddressSpace();
6218 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6219 Args[0] = Builder.CreateAddrSpaceCast(
6220 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6221
6222 // Append the missing trailing flag_valid_pattern (0 = disabled).
6223 Args.push_back(Builder.getInt32(0));
6224
6225 NewCall = Builder.CreateCall(NewFn, Args);
6226 NewCall->takeName(CI);
6227 CI->replaceAllUsesWith(NewCall);
6228 CI->eraseFromParent();
6229 return;
6230 }
6231 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta: {
6232 // (dst, mbar, src, size, ch, flag_ch)
6233 // -> (dst, mbar, src, size, i32 0, i32 0, ch, flag_ch, i1 false,
6234 // i32 0 /* flag_valid_pattern=disabled */)
6235 SmallVector<Value *, 10> Args;
6236 for (unsigned I = 0; I < 4; ++I)
6237 Args.push_back(CI->getArgOperand(I));
6238 Args.push_back(Builder.getInt32(0)); // ignore_bytes_left
6239 Args.push_back(Builder.getInt32(0)); // ignore_bytes_right
6240 Args.push_back(CI->getArgOperand(4)); // cache_hint
6241 Args.push_back(CI->getArgOperand(5)); // flag_ch
6242 Args.push_back(Builder.getInt1(false)); // flag_oob
6243 Args.push_back(Builder.getInt32(0)); // flag_valid_pattern
6244
6245 NewCall = Builder.CreateCall(NewFn, Args);
6246 NewCall->takeName(CI);
6247 CI->replaceAllUsesWith(NewCall);
6248 CI->eraseFromParent();
6249 return;
6250 }
6251 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
6252 // Create a new call with the correct address space.
6253 SmallVector<Value *, 4> Args(CI->args());
6254 Args[0] = Builder.CreateAddrSpaceCast(
6255 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6256
6257 NewCall = Builder.CreateCall(NewFn, Args);
6258 NewCall->takeName(CI);
6259 CI->replaceAllUsesWith(NewCall);
6260 CI->eraseFromParent();
6261 return;
6262 }
6263 // clang-format off
6264#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME) \
6265 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX:
6266 NVVM_TMA_G2S_MODES(G2S_CLUSTER_CASE)
6267#undef G2S_CLUSTER_CASE
6268 {
6269 SmallVector<Value *, 16> Args(CI->args());
6270 unsigned AS = CI->getArgOperand(0)->getType()->getPointerAddressSpace();
6271 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6272 Args[0] = Builder.CreateAddrSpaceCast(
6273 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6274
6275 // Append the missing trailing arguments with default values (cta_group,
6276 // flag_valid_pattern).
6277 while (Args.size() < NewFn->getFunctionType()->getNumParams())
6278 Args.push_back(Builder.getInt32(0));
6279
6280 NewCall = Builder.CreateCall(NewFn, Args);
6281 NewCall->takeName(CI);
6282 CI->replaceAllUsesWith(NewCall);
6283 CI->eraseFromParent();
6284 return;
6285 }
6286
6287#define G2S_CTA_CASE(ID_SUFFIX, NAME) \
6288 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX:
6289 NVVM_TMA_G2S_MODES(G2S_CTA_CASE)
6290#undef G2S_CTA_CASE
6291 {
6292 SmallVector<Value *, 16> Args(CI->args());
6293 // Append the missing trailing flag_valid_pattern argument with default
6294 // value 0.
6295 assert(Args.size() + 1 == NewFn->getFunctionType()->getNumParams() &&
6296 "expected only the trailing flag_valid_pattern to be missing");
6297 Args.push_back(Builder.getInt32(0));
6298
6299 NewCall = Builder.CreateCall(NewFn, Args);
6300 NewCall->takeName(CI);
6301 CI->replaceAllUsesWith(NewCall);
6302 CI->eraseFromParent();
6303 return;
6304 }
6305#undef NVVM_TMA_G2S_MODES
6306 // clang-format on
6307
6308 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d:
6309 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d:
6310 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d:
6311 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d:
6312 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d:
6313 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d:
6314 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d:
6315 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d: {
6316 StringRef Name = F->getName();
6317 Name.consume_front("llvm.nvvm.cp.async.bulk.tensor.reduce.");
6318 auto RedOp = getNVPTXTMAReductionOp(Name.split('.').first);
6319
6320 SmallVector<Value *, 16> Args(CI->args());
6321 Args.insert(Args.end() - 1, Builder.getInt32(*RedOp));
6322 NewCall = Builder.CreateCall(NewFn, Args);
6323 break;
6324 }
6325 case Intrinsic::nvvm_mbarrier_init: {
6326 SmallVector<Value *, 3> Args(CI->args());
6327 // The .shared variant folded into the overloaded form without gaining an
6328 // operand, so only the pre-layout two-argument form needs one appended.
6329 if (Args.size() == 2)
6330 Args.push_back(Builder.getInt32(0)); // layout = default(0)
6331 NewCall = Builder.CreateCall(NewFn, Args);
6332 break;
6333 }
6334 case Intrinsic::riscv_sha256sig0:
6335 case Intrinsic::riscv_sha256sig1:
6336 case Intrinsic::riscv_sha256sum0:
6337 case Intrinsic::riscv_sha256sum1:
6338 case Intrinsic::riscv_sm3p0:
6339 case Intrinsic::riscv_sm3p1: {
6340 // The last argument to these intrinsics used to be i8 and changed to i32.
6341 // The type overload for sm4ks and sm4ed was removed.
6342 if (!CI->getType()->isIntegerTy(64))
6343 return;
6344
6345 Value *Arg =
6346 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty());
6347
6348 NewCall = Builder.CreateCall(NewFn, Arg);
6349 Value *Res =
6350 Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6351 NewCall->takeName(CI);
6352 CI->replaceAllUsesWith(Res);
6353 CI->eraseFromParent();
6354 return;
6355 }
6356
6357 case Intrinsic::x86_xop_vfrcz_ss:
6358 case Intrinsic::x86_xop_vfrcz_sd:
6359 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
6360 break;
6361
6362 case Intrinsic::x86_xop_vpermil2pd:
6363 case Intrinsic::x86_xop_vpermil2ps:
6364 case Intrinsic::x86_xop_vpermil2pd_256:
6365 case Intrinsic::x86_xop_vpermil2ps_256: {
6366 SmallVector<Value *, 4> Args(CI->args());
6367 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
6368 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
6369 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
6370 NewCall = Builder.CreateCall(NewFn, Args);
6371 break;
6372 }
6373
6374 case Intrinsic::x86_sse41_ptestc:
6375 case Intrinsic::x86_sse41_ptestz:
6376 case Intrinsic::x86_sse41_ptestnzc: {
6377 // The arguments for these intrinsics used to be v4f32, and changed
6378 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
6379 // So, the only thing required is a bitcast for both arguments.
6380 // First, check the arguments have the old type.
6381 Value *Arg0 = CI->getArgOperand(0);
6382 if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
6383 return;
6384
6385 // Old intrinsic, add bitcasts
6386 Value *Arg1 = CI->getArgOperand(1);
6387
6388 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
6389
6390 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
6391 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
6392
6393 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
6394 break;
6395 }
6396
6397 case Intrinsic::x86_rdtscp: {
6398 // This used to take 1 arguments. If we have no arguments, it is already
6399 // upgraded.
6400 if (CI->getNumOperands() == 0)
6401 return;
6402
6403 NewCall = Builder.CreateCall(NewFn);
6404 // Extract the second result and store it.
6405 Value *Data = Builder.CreateExtractValue(NewCall, 1);
6406 Builder.CreateAlignedStore(Data, CI->getArgOperand(0), Align(1));
6407 // Replace the original call result with the first result of the new call.
6408 Value *TSC = Builder.CreateExtractValue(NewCall, 0);
6409
6410 NewCall->takeName(CI);
6411 CI->replaceAllUsesWith(TSC);
6412 CI->eraseFromParent();
6413 return;
6414 }
6415
6416 case Intrinsic::x86_sse41_insertps:
6417 case Intrinsic::x86_sse41_dppd:
6418 case Intrinsic::x86_sse41_dpps:
6419 case Intrinsic::x86_sse41_mpsadbw:
6420 case Intrinsic::x86_avx_dp_ps_256:
6421 case Intrinsic::x86_avx2_mpsadbw: {
6422 // Need to truncate the last argument from i32 to i8 -- this argument models
6423 // an inherently 8-bit immediate operand to these x86 instructions.
6424 SmallVector<Value *, 4> Args(CI->args());
6425
6426 // Replace the last argument with a trunc.
6427 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
6428 NewCall = Builder.CreateCall(NewFn, Args);
6429 break;
6430 }
6431
6432 case Intrinsic::x86_avx512_mask_cmp_pd_128:
6433 case Intrinsic::x86_avx512_mask_cmp_pd_256:
6434 case Intrinsic::x86_avx512_mask_cmp_pd_512:
6435 case Intrinsic::x86_avx512_mask_cmp_ps_128:
6436 case Intrinsic::x86_avx512_mask_cmp_ps_256:
6437 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
6438 SmallVector<Value *, 4> Args(CI->args());
6439 unsigned NumElts =
6440 cast<FixedVectorType>(Args[0]->getType())->getNumElements();
6441 Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
6442
6443 NewCall = Builder.CreateCall(NewFn, Args);
6444 Value *Res = applyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
6445
6446 NewCall->takeName(CI);
6447 CI->replaceAllUsesWith(Res);
6448 CI->eraseFromParent();
6449 return;
6450 }
6451
6452 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
6453 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
6454 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
6455 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
6456 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
6457 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
6458 SmallVector<Value *, 4> Args(CI->args());
6459 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
6460 if (NewFn->getIntrinsicID() ==
6461 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
6462 Args[1] = Builder.CreateBitCast(
6463 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6464
6465 NewCall = Builder.CreateCall(NewFn, Args);
6466 Value *Res = Builder.CreateBitCast(
6467 NewCall, FixedVectorType::get(Builder.getInt16Ty(), NumElts));
6468
6469 NewCall->takeName(CI);
6470 CI->replaceAllUsesWith(Res);
6471 CI->eraseFromParent();
6472 return;
6473 }
6474 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6475 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6476 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
6477 SmallVector<Value *, 4> Args(CI->args());
6478 unsigned NumElts =
6479 cast<FixedVectorType>(CI->getType())->getNumElements() * 2;
6480 Args[1] = Builder.CreateBitCast(
6481 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6482 Args[2] = Builder.CreateBitCast(
6483 Args[2], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6484
6485 NewCall = Builder.CreateCall(NewFn, Args);
6486 break;
6487 }
6488
6489 case Intrinsic::amdgcn_asyncmark:
6490 case Intrinsic::amdgcn_wait_asyncmark: {
6491 // Legacy asyncmark intrinsics missed the stage mask operand. Append a zero
6492 // mask, which names every stage and so preserves the behavior they had.
6493 SmallVector<Value *, 2> Args(CI->args());
6494 Args.push_back(Builder.getInt32(0));
6495
6496 SmallVector<OperandBundleDef, 1> Bundles;
6497 CI->getOperandBundlesAsDefs(Bundles);
6498
6499 NewCall = Builder.CreateCall(NewFn, Args, Bundles);
6500 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6501 NewCall->setCallingConv(CI->getCallingConv());
6502 NewCall->setAttributes(CI->getAttributes());
6503 NewCall->copyMetadata(*CI);
6504 break;
6505 }
6506
6507 case Intrinsic::thread_pointer: {
6508 NewCall = Builder.CreateCall(NewFn, {});
6509 break;
6510 }
6511
6512 case Intrinsic::memcpy:
6513 case Intrinsic::memmove:
6514 case Intrinsic::memset: {
6515 // We have to make sure that the call signature is what we're expecting.
6516 // We only want to change the old signatures by removing the alignment arg:
6517 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
6518 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
6519 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
6520 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
6521 // Note: i8*'s in the above can be any pointer type
6522 if (CI->arg_size() != 5) {
6523 DefaultCase();
6524 return;
6525 }
6526 // Remove alignment argument (3), and add alignment attributes to the
6527 // dest/src pointers.
6528 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
6529 CI->getArgOperand(2), CI->getArgOperand(4)};
6530 NewCall = Builder.CreateCall(NewFn, Args);
6531 AttributeList OldAttrs = CI->getAttributes();
6532 AttributeList NewAttrs = AttributeList::get(
6533 C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),
6534 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
6535 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
6536 NewCall->setAttributes(NewAttrs);
6537 auto *MemCI = cast<MemIntrinsic>(NewCall);
6538 // All mem intrinsics support dest alignment.
6539 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
6540 MemCI->setDestAlignment(Align->getMaybeAlignValue());
6541 // Memcpy/Memmove also support source alignment.
6542 if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
6543 MTI->setSourceAlignment(Align->getMaybeAlignValue());
6544 break;
6545 }
6546
6547 case Intrinsic::masked_load:
6548 case Intrinsic::masked_gather:
6549 case Intrinsic::masked_store:
6550 case Intrinsic::masked_scatter: {
6551 if (CI->arg_size() != 4) {
6552 DefaultCase();
6553 return;
6554 }
6555
6556 auto GetMaybeAlign = [](Value *Op) {
6557 if (auto *CI = dyn_cast<ConstantInt>(Op)) {
6558 uint64_t Val = CI->getZExtValue();
6559 if (Val == 0)
6560 return MaybeAlign();
6561 if (isPowerOf2_64(Val))
6562 return MaybeAlign(Val);
6563 }
6564 reportFatalUsageError("Invalid alignment argument");
6565 };
6566 auto GetAlign = [&](Value *Op) {
6567 MaybeAlign Align = GetMaybeAlign(Op);
6568 if (Align)
6569 return *Align;
6570 reportFatalUsageError("Invalid zero alignment argument");
6571 };
6572
6573 const DataLayout &DL = CI->getDataLayout();
6574 switch (NewFn->getIntrinsicID()) {
6575 case Intrinsic::masked_load:
6576 NewCall = Builder.CreateMaskedLoad(
6577 CI->getType(), CI->getArgOperand(0), GetAlign(CI->getArgOperand(1)),
6578 CI->getArgOperand(2), CI->getArgOperand(3));
6579 break;
6580 case Intrinsic::masked_gather:
6581 NewCall = Builder.CreateMaskedGather(
6582 CI->getType(), CI->getArgOperand(0),
6583 DL.getValueOrABITypeAlignment(GetMaybeAlign(CI->getArgOperand(1)),
6584 CI->getType()->getScalarType()),
6585 CI->getArgOperand(2), CI->getArgOperand(3));
6586 break;
6587 case Intrinsic::masked_store:
6588 NewCall = Builder.CreateMaskedStore(
6589 CI->getArgOperand(0), CI->getArgOperand(1),
6590 GetAlign(CI->getArgOperand(2)), CI->getArgOperand(3));
6591 break;
6592 case Intrinsic::masked_scatter:
6593 NewCall = Builder.CreateMaskedScatter(
6594 CI->getArgOperand(0), CI->getArgOperand(1),
6595 DL.getValueOrABITypeAlignment(
6596 GetMaybeAlign(CI->getArgOperand(2)),
6597 CI->getArgOperand(0)->getType()->getScalarType()),
6598 CI->getArgOperand(3));
6599 break;
6600 default:
6601 llvm_unreachable("Unexpected intrinsic ID");
6602 }
6603 // Previous metadata is still valid.
6604 NewCall->copyMetadata(*CI);
6605 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6606 break;
6607 }
6608
6609 case Intrinsic::lifetime_start:
6610 case Intrinsic::lifetime_end: {
6611 if (CI->arg_size() != 2) {
6612 DefaultCase();
6613 return;
6614 }
6615
6616 Value *Ptr = CI->getArgOperand(1);
6617 // Try to strip pointer casts, such that the lifetime works on an alloca.
6618 Ptr = Ptr->stripPointerCasts();
6619 if (isa<AllocaInst>(Ptr)) {
6620 // Don't use NewFn, as we might have looked through an addrspacecast.
6621 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
6622 NewCall = Builder.CreateLifetimeStart(Ptr);
6623 else
6624 NewCall = Builder.CreateLifetimeEnd(Ptr);
6625 break;
6626 }
6627
6628 // Otherwise remove the lifetime marker.
6629 CI->eraseFromParent();
6630 return;
6631 }
6632
6633 case Intrinsic::x86_avx512_vpdpbusd_128:
6634 case Intrinsic::x86_avx512_vpdpbusd_256:
6635 case Intrinsic::x86_avx512_vpdpbusd_512:
6636 case Intrinsic::x86_avx512_vpdpbusds_128:
6637 case Intrinsic::x86_avx512_vpdpbusds_256:
6638 case Intrinsic::x86_avx512_vpdpbusds_512:
6639 case Intrinsic::x86_avx2_vpdpbssd_128:
6640 case Intrinsic::x86_avx2_vpdpbssd_256:
6641 case Intrinsic::x86_avx10_vpdpbssd_512:
6642 case Intrinsic::x86_avx2_vpdpbssds_128:
6643 case Intrinsic::x86_avx2_vpdpbssds_256:
6644 case Intrinsic::x86_avx10_vpdpbssds_512:
6645 case Intrinsic::x86_avx2_vpdpbsud_128:
6646 case Intrinsic::x86_avx2_vpdpbsud_256:
6647 case Intrinsic::x86_avx10_vpdpbsud_512:
6648 case Intrinsic::x86_avx2_vpdpbsuds_128:
6649 case Intrinsic::x86_avx2_vpdpbsuds_256:
6650 case Intrinsic::x86_avx10_vpdpbsuds_512:
6651 case Intrinsic::x86_avx2_vpdpbuud_128:
6652 case Intrinsic::x86_avx2_vpdpbuud_256:
6653 case Intrinsic::x86_avx10_vpdpbuud_512:
6654 case Intrinsic::x86_avx2_vpdpbuuds_128:
6655 case Intrinsic::x86_avx2_vpdpbuuds_256:
6656 case Intrinsic::x86_avx10_vpdpbuuds_512: {
6657 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8;
6658 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6659 CI->getArgOperand(2)};
6660 Type *NewArgType = VectorType::get(Builder.getInt8Ty(), NumElts, false);
6661 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6662 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6663
6664 NewCall = Builder.CreateCall(NewFn, Args);
6665 break;
6666 }
6667 case Intrinsic::x86_avx512_vpdpwssd_128:
6668 case Intrinsic::x86_avx512_vpdpwssd_256:
6669 case Intrinsic::x86_avx512_vpdpwssd_512:
6670 case Intrinsic::x86_avx512_vpdpwssds_128:
6671 case Intrinsic::x86_avx512_vpdpwssds_256:
6672 case Intrinsic::x86_avx512_vpdpwssds_512:
6673 case Intrinsic::x86_avx2_vpdpwsud_128:
6674 case Intrinsic::x86_avx2_vpdpwsud_256:
6675 case Intrinsic::x86_avx10_vpdpwsud_512:
6676 case Intrinsic::x86_avx2_vpdpwsuds_128:
6677 case Intrinsic::x86_avx2_vpdpwsuds_256:
6678 case Intrinsic::x86_avx10_vpdpwsuds_512:
6679 case Intrinsic::x86_avx2_vpdpwusd_128:
6680 case Intrinsic::x86_avx2_vpdpwusd_256:
6681 case Intrinsic::x86_avx10_vpdpwusd_512:
6682 case Intrinsic::x86_avx2_vpdpwusds_128:
6683 case Intrinsic::x86_avx2_vpdpwusds_256:
6684 case Intrinsic::x86_avx10_vpdpwusds_512:
6685 case Intrinsic::x86_avx2_vpdpwuud_128:
6686 case Intrinsic::x86_avx2_vpdpwuud_256:
6687 case Intrinsic::x86_avx10_vpdpwuud_512:
6688 case Intrinsic::x86_avx2_vpdpwuuds_128:
6689 case Intrinsic::x86_avx2_vpdpwuuds_256:
6690 case Intrinsic::x86_avx10_vpdpwuuds_512:
6691 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16;
6692 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6693 CI->getArgOperand(2)};
6694 Type *NewArgType = VectorType::get(Builder.getInt16Ty(), NumElts, false);
6695 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6696 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6697
6698 NewCall = Builder.CreateCall(NewFn, Args);
6699 break;
6700 }
6701 assert(NewCall && "Should have either set this variable or returned through "
6702 "the default case");
6703 NewCall->takeName(CI);
6704 CI->replaceAllUsesWith(NewCall);
6705 CI->eraseFromParent();
6706}
6707
6708void llvm::UpgradeCallsToIntrinsic(Function *F) {
6709 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
6710
6711 // Check if this function should be upgraded and get the replacement function
6712 // if there is one.
6713 Function *NewFn;
6714 if (UpgradeIntrinsicFunction(F, NewFn)) {
6715 // Replace all users of the old function with the new function or new
6716 // instructions. This is not a range loop because the call is deleted.
6717 for (User *U : make_early_inc_range(F->users()))
6718 if (CallBase *CB = dyn_cast<CallBase>(U))
6719 UpgradeIntrinsicCall(CB, NewFn);
6720
6721 // Remove old function, no longer used, from the module.
6722 if (F != NewFn)
6723 F->eraseFromParent();
6724 }
6725}
6726
6727MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
6728 const unsigned NumOperands = MD.getNumOperands();
6729 if (NumOperands == 0)
6730 return &MD; // Invalid, punt to a verifier error.
6731
6732 // Check if the tag uses struct-path aware TBAA format.
6733 if (isa<MDNode>(MD.getOperand(0)) && NumOperands >= 3)
6734 return &MD;
6735
6736 auto &Context = MD.getContext();
6737 if (NumOperands == 3) {
6738 Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
6739 MDNode *ScalarType = MDNode::get(Context, Elts);
6740 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
6741 Metadata *Elts2[] = {ScalarType, ScalarType,
6742 ConstantAsMetadata::get(
6743 Constant::getNullValue(Type::getInt64Ty(Context))),
6744 MD.getOperand(2)};
6745 return MDNode::get(Context, Elts2);
6746 }
6747 // Create a MDNode <MD, MD, offset 0>
6748 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
6749 Type::getInt64Ty(Context)))};
6750 return MDNode::get(Context, Elts);
6751}
6752
6753MDNode *llvm::UpgradeTBAAStructNode(MDNode &MD) {
6754 // !tbaa.struct is a list of (offset, size, tag) triples. Upgrade any
6755 // old-style scalar field tag to struct-path form via UpgradeTBAANode.
6756 unsigned NumOperands = MD.getNumOperands();
6757 if (NumOperands == 0 || NumOperands % 3 != 0)
6758 return &MD; // Malformed; leave it for the verifier to reject.
6759
6760 SmallVector<Metadata *, 12> Elts(MD.op_begin(), MD.op_end());
6761 bool Changed = false;
6762 for (unsigned I = 2; I < NumOperands; I += 3) {
6763 auto *Tag = dyn_cast_or_null<MDNode>(Elts[I]);
6764 if (!Tag)
6765 continue;
6766 MDNode *Upgraded = UpgradeTBAANode(*Tag);
6767 if (Upgraded == Tag)
6768 continue;
6769 Elts[I] = Upgraded;
6770 Changed = true;
6771 }
6772 return Changed ? MDNode::get(MD.getContext(), Elts) : &MD;
6773}
6774
6775Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
6776 Instruction *&Temp) {
6777 if (Opc != Instruction::BitCast)
6778 return nullptr;
6779
6780 Temp = nullptr;
6781 Type *SrcTy = V->getType();
6782 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6783 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6784 LLVMContext &Context = V->getContext();
6785
6786 // We have no information about target data layout, so we assume that
6787 // the maximum pointer size is 64bit.
6788 Type *MidTy = Type::getInt64Ty(Context);
6789 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
6790
6791 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
6792 }
6793
6794 return nullptr;
6795}
6796
6797Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
6798 if (Opc != Instruction::BitCast)
6799 return nullptr;
6800
6801 Type *SrcTy = C->getType();
6802 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6803 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6804 LLVMContext &Context = C->getContext();
6805
6806 // We have no information about target data layout, so we assume that
6807 // the maximum pointer size is 64bit.
6808 Type *MidTy = Type::getInt64Ty(Context);
6809
6810 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
6811 DestTy);
6812 }
6813
6814 return nullptr;
6815}
6816
6817static std::optional<StringRef> getModuleFlagNameSafely(const MDNode &Flag) {
6818 if (Flag.getNumOperands() < 3)
6819 return std::nullopt;
6820 if (MDString *Name = dyn_cast_or_null<MDString>(Flag.getOperand(1)))
6821 return Name->getString();
6822 return std::nullopt;
6823}
6824
6825/// Check the debug info version number, if it is out-dated, drop the debug
6826/// info. Return true if module is modified.
6827bool llvm::UpgradeDebugInfo(Module &M) {
6828 if (!getIROptions(M.getContext()).auto_upgrade_debug_info)
6829 return false;
6830
6831 llvm::TimeTraceScope timeScope("Upgrade debug info");
6832 // We need to get metadata before the module is verified (i.e., getModuleFlag
6833 // makes assumptions that we haven't verified yet). Carefully extract the flag
6834 // from the metadata.
6835 unsigned Version = 0;
6836 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6837 auto OpIt = find_if(ModFlags->operands(), [](const MDNode *Flag) {
6838 if (auto Name = getModuleFlagNameSafely(*Flag))
6839 return *Name == "Debug Info Version";
6840 return false;
6841 });
6842 if (OpIt != ModFlags->op_end()) {
6843 const MDOperand &ValOp = (*OpIt)->getOperand(2);
6844 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(ValOp))
6845 Version = CI->getZExtValue();
6846 }
6847 }
6848
6849 if (Version == DEBUG_METADATA_VERSION) {
6850 bool BrokenDebugInfo = false;
6851 if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
6852 report_fatal_error("Broken module found, compilation aborted!");
6853 if (!BrokenDebugInfo)
6854 // Everything is ok.
6855 return false;
6856 else {
6857 // Diagnose malformed debug info.
6858 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
6859 M.getContext().diagnose(Diag);
6860 }
6861 }
6862 bool Modified = StripDebugInfo(M);
6863 if (Modified && Version != DEBUG_METADATA_VERSION) {
6864 // Diagnose a version mismatch.
6865 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
6866 M.getContext().diagnose(DiagVersion);
6867 }
6868 return Modified;
6869}
6870
6871static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
6872 GlobalValue *GV, const Metadata *V) {
6873 Function *F = cast<Function>(GV);
6874
6875 constexpr StringLiteral DefaultValue = "1";
6876 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
6877 unsigned Length = 0;
6878
6879 if (F->hasFnAttribute(Attr)) {
6880 // We expect the existing attribute to have the form "x[,y[,z]]". Here we
6881 // parse these elements placing them into Vect3
6882 StringRef S = F->getFnAttribute(Attr).getValueAsString();
6883 for (; Length < 3 && !S.empty(); Length++) {
6884 auto [Part, Rest] = S.split(',');
6885 Vect3[Length] = Part.trim();
6886 S = Rest;
6887 }
6888 }
6889
6890 const unsigned Dim = DimC - 'x';
6891 assert(Dim < 3 && "Unexpected dim char");
6892
6893 const uint64_t VInt = mdconst::extract<ConstantInt>(V)->getZExtValue();
6894
6895 // local variable required for StringRef in Vect3 to point to.
6896 const std::string VStr = llvm::utostr(VInt);
6897 Vect3[Dim] = VStr;
6898 Length = std::max(Length, Dim + 1);
6899
6900 const std::string NewAttr = llvm::join(ArrayRef(Vect3, Length), ",");
6901 F->addFnAttr(Attr, NewAttr);
6902}
6903
6904static inline bool isXYZ(StringRef S) {
6905 return S == "x" || S == "y" || S == "z";
6906}
6907
6908bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
6909 const Metadata *V) {
6910 if (K == "kernel") {
6911 if (!mdconst::extract<ConstantInt>(V)->isZero())
6912 cast<Function>(GV)->setCallingConv(CallingConv::PTX_Kernel);
6913 return true;
6914 }
6915 if (K == "align") {
6916 // V is a bitfeild specifying two 16-bit values. The alignment value is
6917 // specfied in low 16-bits, The index is specified in the high bits. For the
6918 // index, 0 indicates the return value while higher values correspond to
6919 // each parameter (idx = param + 1).
6920 const uint64_t AlignIdxValuePair =
6921 mdconst::extract<ConstantInt>(V)->getZExtValue();
6922 const unsigned Idx = (AlignIdxValuePair >> 16);
6923 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
6924 cast<Function>(GV)->addAttributeAtIndex(
6925 Idx, Attribute::getWithStackAlignment(GV->getContext(), StackAlign));
6926 return true;
6927 }
6928 if (K == "maxclusterrank" || K == "cluster_max_blocks") {
6929 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6930 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxClusterRank, llvm::utostr(CV));
6931 return true;
6932 }
6933 if (K == "minctasm") {
6934 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6935 cast<Function>(GV)->addFnAttr(NVVMAttr::MinCTASm, llvm::utostr(CV));
6936 return true;
6937 }
6938 if (K == "maxnreg") {
6939 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6940 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxNReg, llvm::utostr(CV));
6941 return true;
6942 }
6943 if (K.consume_front("maxntid") && isXYZ(K)) {
6944 upgradeNVVMFnVectorAttr(NVVMAttr::MaxNTID, K[0], GV, V);
6945 return true;
6946 }
6947 if (K.consume_front("reqntid") && isXYZ(K)) {
6948 upgradeNVVMFnVectorAttr(NVVMAttr::ReqNTID, K[0], GV, V);
6949 return true;
6950 }
6951 if (K.consume_front("cluster_dim_") && isXYZ(K)) {
6952 upgradeNVVMFnVectorAttr(NVVMAttr::ClusterDim, K[0], GV, V);
6953 return true;
6954 }
6955 if (K == "grid_constant") {
6956 const auto Attr = Attribute::get(GV->getContext(), NVVMAttr::GridConstant);
6957 for (const auto &Op : cast<MDNode>(V)->operands()) {
6958 // For some reason, the index is 1-based in the metadata. Good thing we're
6959 // able to auto-upgrade it!
6960 const auto Index = mdconst::extract<ConstantInt>(Op)->getZExtValue() - 1;
6961 cast<Function>(GV)->addParamAttr(Index, Attr);
6962 }
6963 return true;
6964 }
6965
6966 return false;
6967}
6968
6969void llvm::UpgradeNVVMAnnotations(Module &M) {
6970 NamedMDNode *NamedMD = M.getNamedMetadata("nvvm.annotations");
6971 if (!NamedMD)
6972 return;
6973
6974 SmallVector<MDNode *, 8> NewNodes;
6975 SmallPtrSet<const MDNode *, 8> SeenNodes;
6976 for (MDNode *MD : NamedMD->operands()) {
6977 if (!SeenNodes.insert(MD).second)
6978 continue;
6979
6980 auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD->getOperand(0));
6981 if (!GV)
6982 continue;
6983
6984 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
6985
6986 SmallVector<Metadata *, 8> NewOperands{MD->getOperand(0)};
6987 // Each nvvm.annotations metadata entry will be of the following form:
6988 // !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
6989 // start index = 1, to skip the global variable key
6990 // increment = 2, to skip the value for each property-value pairs
6991 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
6992 MDString *K = cast<MDString>(MD->getOperand(j));
6993 const MDOperand &V = MD->getOperand(j + 1);
6994 bool Upgraded = upgradeSingleNVVMAnnotation(GV, K->getString(), V);
6995 if (!Upgraded)
6996 NewOperands.append({K, V});
6997 }
6998
6999 if (NewOperands.size() > 1)
7000 NewNodes.push_back(MDNode::get(M.getContext(), NewOperands));
7001 }
7002
7003 NamedMD->clearOperands();
7004 for (MDNode *N : NewNodes)
7005 NamedMD->addOperand(N);
7006}
7007
7008/// This checks for objc retain release marker which should be upgraded. It
7009/// returns true if module is modified.
7010static bool upgradeRetainReleaseMarker(Module &M) {
7011 bool Changed = false;
7012 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
7013 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
7014 if (ModRetainReleaseMarker) {
7015 MDNode *Op = ModRetainReleaseMarker->getOperand(0);
7016 if (Op) {
7017 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
7018 if (ID) {
7019 SmallVector<StringRef, 4> ValueComp;
7020 ID->getString().split(ValueComp, "#");
7021 if (ValueComp.size() == 2) {
7022 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
7023 ID = MDString::get(M.getContext(), NewValue);
7024 }
7025 M.addModuleFlag(Module::Error, MarkerKey, ID);
7026 M.eraseNamedMetadata(ModRetainReleaseMarker);
7027 Changed = true;
7028 }
7029 }
7030 }
7031 return Changed;
7032}
7033
7034void llvm::UpgradeARCRuntime(Module &M) {
7035 // This lambda converts normal function calls to ARC runtime functions to
7036 // intrinsic calls.
7037 auto UpgradeToIntrinsic = [&](const char *OldFunc,
7038 llvm::Intrinsic::ID IntrinsicFunc) {
7039 Function *Fn = M.getFunction(OldFunc);
7040
7041 if (!Fn)
7042 return;
7043
7044 Function *NewFn =
7045 llvm::Intrinsic::getOrInsertDeclaration(&M, IntrinsicFunc);
7046
7047 for (User *U : make_early_inc_range(Fn->users())) {
7048 CallInst *CI = dyn_cast<CallInst>(U);
7049 if (!CI || CI->getCalledFunction() != Fn)
7050 continue;
7051
7052 IRBuilder<> Builder(CI->getIterator());
7053 FunctionType *NewFuncTy = NewFn->getFunctionType();
7054 SmallVector<Value *, 2> Args;
7055
7056 // Don't upgrade the intrinsic if it's not valid to bitcast the return
7057 // value to the return type of the old function.
7058 if (NewFuncTy->getReturnType() != CI->getType() &&
7059 !CastInst::castIsValid(Instruction::BitCast, CI,
7060 NewFuncTy->getReturnType()))
7061 continue;
7062
7063 bool InvalidCast = false;
7064
7065 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
7066 Value *Arg = CI->getArgOperand(I);
7067
7068 // Bitcast argument to the parameter type of the new function if it's
7069 // not a variadic argument.
7070 if (I < NewFuncTy->getNumParams()) {
7071 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
7072 // to the parameter type of the new function.
7073 if (!CastInst::castIsValid(Instruction::BitCast, Arg,
7074 NewFuncTy->getParamType(I))) {
7075 InvalidCast = true;
7076 break;
7077 }
7078 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
7079 }
7080 Args.push_back(Arg);
7081 }
7082
7083 if (InvalidCast)
7084 continue;
7085
7086 // Create a call instruction that calls the new function.
7087 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
7088 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
7089 NewCall->takeName(CI);
7090
7091 // Bitcast the return value back to the type of the old call.
7092 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
7093
7094 if (!CI->use_empty())
7095 CI->replaceAllUsesWith(NewRetVal);
7096 CI->eraseFromParent();
7097 }
7098
7099 if (Fn->use_empty())
7100 Fn->eraseFromParent();
7101 };
7102
7103 // Unconditionally convert a call to "clang.arc.use" to a call to
7104 // "llvm.objc.clang.arc.use".
7105 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
7106
7107 // Upgrade the retain release marker. If there is no need to upgrade
7108 // the marker, that means either the module is already new enough to contain
7109 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
7110 if (!upgradeRetainReleaseMarker(M))
7111 return;
7112
7113 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
7114 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
7115 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
7116 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
7117 {"objc_autoreleaseReturnValue",
7118 llvm::Intrinsic::objc_autoreleaseReturnValue},
7119 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
7120 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
7121 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
7122 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
7123 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
7124 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
7125 {"objc_release", llvm::Intrinsic::objc_release},
7126 {"objc_retain", llvm::Intrinsic::objc_retain},
7127 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
7128 {"objc_retainAutoreleaseReturnValue",
7129 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
7130 {"objc_retainAutoreleasedReturnValue",
7131 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
7132 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
7133 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
7134 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
7135 {"objc_unsafeClaimAutoreleasedReturnValue",
7136 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
7137 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
7138 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
7139 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
7140 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
7141 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
7142 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
7143 {"objc_arc_annotation_topdown_bbstart",
7144 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
7145 {"objc_arc_annotation_topdown_bbend",
7146 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
7147 {"objc_arc_annotation_bottomup_bbstart",
7148 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
7149 {"objc_arc_annotation_bottomup_bbend",
7150 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
7151
7152 for (auto &I : RuntimeFuncs)
7153 UpgradeToIntrinsic(I.first, I.second);
7154}
7155
7156// Upgrade the way signing of pointers to init/fini functions is described.
7157//
7158// Originally, the `@llvm.global_(ctors|dtors)` arrays contained `ptrauth`
7159// constants, if signing was requested. After the upgrade, these arrays contain
7160// plain function pointers and the desired signing schema is described via a
7161// pair of module flags.
7162//
7163// Note that the upgrade is only performed if all elements of *both* arrays
7164// agree on a common signing schema.
7165static bool upgradePtrauthInitFiniArrays(Module &M) {
7166 // As we cannot always decide whether the particular module should have
7167 // ptrauth-init-fini flags, we have to treat absent flags as having zero
7168 // values for compatibility reasons. Thus, upgradePtrauthInitFiniArrays
7169 // returns as soon as it spots any non-signed init/fini pointer: either we
7170 // should request non-signed pointers (safe to omit both flags) or there is
7171 // no common schema (and thus we do not modify anything).
7172 //
7173 // UseAddressDisc's value either represents "not decided yet" state (nullopt)
7174 // or whether we should request address diversity in addition to the basic
7175 // constant diversity. There is no value representing "decided not to sign"
7176 // for the reasons explained above.
7177 std::optional<bool> UseAddressDisc;
7178
7179 // Do not attempt upgrading if the new module flags already exist.
7180 if (const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
7181 for (const MDNode *Flag : ModFlags->operands()) {
7182 std::optional<StringRef> Name = getModuleFlagNameSafely(*Flag);
7183 if (Name && (*Name == "ptrauth-init-fini" ||
7184 *Name == "ptrauth-init-fini-address-discrimination"))
7185 return false;
7186 }
7187 }
7188
7189 auto UpgradeSinglePointer = [&UseAddressDisc](Constant *CV) -> Constant * {
7190 constexpr unsigned ExpectedConstDisc = 0xD9D4;
7191 constexpr unsigned ExpectedAddressMarker = 1;
7192
7193 auto *CPA = dyn_cast<ConstantPtrAuth>(CV);
7194 if (!CPA || !CPA->getDiscriminator()->equalsInt(ExpectedConstDisc))
7195 return nullptr; // Nothing to upgrade or unknown pattern found.
7196
7197 bool HasAddressDisc;
7198 if (!CPA->hasAddressDiscriminator())
7199 HasAddressDisc = false;
7200 else if (CPA->hasSpecialAddressDiscriminator(ExpectedAddressMarker))
7201 HasAddressDisc = true;
7202 else
7203 return nullptr; // Unknown pattern.
7204
7205 if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
7206 return nullptr; // Disagreement with the decided mode.
7207
7208 UseAddressDisc = HasAddressDisc;
7209 return CPA->getPointer();
7210 };
7211
7212 // Do not apply any changes until we know the upgrade is non-ambiguous.
7213 using PendingUpgrade = std::pair<GlobalVariable *, Constant *>;
7214 SmallVector<PendingUpgrade, 2> GlobalArraysToUpgrade;
7215
7216 for (const char *Name : {"llvm.global_ctors", "llvm.global_dtors"}) {
7217 auto *GV = dyn_cast_if_present<GlobalVariable>(M.getNamedValue(Name));
7218 if (!GV || !GV->hasInitializer())
7219 continue; // Skip, but it is okay to upgrade the other variable.
7220
7221 auto *OldStructorsArray = dyn_cast<ConstantArray>(GV->getInitializer());
7222 if (!OldStructorsArray || OldStructorsArray->getNumOperands() == 0)
7223 return false;
7224
7225 std::vector<Constant *> NewStructors;
7226 NewStructors.reserve(OldStructorsArray->getNumOperands());
7227
7228 for (Use &U : OldStructorsArray->operands()) {
7229 ConstantStruct *Structor = dyn_cast<ConstantStruct>(U.get());
7230 if (!Structor || Structor->getNumOperands() != 3)
7231 return false;
7232
7233 Constant *Prio = Structor->getOperand(0);
7234 Constant *Func = Structor->getOperand(1);
7235 Constant *Arg = Structor->getOperand(2);
7236
7237 Func = UpgradeSinglePointer(Func);
7238 if (!Func)
7239 return false;
7240
7241 NewStructors.push_back(
7242 ConstantStruct::get(Structor->getType(), {Prio, Func, Arg}));
7243 }
7244
7245 Constant *NewInit =
7246 ConstantArray::get(OldStructorsArray->getType(), NewStructors);
7247 GlobalArraysToUpgrade.emplace_back(GV, NewInit);
7248 }
7249
7250 if (GlobalArraysToUpgrade.empty())
7251 return false;
7252 assert(UseAddressDisc.has_value());
7253
7254 for (auto [GV, NewInit] : GlobalArraysToUpgrade)
7255 GV->setInitializer(NewInit);
7256
7257 M.addModuleFlag(Module::Error, "ptrauth-init-fini", 1);
7258 M.addModuleFlag(Module::Error, "ptrauth-init-fini-address-discrimination",
7259 *UseAddressDisc);
7260
7261 return true;
7262}
7263
7264bool llvm::UpgradeModuleFlags(Module &M) {
7265 bool Changed = false;
7266 Changed |= upgradePtrauthInitFiniArrays(M);
7267
7268 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7269 if (!ModFlags)
7270 return Changed;
7271
7272 bool HasObjCFlag = false, HasClassProperties = false;
7273 bool HasSwiftVersionFlag = false;
7274 uint8_t SwiftMajorVersion, SwiftMinorVersion;
7275 uint32_t SwiftABIVersion;
7276 auto Int8Ty = Type::getInt8Ty(M.getContext());
7277 auto Int32Ty = Type::getInt32Ty(M.getContext());
7278
7279 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7280 MDNode *Op = ModFlags->getOperand(I);
7281 if (Op->getNumOperands() != 3)
7282 continue;
7283 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7284 if (!ID)
7285 continue;
7286 auto SetBehavior = [&](Module::ModFlagBehavior B) {
7287 Metadata *Ops[3] = {ConstantAsMetadata::get(ConstantInt::get(
7288 Type::getInt32Ty(M.getContext()), B)),
7289 MDString::get(M.getContext(), ID->getString()),
7290 Op->getOperand(2)};
7291 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7292 Changed = true;
7293 };
7294
7295 if (ID->getString() == "Objective-C Image Info Version")
7296 HasObjCFlag = true;
7297 if (ID->getString() == "Objective-C Class Properties")
7298 HasClassProperties = true;
7299 // Upgrade PIC from Error/Max to Min.
7300 if (ID->getString() == "PIC Level") {
7301 if (auto *Behavior =
7302 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
7303 uint64_t V = Behavior->getLimitedValue();
7304 if (V == Module::Error || V == Module::Max)
7305 SetBehavior(Module::Min);
7306 }
7307 }
7308 // Upgrade "PIE Level" from Error to Max.
7309 if (ID->getString() == "PIE Level")
7310 if (auto *Behavior =
7311 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0)))
7312 if (Behavior->getLimitedValue() == Module::Error)
7313 SetBehavior(Module::Max);
7314
7315 // Upgrade branch protection and return address signing module flags. The
7316 // module flag behavior for these fields were Error and now they are Min.
7317 // The one exception is "sign-return-address-harden".
7318 if (ID->getString() == "branch-target-enforcement" ||
7319 (ID->getString().starts_with("sign-return-address") &&
7320 ID->getString() != "sign-return-address-harden")) {
7321 if (auto *Behavior =
7322 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
7323 if (Behavior->getLimitedValue() == Module::Error) {
7324 Type *Int32Ty = Type::getInt32Ty(M.getContext());
7325 Metadata *Ops[3] = {
7326 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Min)),
7327 Op->getOperand(1), Op->getOperand(2)};
7328 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7329 Changed = true;
7330 }
7331 }
7332 }
7333
7334 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
7335 // section name so that llvm-lto will not complain about mismatching
7336 // module flags that is functionally the same.
7337 if (ID->getString() == "Objective-C Image Info Section") {
7338 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
7339 SmallVector<StringRef, 4> ValueComp;
7340 Value->getString().split(ValueComp, " ");
7341 if (ValueComp.size() != 1) {
7342 std::string NewValue;
7343 for (auto &S : ValueComp)
7344 NewValue += S.str();
7345 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
7346 MDString::get(M.getContext(), NewValue)};
7347 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7348 Changed = true;
7349 }
7350 }
7351 }
7352
7353 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
7354 // If the higher bits are set, it adds new module flag for swift info.
7355 if (ID->getString() == "Objective-C Garbage Collection") {
7356 auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
7357 if (Md) {
7358 assert(Md->getValue() && "Expected non-empty metadata");
7359 auto Type = Md->getValue()->getType();
7360 if (Type == Int8Ty)
7361 continue;
7362 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
7363 if ((Val & 0xff) != Val) {
7364 HasSwiftVersionFlag = true;
7365 SwiftABIVersion = (Val & 0xff00) >> 8;
7366 SwiftMajorVersion = (Val & 0xff000000) >> 24;
7367 SwiftMinorVersion = (Val & 0xff0000) >> 16;
7368 }
7369 Metadata *Ops[3] = {
7370 ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
7371 Op->getOperand(1),
7372 ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
7373 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7374 Changed = true;
7375 }
7376 }
7377
7378 if (ID->getString() == "amdgpu_code_object_version") {
7379 Metadata *Ops[3] = {
7380 Op->getOperand(0),
7381 MDString::get(M.getContext(), "amdhsa_code_object_version"),
7382 Op->getOperand(2)};
7383 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7384 Changed = true;
7385 }
7386
7387 // clang/PowerPC used to use "float-abi" to describe the long double format;
7388 // it has been renamed to "long-double-type", with its values changed to the
7389 // corresponding IR floating-point type names.
7390 if (M.getTargetTriple().isPPC() && ID->getString() == "float-abi") {
7391 StringRef Format;
7392 if (auto *S = dyn_cast_or_null<MDString>(Op->getOperand(2)))
7393 Format = S->getString();
7394
7395 // The "float-abi" key is now reserved for the target-independent
7396 // soft/hard ABI flag, so leave a valid value alone. Map any other value
7397 // (including unrecognized ones, which were never valid) to the default.
7398 if (!FloatABI::parseABIType(Format)) {
7399 LongDoubleFormat NewFormat =
7400 StringSwitch<LongDoubleFormat>(Format)
7401 .Case("ieeequad", LongDoubleFormat::IEEEquad)
7402 .Case("ieeedouble", LongDoubleFormat::IEEEdouble)
7403 .Default(LongDoubleFormat::PPCDoubleDouble);
7404 Metadata *Ops[3] = {
7405 Op->getOperand(0),
7406 MDString::get(M.getContext(), "long-double-type"),
7407 MDString::get(M.getContext(), getLongDoubleFormatName(NewFormat))};
7408 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7409 Changed = true;
7410 }
7411 }
7412 }
7413
7414 // "Objective-C Class Properties" is recently added for Objective-C. We
7415 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
7416 // flag of value 0, so we can correclty downgrade this flag when trying to
7417 // link an ObjC bitcode without this module flag with an ObjC bitcode with
7418 // this module flag.
7419 if (HasObjCFlag && !HasClassProperties) {
7420 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
7421 (uint32_t)0);
7422 Changed = true;
7423 }
7424
7425 if (HasSwiftVersionFlag) {
7426 M.addModuleFlag(Module::Error, "Swift ABI Version",
7427 SwiftABIVersion);
7428 M.addModuleFlag(Module::Error, "Swift Major Version",
7429 ConstantInt::get(Int8Ty, SwiftMajorVersion));
7430 M.addModuleFlag(Module::Error, "Swift Minor Version",
7431 ConstantInt::get(Int8Ty, SwiftMinorVersion));
7432 Changed = true;
7433 }
7434
7435 return Changed;
7436}
7437
7438bool llvm::UpgradeCFIFunctionsMetadata(Module &M) {
7439 NamedMDNode *CFIConsts = M.getNamedMetadata("cfi.functions");
7440 // If this metadata has operands, we expect all of them to be either from
7441 // before or from after the format change handled here, so we can bail out
7442 // fast if the first (if any) operands is of the new format.
7443 auto MatchesVersion = [](const MDNode *Op) {
7444 return Op->getNumOperands() >= 3 &&
7445 isa<ConstantAsMetadata>(Op->getOperand(2)) &&
7446 cast<ConstantAsMetadata>(Op->getOperand(2))
7447 ->getType()
7448 ->isIntegerTy(64);
7449 };
7450
7451 if (!CFIConsts || !CFIConsts->getNumOperands() ||
7452 MatchesVersion(CFIConsts->getOperand(0)))
7453 return false;
7454
7455 bool Changed = false;
7456 for (unsigned I = 0, E = CFIConsts->getNumOperands(); I != E; ++I) {
7457 MDNode *Op = CFIConsts->getOperand(I);
7458 assert(!MatchesVersion(Op) && "Unexpected mix of CFIConstant formats");
7459 assert(Op->getNumOperands() >= 2 &&
7460 "Expected at least 2 operands - name and linkage type");
7461 MDString *NameMD = dyn_cast<MDString>(Op->getOperand(0));
7462 StringRef Name = NameMD->getString();
7463 GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage(
7464 GlobalValue::dropLLVMManglingEscape(Name));
7465
7466 SmallVector<Metadata *, 4> Elts;
7467 Elts.push_back(Op->getOperand(0));
7468 Elts.push_back(Op->getOperand(1));
7469 Elts.push_back(ConstantAsMetadata::get(
7470 ConstantInt::get(Type::getInt64Ty(M.getContext()), GUID)));
7471
7472 for (unsigned J = 2, EJ = Op->getNumOperands(); J != EJ; ++J)
7473 Elts.push_back(Op->getOperand(J));
7474
7475 CFIConsts->setOperand(I, MDNode::get(M.getContext(), Elts));
7476 Changed = true;
7477 }
7478
7479 return Changed;
7480}
7481
7482void llvm::UpgradeSectionAttributes(Module &M) {
7483 auto TrimSpaces = [](StringRef Section) -> std::string {
7484 SmallVector<StringRef, 5> Components;
7485 Section.split(Components, ',');
7486
7487 SmallString<32> Buffer;
7488 raw_svector_ostream OS(Buffer);
7489
7490 for (auto Component : Components)
7491 OS << ',' << Component.trim();
7492
7493 return std::string(OS.str().substr(1));
7494 };
7495
7496 for (auto &GV : M.globals()) {
7497 if (!GV.hasSection())
7498 continue;
7499
7500 StringRef Section = GV.getSection();
7501
7502 if (!Section.starts_with("__DATA, __objc_catlist"))
7503 continue;
7504
7505 // __DATA, __objc_catlist, regular, no_dead_strip
7506 // __DATA,__objc_catlist,regular,no_dead_strip
7507 GV.setSection(TrimSpaces(Section));
7508 }
7509}
7510
7511namespace {
7512// Prior to LLVM 10.0, the strictfp attribute could be used on individual
7513// callsites within a function that did not also have the strictfp attribute.
7514// Since 10.0, if strict FP semantics are needed within a function, the
7515// function must have the strictfp attribute and all calls within the function
7516// must also have the strictfp attribute. This latter restriction is
7517// necessary to prevent unwanted libcall simplification when a function is
7518// being cloned (such as for inlining).
7519//
7520// The "dangling" strictfp attribute usage was only used to prevent constant
7521// folding and other libcall simplification. The nobuiltin attribute on the
7522// callsite has the same effect.
7523struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
7524 StrictFPUpgradeVisitor() = default;
7525
7526 void visitCallBase(CallBase &Call) {
7527 if (!Call.isStrictFP())
7528 return;
7529 if (isa<ConstrainedFPIntrinsic>(&Call))
7530 return;
7531 // If we get here, the caller doesn't have the strictfp attribute
7532 // but this callsite does. Replace the strictfp attribute with nobuiltin.
7533 Call.removeFnAttr(Attribute::StrictFP);
7534 Call.addFnAttr(Attribute::NoBuiltin);
7535 }
7536};
7537
7538/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
7539struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
7540 : public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
7541 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
7542
7543 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
7544 if (!RMW.isFloatingPointOperation())
7545 return;
7546
7547 MDNode *Empty = MDNode::get(RMW.getContext(), {});
7548 RMW.setMetadata("amdgpu.no.fine.grained.host.memory", Empty);
7549 RMW.setMetadata("amdgpu.no.remote.memory.access", Empty);
7550 RMW.setMetadata(LLVMContext::MD_atomic_ignore_denormal_mode, Empty);
7551 }
7552};
7553} // namespace
7554
7555void llvm::UpgradeFunctionAttributes(Function &F) {
7556 // If a function definition doesn't have the strictfp attribute,
7557 // convert any callsite strictfp attributes to nobuiltin.
7558 if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
7559 StrictFPUpgradeVisitor SFPV;
7560 SFPV.visit(F);
7561 }
7562
7563 // Remove all incompatibile attributes from function.
7564 F.removeRetAttrs(AttributeFuncs::typeIncompatible(
7565 F.getReturnType(), F.getAttributes().getRetAttrs()));
7566 for (auto &Arg : F.args())
7567 Arg.removeAttrs(
7568 AttributeFuncs::typeIncompatible(Arg.getType(), Arg.getAttributes()));
7569
7570 bool AddingAttrs = false, RemovingAttrs = false;
7571 AttrBuilder AttrsToAdd(F.getContext());
7572 AttributeMask AttrsToRemove;
7573
7574 // Older versions of LLVM treated an "implicit-section-name" attribute
7575 // similarly to directly setting the section on a Function.
7576 if (Attribute A = F.getFnAttribute("implicit-section-name");
7577 A.isValid() && A.isStringAttribute()) {
7578 F.setSection(A.getValueAsString());
7579 AttrsToRemove.addAttribute("implicit-section-name");
7580 RemovingAttrs = true;
7581 }
7582
7583 if (Attribute A = F.getFnAttribute("nooutline");
7584 A.isValid() && A.isStringAttribute()) {
7585 AttrsToRemove.addAttribute("nooutline");
7586 AttrsToAdd.addAttribute(Attribute::NoOutline);
7587 AddingAttrs = RemovingAttrs = true;
7588 }
7589
7590 if (Attribute A = F.getFnAttribute("uniform-work-group-size");
7591 A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
7592 AttrsToRemove.addAttribute("uniform-work-group-size");
7593 RemovingAttrs = true;
7594 if (A.getValueAsString() == "true") {
7595 AttrsToAdd.addAttribute("uniform-work-group-size");
7596 AddingAttrs = true;
7597 }
7598 }
7599
7600 if (!F.empty()) {
7601 // For some reason this is called twice, and the first time is before any
7602 // instructions are loaded into the body.
7603
7604 if (Attribute A = F.getFnAttribute("amdgpu-unsafe-fp-atomics");
7605 A.isValid()) {
7606
7607 if (A.getValueAsBool()) {
7608 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
7609 Visitor.visit(F);
7610 }
7611
7612 // We will leave behind dead attribute uses on external declarations, but
7613 // clang never added these to declarations anyway.
7614 AttrsToRemove.addAttribute("amdgpu-unsafe-fp-atomics");
7615 RemovingAttrs = true;
7616 }
7617 }
7618
7619 DenormalMode DenormalFPMath = DenormalMode::getIEEE();
7620 DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
7621
7622 bool HandleDenormalMode = false;
7623
7624 if (Attribute Attr = F.getFnAttribute("denormal-fp-math"); Attr.isValid()) {
7625 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7626 if (ParsedMode.isValid()) {
7627 DenormalFPMath = ParsedMode;
7628 AttrsToRemove.addAttribute("denormal-fp-math");
7629 AddingAttrs = RemovingAttrs = true;
7630 HandleDenormalMode = true;
7631 }
7632 }
7633
7634 if (Attribute Attr = F.getFnAttribute("denormal-fp-math-f32");
7635 Attr.isValid()) {
7636 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7637 if (ParsedMode.isValid()) {
7638 DenormalFPMathF32 = ParsedMode;
7639 AttrsToRemove.addAttribute("denormal-fp-math-f32");
7640 AddingAttrs = RemovingAttrs = true;
7641 HandleDenormalMode = true;
7642 }
7643 }
7644
7645 if (HandleDenormalMode)
7646 AttrsToAdd.addDenormalFPEnvAttr(
7647 DenormalFPEnv(DenormalFPMath, DenormalFPMathF32));
7648
7649 if (RemovingAttrs)
7650 F.removeFnAttrs(AttrsToRemove);
7651
7652 if (AddingAttrs)
7653 F.addFnAttrs(AttrsToAdd);
7654}
7655
7656// Check if the function attribute is not present and set it.
7657static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName,
7658 StringRef Value) {
7659 if (!F.hasFnAttribute(FnAttrName)) {
7660 F.addFnAttr(FnAttrName, Value);
7661 LLVM_DEBUG(dbgs() << "Set attribute: " << FnAttrName << "=\"" << Value
7662 << "\", function: " << F.getName() << "\n");
7663 }
7664}
7665
7666// Check if the function attribute is not present and set it if needed.
7667// If the attribute is "false" then removes it.
7668// If the attribute is "true" resets it to a valueless attribute.
7669static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
7670 if (!F.hasFnAttribute(FnAttrName)) {
7671 if (Set) {
7672 F.addFnAttr(FnAttrName);
7673 LLVM_DEBUG(dbgs() << "Added attribute: " << FnAttrName
7674 << ", function: " << F.getName() << "\n");
7675 }
7676 } else {
7677 auto A = F.getFnAttribute(FnAttrName);
7678 if ("false" == A.getValueAsString()) {
7679 F.removeFnAttr(FnAttrName);
7680 LLVM_DEBUG(dbgs() << "Removed attribute: " << FnAttrName
7681 << "=\"false\", function: " << F.getName() << "\n");
7682 } else if ("true" == A.getValueAsString()) {
7683 F.removeFnAttr(FnAttrName);
7684 F.addFnAttr(FnAttrName);
7685 LLVM_DEBUG(dbgs() << "Converted attribute: " << FnAttrName
7686 << "=\"true\", function: " << F.getName() << "\n");
7687 }
7688 }
7689}
7690
7691static void ConvertModuleFlag(Module &M, Module::ModFlagBehavior Behavior,
7692 StringRef Key, uint32_t Val) {
7693 M.setModuleFlag(Behavior, Key, Val);
7694 LLVM_DEBUG(dbgs() << "Converted module flag: " << "{" << Behavior << ", "
7695 << Key << ", " << Val << "}\n");
7696}
7697
7698void llvm::copyModuleAttrToFunctions(Module &M) {
7699 Triple T(M.getTargetTriple());
7700 if (!T.isThumb() && !T.isARM() && !T.isAArch64())
7701 return;
7702
7703 uint64_t BTEValue = 0;
7704 uint64_t BPPLRValue = 0;
7705 uint64_t GCSValue = 0;
7706 uint64_t SRAValue = 0;
7707 uint64_t SRAALLValue = 0;
7708 uint64_t SRABKeyValue = 0;
7709
7710 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7711 if (ModFlags) {
7712 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7713 MDNode *Op = ModFlags->getOperand(I);
7714 if (Op->getNumOperands() != 3)
7715 continue;
7716
7717 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7718 auto *CI = mdconst::dyn_extract<ConstantInt>(Op->getOperand(2));
7719 if (!ID || !CI)
7720 continue;
7721
7722 StringRef IDStr = ID->getString();
7723 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
7724 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
7725 : IDStr == "guarded-control-stack" ? &GCSValue
7726 : IDStr == "sign-return-address" ? &SRAValue
7727 : IDStr == "sign-return-address-all" ? &SRAALLValue
7728 : IDStr == "sign-return-address-with-bkey"
7729 ? &SRABKeyValue
7730 : nullptr;
7731 if (!ValPtr)
7732 continue;
7733
7734 *ValPtr = CI->getZExtValue();
7735 if (*ValPtr == 2)
7736 return;
7737
7738 LLVM_DEBUG(dbgs() << "Found module flag: " << IDStr << "(" << *ValPtr
7739 << ")\n");
7740 }
7741 }
7742
7743 bool BTE = BTEValue == 1;
7744 bool BPPLR = BPPLRValue == 1;
7745 bool GCS = GCSValue == 1;
7746 bool SRA = SRAValue == 1;
7747
7748 StringRef SignTypeValue = "non-leaf";
7749 if (SRA && SRAALLValue == 1)
7750 SignTypeValue = "all";
7751
7752 StringRef SignKeyValue = "a_key";
7753 if (SRA && SRABKeyValue == 1)
7754 SignKeyValue = "b_key";
7755
7756 for (Function &F : M.getFunctionList()) {
7757 if (F.isDeclaration())
7758 continue;
7759
7760 if (SRA) {
7761 setFunctionAttrIfNotSet(F, "sign-return-address", SignTypeValue);
7762 setFunctionAttrIfNotSet(F, "sign-return-address-key", SignKeyValue);
7763 } else {
7764 if (auto A = F.getFnAttribute("sign-return-address");
7765 A.isValid() && "none" == A.getValueAsString()) {
7766 F.removeFnAttr("sign-return-address");
7767 F.removeFnAttr("sign-return-address-key");
7768 }
7769 }
7770 ConvertFunctionAttr(F, BTE, "branch-target-enforcement");
7771 ConvertFunctionAttr(F, BPPLR, "branch-protection-pauth-lr");
7772 ConvertFunctionAttr(F, GCS, "guarded-control-stack");
7773 }
7774
7775 if (BTE)
7776 ConvertModuleFlag(M, llvm::Module::Min, "branch-target-enforcement", 2);
7777 if (BPPLR)
7778 ConvertModuleFlag(M, llvm::Module::Min, "branch-protection-pauth-lr", 2);
7779 if (GCS)
7780 ConvertModuleFlag(M, llvm::Module::Min, "guarded-control-stack", 2);
7781 if (SRA) {
7782 ConvertModuleFlag(M, llvm::Module::Min, "sign-return-address", 2);
7783 if (SRAALLValue == 1)
7784 ConvertModuleFlag(M, llvm::Module::Min, "sign-return-address-all", 2);
7785 if (SRABKeyValue == 1) {
7786 ConvertModuleFlag(M, llvm::Module::Min, "sign-return-address-with-bkey",
7787 2);
7788 }
7789 }
7790}
7791
7792/// Return the replacement tags if \p T still uses a removed two-operand form.
7793static const BooleanLoopTags *getOldBooleanLoopTags(const MDTuple *T) {
7794 if (T->getNumOperands() != 2 || !mdconst::hasa<ConstantInt>(T->getOperand(1)))
7795 return nullptr;
7796 auto *Tag = dyn_cast_or_null<MDString>(T->getOperand(0));
7797 return Tag ? findBooleanLoopTags(Tag->getString()) : nullptr;
7798}
7799
7800/// Build the single-operand node that replaces a boolean operand: nonzero
7801/// selects the enable tag, zero the disable tag.
7802static Metadata *makeBooleanLoopNode(LLVMContext &C,
7803 const BooleanLoopTags &Tags,
7804 const MDOperand &Op) {
7805 bool Enable = !mdconst::extract<ConstantInt>(Op)->isZero();
7806 return MDTuple::get(C,
7807 {MDString::get(C, Enable ? Tags.Enable : Tags.Disable)});
7808}
7809
7810static bool isOldLoopArgument(Metadata *MD) {
7811 auto *T = dyn_cast_or_null<MDTuple>(MD);
7812 if (!T)
7813 return false;
7814 if (T->getNumOperands() < 1)
7815 return false;
7816 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
7817 if (!S)
7818 return false;
7819 if (S->getString().starts_with("llvm.vectorizer."))
7820 return true;
7821 return getOldBooleanLoopTags(T) != nullptr;
7822}
7823
7824static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
7825 StringRef OldPrefix = "llvm.vectorizer.";
7826 assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
7827
7828 if (OldTag == "llvm.vectorizer.unroll")
7829 return MDString::get(C, "llvm.loop.interleave.count");
7830
7831 return MDString::get(
7832 C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
7833 .str());
7834}
7835
7836static Metadata *upgradeLoopArgument(Metadata *MD) {
7837 auto *T = dyn_cast_or_null<MDTuple>(MD);
7838 if (!T)
7839 return MD;
7840 if (T->getNumOperands() < 1)
7841 return MD;
7842 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
7843 if (!OldTag)
7844 return MD;
7845
7846 LLVMContext &C = T->getContext();
7847
7848 /// Rewrite a removed two-operand boolean form to the single-operand pair.
7849 if (const BooleanLoopTags *Tags = getOldBooleanLoopTags(T))
7850 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7851
7852 if (!OldTag->getString().starts_with("llvm.vectorizer."))
7853 return MD;
7854
7855 // This has an old tag. Upgrade it.
7856 MDString *NewTag = upgradeLoopTag(C, OldTag->getString());
7857
7858 // The legacy !{!"llvm.vectorizer.enable", i1 X} maps onto the single-operand
7859 // vectorize.enable/disable pair, not a two-operand enable node.
7860 if (T->getNumOperands() == 2 && mdconst::hasa<ConstantInt>(T->getOperand(1)))
7861 if (const BooleanLoopTags *Tags = findBooleanLoopTags(NewTag->getString()))
7862 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7863
7864 SmallVector<Metadata *, 8> Ops;
7865 Ops.reserve(T->getNumOperands());
7866 Ops.push_back(NewTag);
7867 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
7868 Ops.push_back(T->getOperand(I));
7869
7870 return MDTuple::get(C, Ops);
7871}
7872
7873MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
7874 auto *T = dyn_cast<MDTuple>(&N);
7875 if (!T)
7876 return &N;
7877
7878 if (none_of(T->operands(), isOldLoopArgument))
7879 return &N;
7880
7881 // Fix the removed two-operand boolean nodes in place: the Verifier rejects
7882 // any MDNode carrying those tags with more than one operand, so a leftover
7883 // reference (from the distinct loop-ID) would still trigger a diagnostic.
7884 // In-place mutation is safe on distinct MDNodes.
7885 if (T->isDistinct()) {
7886 for (unsigned I = 0, E = T->getNumOperands(); I < E; ++I) {
7887 auto *OpT = dyn_cast_or_null<MDTuple>(T->getOperand(I));
7888 if (OpT && getOldBooleanLoopTags(OpT))
7889 T->replaceOperandWith(I, upgradeLoopArgument(OpT));
7890 }
7891 if (none_of(T->operands(), isOldLoopArgument))
7892 return &N;
7893 }
7894
7895 // Remaining old arguments (e.g. llvm.vectorizer.*) are handled via a wrapper
7896 // attachment; the original distinct loop-ID is kept as the first operand.
7897 SmallVector<Metadata *, 8> Ops;
7898 Ops.reserve(T->getNumOperands());
7899 for (Metadata *MD : T->operands())
7900 Ops.push_back(upgradeLoopArgument(MD));
7901
7902 return MDTuple::get(T->getContext(), Ops);
7903}
7904
7905std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
7906 Triple T(TT);
7907 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
7908 // the address space of globals to 1. This does not apply to SPIRV Logical.
7909 if ((T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical())) &&
7910 !DL.contains("-G") && !DL.starts_with("G")) {
7911 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
7912 }
7913
7914 if (T.isLoongArch64() || T.isRISCV64()) {
7915 // Make i32 a native type for 64-bit LoongArch and RISC-V.
7916 auto I = DL.find("-n64-");
7917 if (I != StringRef::npos)
7918 return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();
7919 return DL.str();
7920 }
7921
7922 // AMDGPU data layout upgrades.
7923 std::string Res = DL.str();
7924 if (T.isAMDGPU()) {
7925 // Define address spaces for constants.
7926 if (!DL.contains("-G") && !DL.starts_with("G"))
7927 Res.append(Res.empty() ? "G1" : "-G1");
7928
7929 // AMDGCN data layout upgrades.
7930 if (T.isAMDGCN()) {
7931
7932 // Add missing non-integral declarations.
7933 // This goes before adding new address spaces to prevent incoherent string
7934 // values.
7935 if (!DL.contains("-ni") && !DL.starts_with("ni"))
7936 Res.append("-ni:7:8:9");
7937 // Update ni:7 to ni:7:8:9.
7938 if (DL.ends_with("ni:7"))
7939 Res.append(":8:9");
7940 if (DL.ends_with("ni:7:8"))
7941 Res.append(":9");
7942
7943 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
7944 // resources) An empty data layout has already been upgraded to G1 by now.
7945 if (!DL.contains("-p7") && !DL.starts_with("p7"))
7946 Res.append("-p7:160:256:256:32");
7947 if (!DL.contains("-p8") && !DL.starts_with("p8"))
7948 Res.append("-p8:128:128:128:48");
7949 constexpr StringRef OldP8("-p8:128:128-");
7950 if (DL.contains(OldP8))
7951 Res.replace(Res.find(OldP8), OldP8.size(), "-p8:128:128:128:48-");
7952 if (!DL.contains("-p9") && !DL.starts_with("p9"))
7953 Res.append("-p9:192:256:256:32");
7954
7955 // Add sizing for address space 10 through 15.
7956 // AS 10-14 are reserved and defaulted to 32:32
7957 // AS 15 is in use and is 32:32.
7958 for (StringRef AS : {"p10", "p11", "p12", "p13", "p14", "p15"}) {
7959 if (!DL.contains(("-" + AS).str()) && !DL.starts_with(AS))
7960 Res.append(("-" + AS + ":32:32").str());
7961 }
7962 }
7963
7964 // Upgrade the ELF mangling mode.
7965 if (!DL.contains("m:e"))
7966 Res = Res.empty() ? "m:e" : "m:e-" + Res;
7967
7968 return Res;
7969 }
7970
7971 if (T.isSystemZ() && !DL.empty()) {
7972 // Make sure the stack alignment is present.
7973 if (!DL.contains("-S64"))
7974 return "E-S64" + DL.drop_front(1).str();
7975 return DL.str();
7976 }
7977
7978 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
7979 // If the datalayout matches the expected format, add pointer size address
7980 // spaces to the datalayout.
7981 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
7982 if (!DL.contains(AddrSpaces)) {
7983 SmallVector<StringRef, 4> Groups;
7984 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
7985 if (R.match(Res, &Groups))
7986 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
7987 }
7988 };
7989
7990 // AArch64 data layout upgrades.
7991 if (T.isAArch64()) {
7992 // Add "-Fn32"
7993 if (!DL.empty() && !DL.contains("-Fn32"))
7994 Res.append("-Fn32");
7995 AddPtr32Ptr64AddrSpaces();
7996 return Res;
7997 }
7998
7999 if (T.isSPARC() || (T.isMIPS64() && !DL.contains("m:m")) || T.isPPC64() ||
8000 T.isWasm()) {
8001 // Mips64 with o32 ABI did not add "-i128:128".
8002 // Add "-i128:128"
8003 std::string I64 = "-i64:64";
8004 std::string I128 = "-i128:128";
8005 if (!StringRef(Res).contains(I128)) {
8006 size_t Pos = Res.find(I64);
8007 if (Pos != size_t(-1))
8008 Res.insert(Pos + I64.size(), I128);
8009 }
8010 }
8011
8012 if (T.isPPC() && T.isOSAIX() && !DL.contains("f64:32:64") && !DL.empty()) {
8013 size_t Pos = Res.find("-S128");
8014 if (Pos == StringRef::npos)
8015 Pos = Res.size();
8016 Res.insert(Pos, "-f64:32:64");
8017 }
8018
8019 // ARM data layout upgrades.
8020 // Add -Fi8 if a -F has not already been specified.
8021 if (T.isARM() && !DL.empty() && !DL.contains("Fi") && !DL.contains("Fn")) {
8022 const StringRef p3232 = "p:32:32";
8023 size_t Pos = Res.find(p3232);
8024 if (Pos != StringRef::npos)
8025 Res.insert(Pos + p3232.size(), "-Fi8");
8026 }
8027
8028 if (!T.isX86())
8029 return Res;
8030
8031 AddPtr32Ptr64AddrSpaces();
8032
8033 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
8034 // for i128 operations prior to this being reflected in the data layout, and
8035 // clang mostly produced LLVM IR that already aligned i128 to 16 byte
8036 // boundaries, so although this is a breaking change, the upgrade is expected
8037 // to fix more IR than it breaks.
8038 // Intel MCU is an exception and uses 4-byte-alignment.
8039 if (!T.isOSIAMCU()) {
8040 std::string I128 = "-i128:128";
8041 if (StringRef Ref = Res; !Ref.contains(I128)) {
8042 SmallVector<StringRef, 4> Groups;
8043 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
8044 if (R.match(Res, &Groups))
8045 Res = (Groups[1] + I128 + Groups[3]).str();
8046 }
8047 }
8048
8049 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
8050 // Raising the alignment is safe because Clang did not produce f80 values in
8051 // the MSVC environment before this upgrade was added.
8052 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
8053 StringRef Ref = Res;
8054 auto I = Ref.find("-f80:32-");
8055 if (I != StringRef::npos)
8056 Res = (Ref.take_front(I) + "-f80:128-" + Ref.drop_front(I + 8)).str();
8057 }
8058
8059 return Res;
8060}
8061
8062void llvm::UpgradeAttributes(AttrBuilder &B) {
8063 StringRef FramePointer;
8064 Attribute A = B.getAttribute("no-frame-pointer-elim");
8065 if (A.isValid()) {
8066 // The value can be "true" or "false".
8067 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
8068 B.removeAttribute("no-frame-pointer-elim");
8069 }
8070 if (B.contains("no-frame-pointer-elim-non-leaf")) {
8071 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
8072 if (FramePointer != "all")
8073 FramePointer = "non-leaf";
8074 B.removeAttribute("no-frame-pointer-elim-non-leaf");
8075 }
8076 if (!FramePointer.empty())
8077 B.addAttribute("frame-pointer", FramePointer);
8078
8079 A = B.getAttribute("null-pointer-is-valid");
8080 if (A.isValid()) {
8081 // The value can be "true" or "false".
8082 bool NullPointerIsValid = A.getValueAsString() == "true";
8083 B.removeAttribute("null-pointer-is-valid");
8084 if (NullPointerIsValid)
8085 B.addAttribute(Attribute::NullPointerIsValid);
8086 }
8087
8088 A = B.getAttribute("uniform-work-group-size");
8089 if (A.isValid()) {
8090 StringRef Val = A.getValueAsString();
8091 if (!Val.empty()) {
8092 bool IsTrue = Val == "true";
8093 B.removeAttribute("uniform-work-group-size");
8094 if (IsTrue)
8095 B.addAttribute("uniform-work-group-size");
8096 }
8097 }
8098}
8099
8100void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
8101 // clang.arc.attachedcall bundles are now required to have an operand.
8102 // If they don't, it's okay to drop them entirely: when there is an operand,
8103 // the "attachedcall" is meaningful and required, but without an operand,
8104 // it's just a marker NOP. Dropping it merely prevents an optimization.
8105 erase_if(Bundles, [&](OperandBundleDef &OBD) {
8106 return OBD.getTag() == "clang.arc.attachedcall" &&
8107 OBD.inputs().empty();
8108 });
8109}
Pred
SDValue Pred
Definition AArch64ISelLowering.cpp:28553
assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPUAddrSpace.h
AMDGPU address space definition.
uint64_t
uint64_t uint64_t
Definition AMDGPUDisassembler.cpp:553
Imm
uint64_t Imm
Definition AMDGPUDisassembler.cpp:552
Select
AMDGPU Register Bank Select
Definition AMDGPURegBankSelect.cpp:70
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73
Attributes.h
This file contains the simple types necessary to represent the attributes associated with functions a...
upgradeX86VPERMT2Intrinsics
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
Definition AutoUpgrade.cpp:2715
isLegacyNVPTXBF16IntSignature
static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:1489
getFullArgCountForDefaultArgUpgrade
static unsigned getFullArgCountForDefaultArgUpgrade(Function *F, Intrinsic::ID IID, SmallVectorImpl< Type * > &OverloadTys)
Definition AutoUpgrade.cpp:1695
G2S_ID
#define G2S_ID(ID_SUFFIX, NAME)
upgradeLoopArgument
static Metadata * upgradeLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7836
shouldUpgradeNVPTXMBarrierInitIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1577
isXYZ
static bool isXYZ(StringRef S)
Definition AutoUpgrade.cpp:6904
upgradeIntrinsicFunction1
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
Definition AutoUpgrade.cpp:1731
upgradeX86PSLLDQIntrinsics
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2555
shouldUpgradeNVPTXSharedClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1372
upgradeVPIntrinsicCall
static Value * upgradeVPIntrinsicCall(StringRef Name, CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5733
getNVPTXTMAReductionOp
static std::optional< unsigned > getNVPTXTMAReductionOp(StringRef Name)
Definition AutoUpgrade.cpp:1339
shouldUpgradeNVPTXTMAReductionIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name)
Definition AutoUpgrade.cpp:1352
upgradeRetainReleaseMarker
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
Definition AutoUpgrade.cpp:7010
upgradeX86vpcom
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
Definition AutoUpgrade.cpp:2816
upgradeMaskToInt
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:3042
convertIntrinsicValidType
static bool convertIntrinsicValidType(StringRef Name, const FunctionType *FuncTy)
Definition AutoUpgrade.cpp:1674
upgradeX86Rotate
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
Definition AutoUpgrade.cpp:2790
upgradeX86MultiplyAddBytes
static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:125
getFunctionalIntrinsicIDForVP
static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name)
Definition AutoUpgrade.cpp:1627
setFunctionAttrIfNotSet
static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName, StringRef Value)
Definition AutoUpgrade.cpp:7657
shouldUpgradeNVPTXBF16Intrinsic
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1431
upgradeSingleNVVMAnnotation
static bool upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K, const Metadata *V)
Definition AutoUpgrade.cpp:6908
unwrapMAVOp
static MDNode * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
Definition AutoUpgrade.cpp:5593
upgradeLoopTag
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
Definition AutoUpgrade.cpp:7824
isValidLegacyNVVMFPArithIntrinsic
static bool isValidLegacyNVVMFPArithIntrinsic(const Intrinsic::ID(&IIDs)[2][3], StringRef Type, bool IsRN, bool IsFTZ, bool IsSat, bool IsRelu)
Definition AutoUpgrade.cpp:1527
getVPIntPredicateFromMD
static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5693
upgradeNVVMFnVectorAttr
static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC, GlobalValue *GV, const Metadata *V)
Definition AutoUpgrade.cpp:6871
upgradeX86MaskedFPCompare
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:112
upgradeX86ALIGNIntrinsics
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
Definition AutoUpgrade.cpp:2670
upgradeAbs
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2927
shouldUpgradeVPIntrinsic
static bool shouldUpgradeVPIntrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1670
emitX86Select
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2641
upgradeAArch64IntrinsicCall
static Value * upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5230
G2S_CTA_ID
#define G2S_CTA_ID(ID_SUFFIX, NAME)
upgradeMaskedMove
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:3028
NVVMFMAIIDs
static constexpr Intrinsic::ID NVVMFMAIIDs[2][3]
Definition AutoUpgrade.cpp:1520
NVVMFMulIIDs
static constexpr Intrinsic::ID NVVMFMulIIDs[2][3]
Definition AutoUpgrade.cpp:1516
getOldBooleanLoopTags
static const BooleanLoopTags * getOldBooleanLoopTags(const MDTuple *T)
Return the replacement tags if T still uses a removed two-operand form.
Definition AutoUpgrade.cpp:7793
upgradeX86IntrinsicFunction
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:556
applyX86MaskOn1BitsVec
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
Definition AutoUpgrade.cpp:2967
getModuleFlagNameSafely
static std::optional< StringRef > getModuleFlagNameSafely(const MDNode &Flag)
Definition AutoUpgrade.cpp:6817
consumeNVVMPtrAddrSpace
static bool consumeNVVMPtrAddrSpace(StringRef &Name)
Definition AutoUpgrade.cpp:1584
makeBooleanLoopNode
static Metadata * makeBooleanLoopNode(LLVMContext &C, const BooleanLoopTags &Tags, const MDOperand &Op)
Build the single-operand node that replaces a boolean operand: nonzero selects the enable tag,...
Definition AutoUpgrade.cpp:7802
G2S_CLUSTER_CASE
#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME)
shouldUpgradeX86Intrinsic
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:179
upgradeX86PSRLDQIntrinsics
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2589
getFunctionalOpcodeForVP
static unsigned getFunctionalOpcodeForVP(StringRef Name)
Definition AutoUpgrade.cpp:1590
shouldUpgradeNVPTXTMAG2SIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
Definition AutoUpgrade.cpp:1219
shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1396
isOldLoopArgument
static bool isOldLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7810
upgradeARMIntrinsicCall
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5300
ConvertModuleFlag
static void ConvertModuleFlag(Module &M, Module::ModFlagBehavior Behavior, StringRef Key, uint32_t Val)
Definition AutoUpgrade.cpp:7691
upgradeX86IntrinsicsWith8BitMask
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:96
upgradeVectorSplice
static Value * upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5663
upgradeAMDGCNIntrinsicCall
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5390
upgradeMaskedLoad
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2906
unwrapMAVMetadataOp
static Metadata * unwrapMAVMetadataOp(CallBase *CI, unsigned Op)
Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
Definition AutoUpgrade.cpp:5605
upgradeX86BF16Intrinsic
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:159
upgradeArmOrAarch64IntrinsicFunction
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:794
upgradeIntrinsicCallWithDefaultArgs
static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5764
getX86MaskVec
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
Definition AutoUpgrade.cpp:2621
emitX86ScalarSelect
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2653
upgradeIntrinsicWithDefaultArgs
static bool upgradeIntrinsicWithDefaultArgs(Function *F, Function *&NewFn)
Definition AutoUpgrade.cpp:1715
upgradeX86ConcatShift
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
Definition AutoUpgrade.cpp:2855
rename
static void rename(GlobalValue *GV)
Definition AutoUpgrade.cpp:67
upgradePTESTIntrinsic
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:80
upgradeX86BF16DPIntrinsic
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:169
NVVM_TMA_G2S_MODES
#define NVVM_TMA_G2S_MODES(M)
Definition AutoUpgrade.cpp:1194
upgradeNVVMFPArithCall
static Value * upgradeNVVMFPArithCall(IRBuilder<> &Builder, CallBase *CI, StringRef Name, const Intrinsic::ID(&IIDs)[2][3])
Definition AutoUpgrade.cpp:3291
upgradeMaskedCompare
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
Definition AutoUpgrade.cpp:2989
upgradeX86BinaryIntrinsics
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2775
upgradeNVVMIntrinsicCall
static Value * upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3317
shouldUpgradeNVPTXBulkG2SClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXBulkG2SClusterIntrinsic(Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
Definition AutoUpgrade.cpp:1295
upgradeX86MaskedShift
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:3021
shouldUpgradeNVPTXTcgen05AllocSharedIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05AllocSharedIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1421
upgradeAVX512MaskToSelect
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
Definition AutoUpgrade.cpp:3051
upgradeDbgIntrinsicToDbgRecord
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
Definition AutoUpgrade.cpp:5615
ConvertFunctionAttr
static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName)
Definition AutoUpgrade.cpp:7669
upgradePMULDQ
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
Definition AutoUpgrade.cpp:2937
getNVVMFPArithUpgrade
static std::optional< std::pair< Intrinsic::ID, RoundingMode > > getNVVMFPArithUpgrade(StringRef Name, const Intrinsic::ID(&IIDs)[2][3])
Definition AutoUpgrade.cpp:1544
reportFatalUsageErrorWithCI
static void reportFatalUsageErrorWithCI(StringRef reason, CallBase *CI)
Definition AutoUpgrade.cpp:71
upgradeMaskedStore
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2888
shouldUpgradeNVPTXBulkG2SCTAIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXBulkG2SCTAIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1323
shouldUpgradeNVPTXTMAG2SCTAIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1262
upgradeConvertIntrinsicCall
static Value * upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5676
G2S_CTA_CASE
#define G2S_CTA_CASE(ID_SUFFIX, NAME)
upgradeX86MultiplyAddWords
static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:143
upgradePtrauthInitFiniArrays
static bool upgradePtrauthInitFiniArrays(Module &M)
Definition AutoUpgrade.cpp:7165
upgradeX86IntrinsicCall
static Value * upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3532
getVPFPPredicateFromMD
static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5711
NVVMFAddIIDs
static constexpr Intrinsic::ID NVVMFAddIIDs[2][3]
Definition AutoUpgrade.cpp:1512
C
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
A
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Constants.h
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Enable
@ Enable
Definition DwarfDebug.cpp:87
Dwarf.h
This file contains constants used for implementing Dwarf debug support.
Module.h
Module.h This file contains the declarations for the Module class.
Ops
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Definition ItaniumDemangle.h:3392
isZero
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:540
F
#define F(x, y, z)
Definition MD5.cpp:54
I
#define I(x, y, z)
Definition MD5.cpp:57
R2
#define R2(n)
Metadata.h
This file contains the declarations for metadata subclasses.
T
#define T
Definition Mips16ISelLowering.cpp:274
T1
#define T1
Definition Mips16ISelLowering.cpp:273
NVPTXAddrSpace.h
NVPTX address space definition.
High
uint64_t High
Definition NVVMIntrRange.cpp:46
NVVMIntrinsicUtils.h
This file contains the definitions of the enumerations and flags associated with NVVM Intrinsics,...
C1
int64_t C1
Definition RISCVLateBranchOpt.cpp:64
CC
RISCVCC::CondCode CC
Definition RISCVLateBranchOpt.cpp:57
contains
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:479
StringExtras.h
This file contains some functions that are useful when dealing with strings.
StringSwitch.h
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
LLVM_DEBUG
#define LLVM_DEBUG(...)
Definition Debug.h:119
getType
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
Empty
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
Definition TypeHashing.cpp:17
Groups
static const X86InstrFMA3Group Groups[]
Definition X86InstrFMA3Info.cpp:81
RHS
Value * RHS
Definition X86PartialReduction.cpp:83
LHS
Value * LHS
Definition X86PartialReduction.cpp:82
llvm::APInt
Class for arbitrary precision integers.
Definition APInt.h:78
llvm::ArrayRef
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
llvm::ArrayType
Class to represent array types.
Definition DerivedTypes.h:453
llvm::ArrayType::get
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
llvm::ArrayType::getElementType
Type * getElementType() const
Definition DerivedTypes.h:466
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition Instructions.h:774
llvm::AtomicRMWInst::setVolatile
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
Definition Instructions.h:934
llvm::AtomicRMWInst::BinOp
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition Instructions.h:786
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition Instructions.h:790
llvm::AtomicRMWInst::FAdd
@ FAdd
*p = old + v
Definition Instructions.h:811
llvm::AtomicRMWInst::USubCond
@ USubCond
Subtract only if no unsigned overflow.
Definition Instructions.h:850
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition Instructions.h:804
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition Instructions.h:794
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition Instructions.h:800
llvm::AtomicRMWInst::USubSat
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
Definition Instructions.h:854
llvm::AtomicRMWInst::UIncWrap
@ UIncWrap
Increment one up to a maximum value.
Definition Instructions.h:842
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition Instructions.h:802
llvm::AtomicRMWInst::FMin
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition Instructions.h:822
llvm::AtomicRMWInst::FMax
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition Instructions.h:818
llvm::AtomicRMWInst::UDecWrap
@ UDecWrap
Decrement one until a minimum value or zero.
Definition Instructions.h:846
llvm::AtomicRMWInst::isFloatingPointOperation
bool isFloatingPointOperation() const
Definition Instructions.h:981
llvm::AttributeMask
This class stores enough information to efficiently remove some attributes from an existing AttrBuild...
Definition AttributeMask.h:29
llvm::AttributeMask::addAttribute
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
Definition AttributeMask.h:44
llvm::Attribute
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:106
llvm::Attribute::getWithStackAlignment
static LLVM_ABI Attribute getWithStackAlignment(LLVMContext &Context, Align Alignment)
Definition Attributes.cpp:226
llvm::Attribute::get
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition Attributes.cpp:95
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition InstrTypes.h:1181
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition InstrTypes.h:1479
llvm::CallBase::getOperandBundlesAsDefs
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition Instructions.cpp:508
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition InstrTypes.h:1417
llvm::CallBase::getCallingConv
CallingConv::ID getCallingConv() const
Definition InstrTypes.h:1475
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition InstrTypes.h:1409
llvm::CallBase::setAttributes
void setAttributes(AttributeList A)
Set the attributes for this call.
Definition InstrTypes.h:1496
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition InstrTypes.h:1361
llvm::CallBase::getFunctionType
FunctionType * getFunctionType() const
Definition InstrTypes.h:1274
llvm::CallBase::getIntrinsicID
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Definition Instructions.cpp:356
llvm::CallBase::args
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition InstrTypes.h:1352
llvm::CallBase::setCalledOperand
void setCalledOperand(Value *V)
Definition InstrTypes.h:1453
llvm::CallBase::arg_size
unsigned arg_size() const
Definition InstrTypes.h:1359
llvm::CallBase::getAttributes
AttributeList getAttributes() const
Return the attributes for this call.
Definition InstrTypes.h:1493
llvm::CallBase::setCalledFunction
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition InstrTypes.h:1456
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition Instructions.h:1595
llvm::CallInst::setTailCallKind
void setTailCallKind(TailCallKind TCK)
Definition Instructions.h:1715
llvm::CastInst::Create
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Definition Instructions.cpp:3198
llvm::CastInst::castIsValid
static LLVM_ABI bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Definition Instructions.cpp:3481
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
llvm::CmpInst::FCMP_OEQ
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
llvm::CmpInst::FCMP_OLT
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
llvm::CmpInst::FCMP_ULE
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
llvm::CmpInst::FCMP_OGT
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
llvm::CmpInst::FCMP_OGE
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
llvm::CmpInst::ICMP_UGE
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
llvm::CmpInst::FCMP_ULT
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
llvm::CmpInst::FCMP_ONE
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
llvm::CmpInst::FCMP_UEQ
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
llvm::CmpInst::FCMP_UGT
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:752
llvm::CmpInst::FCMP_OLE
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
llvm::CmpInst::FCMP_ORD
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition InstrTypes.h:762
llvm::CmpInst::ICMP_SGE
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
llvm::CmpInst::FCMP_UNE
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:756
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
llvm::CmpInst::FCMP_UGE
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
llvm::CmpInst::FCMP_UNO
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
llvm::ConstantAggregateZero::get
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
Definition Constants.cpp:1830
llvm::ConstantArray::get
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1454
llvm::ConstantAsMetadata::get
static ConstantAsMetadata * get(Constant *C)
Definition Metadata.h:548
llvm::ConstantExpr::getIntToPtr
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2521
llvm::ConstantExpr::getPointerCast
static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition Constants.cpp:2453
llvm::ConstantExpr::getPtrToInt
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2507
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition Constants.h:87
llvm::ConstantInt::isZero
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
llvm::ConstantPointerNull::get
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition Constants.cpp:2034
llvm::ConstantStruct::get
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1521
llvm::ConstantStruct::getType
StructType * getType() const
Specialization - reduce amount of casting.
Definition Constants.h:661
llvm::ConstantTokenNone::get
static LLVM_ABI ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition Constants.cpp:1685
llvm::Constant
This is an important base class in LLVM.
Definition Constant.h:44
llvm::Constant::getAllOnesValue
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
Definition Constants.cpp:449
llvm::Constant::getNullValue
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition Constants.cpp:389
llvm::DIExpression
DWARF expression.
Definition DebugInfoMetadata.h:3621
llvm::DIExpression::append
static LLVM_ABI DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
Definition DebugInfoMetadata.cpp:2445
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
llvm::DbgLabelRecord::createUnresolvedDbgLabelRecord
static LLVM_ABI DbgLabelRecord * createUnresolvedDbgLabelRecord(MDNode *Label)
For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved MDNodes.
Definition DebugProgramInstruction.cpp:159
llvm::DbgRecord
Base class for non-instruction debug metadata records that have positions within IR.
Definition DebugProgramInstruction.h:140
llvm::DbgRecord::setDebugLoc
void setDebugLoc(DebugLoc Loc)
Definition DebugProgramInstruction.h:218
llvm::DbgVariableRecord::createUnresolvedDbgVariableRecord
static LLVM_ABI DbgVariableRecord * createUnresolvedDbgVariableRecord(LocationType Type, Metadata *Val, MDNode *Variable, MDNode *Expression, MDNode *AssignID, Metadata *Address, MDNode *AddressExpression)
Used to create DbgVariableRecords during parsing, where some metadata references may still be unresol...
Definition DebugProgramInstruction.cpp:172
llvm::DiagnosticInfoDebugMetadataVersion
Diagnostic information for debug metadata version reporting.
Definition DiagnosticInfo.h:214
llvm::DiagnosticInfoIgnoringInvalidDebugMetadata
Diagnostic information for stripping invalid debug metadata.
Definition DiagnosticInfo.h:242
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
llvm::FastMathFlags::setApproxFunc
void setApproxFunc(bool B=true)
Definition FMF.h:93
llvm::FixedVectorType::get
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:843
llvm::FunctionType
Class to represent function types.
Definition DerivedTypes.h:165
llvm::FunctionType::getNumParams
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition DerivedTypes.h:204
llvm::FunctionType::getParamType
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition DerivedTypes.h:197
llvm::FunctionType::getReturnType
Type * getReturnType() const
Definition DerivedTypes.h:186
llvm::FunctionType::get
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition Function.h:169
llvm::Function::getFunctionType
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:212
llvm::Function::getIntrinsicID
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:247
llvm::Function::getFunction
const Function & getFunction() const
Definition Function.h:167
llvm::Function::eraseFromParent
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition Function.cpp:451
llvm::Function::arg_size
size_t arg_size() const
Definition Function.h:886
llvm::Function::getReturnType
Type * getReturnType() const
Returns the type of the ret val.
Definition Function.h:217
llvm::Function::getArg
Argument * getArg(unsigned i) const
Definition Function.h:871
llvm::GlobalValue::getGUIDAssumingExternalLinkage
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
Definition Globals.cpp:80
llvm::GlobalValue::getLinkage
LinkageTypes getLinkage() const
Definition GlobalValue.h:556
llvm::GlobalValue::GUID
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
Definition GlobalValue.h:602
llvm::GlobalValue::dropLLVMManglingEscape
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition GlobalValue.h:594
llvm::GlobalValue::getParent
Module * getParent()
Get the module that this global value is contained inside of...
Definition GlobalValue.h:728
llvm::GlobalValue::getValueType
Type * getValueType() const
Definition GlobalValue.h:302
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition GlobalVariable.h:155
llvm::GlobalVariable::hasInitializer
bool hasInitializer() const
Definitions have initializers, declarations don't.
Definition GlobalVariable.h:111
llvm::IRBuilderBase::getPtrTy
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Definition IRBuilder.h:565
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2949
llvm::InstVisitor
Base class for instruction visitors.
Definition InstVisitor.h:78
llvm::Instruction::isCast
bool isCast() const
Definition Instruction.h:357
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition Instruction.h:555
llvm::Instruction::getModule
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition Instruction.cpp:83
llvm::Instruction::isBinaryOp
bool isBinaryOp() const
Definition Instruction.h:353
llvm::Instruction::eraseFromParent
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition Instruction.cpp:109
llvm::Instruction::setMetadata
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition Metadata.cpp:1788
llvm::Instruction::getFastMathFlags
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition Instruction.cpp:697
llvm::Instruction::isUnaryOp
bool isUnaryOp() const
Definition Instruction.h:352
llvm::Instruction::copyMetadata
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Definition Instruction.cpp:1543
llvm::Instruction::getDataLayout
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Definition Instruction.cpp:91
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
llvm::LLVMContext::getOrInsertSyncScopeID
LLVM_ABI SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
Definition LLVMContext.cpp:320
llvm::LoadInst
An instruction for reading from memory.
Definition Instructions.h:197
llvm::MDBuilder::createRange
LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
Definition MDBuilder.cpp:92
llvm::MDNode
Metadata node.
Definition Metadata.h:1081
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1437
llvm::MDNode::op_end
op_iterator op_end() const
Definition Metadata.h:1431
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1579
llvm::MDNode::getNumOperands
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1443
llvm::MDNode::op_begin
op_iterator op_begin() const
Definition Metadata.h:1427
llvm::MDNode::getContext
LLVMContext & getContext() const
Definition Metadata.h:1245
llvm::MDOperand
Tracking metadata reference owned by Metadata.
Definition Metadata.h:902
llvm::MDString
A single uniqued string.
Definition Metadata.h:733
llvm::MDString::getString
LLVM_ABI StringRef getString() const
Definition Metadata.cpp:613
llvm::MDString::get
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
Definition Metadata.cpp:595
llvm::MDTuple
Tuple of metadata.
Definition Metadata.h:1496
llvm::MDTuple::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1525
llvm::MetadataAsValue
Metadata wrapper in the Value hierarchy.
Definition Metadata.h:184
llvm::MetadataAsValue::get
static LLVM_ABI MetadataAsValue * get(LLVMContext &Context, Metadata *MD)
Definition Metadata.cpp:105
llvm::Metadata
Root of the metadata hierarchy.
Definition Metadata.h:64
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
llvm::Module::ModFlagBehavior
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
Definition Module.h:118
llvm::Module::Override
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
Definition Module.h:139
llvm::Module::Error
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
Definition Module.h:121
llvm::Module::Min
@ Min
Takes the min of the two values, which are required to be integers.
Definition Module.h:153
llvm::Module::Max
@ Max
Takes the max of the two values, which are required to be integers.
Definition Module.h:150
llvm::NamedMDNode
A tuple of MDNodes.
Definition Metadata.h:1797
llvm::NamedMDNode::setOperand
LLVM_ABI void setOperand(unsigned I, MDNode *New)
Definition Metadata.cpp:1592
llvm::NamedMDNode::getOperand
LLVM_ABI MDNode * getOperand(unsigned i) const
Definition Metadata.cpp:1584
llvm::NamedMDNode::getNumOperands
LLVM_ABI unsigned getNumOperands() const
Definition Metadata.cpp:1580
llvm::NamedMDNode::clearOperands
LLVM_ABI void clearOperands()
Drop all references to this node's operands.
Definition Metadata.cpp:1599
llvm::NamedMDNode::operands
iterator_range< op_iterator > operands()
Definition Metadata.h:1893
llvm::NamedMDNode::addOperand
LLVM_ABI void addOperand(MDNode *M)
Definition Metadata.cpp:1590
llvm::OperandBundleDefT::inputs
ArrayRef< InputTy > inputs() const
Definition InstrTypes.h:1149
llvm::OperandBundleDefT::getTag
StringRef getTag() const
Definition InstrTypes.h:1157
llvm::PoisonValue::get
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition Constants.cpp:2092
llvm::Regex::match
LLVM_ABI bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Definition Regex.cpp:84
llvm::ScalableVectorType::get
static LLVM_ABI ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
Definition Type.cpp:865
llvm::ShuffleVectorInst::getShuffleMask
ArrayRef< int > getShuffleMask() const
Definition Instructions.h:2095
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition SmallPtrSet.h:373
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition SmallPtrSet.h:519
llvm::SmallString
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition SmallString.h:26
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition SmallVector.h:594
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition SmallVector.h:995
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition SmallVector.h:720
llvm::SmallVectorTemplateBase::push_back
void push_back(const T &Elt)
Definition SmallVector.h:425
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition SmallVector.h:1261
llvm::StoreInst
An instruction for storing to memory.
Definition Instructions.h:338
llvm::StringLiteral
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Definition StringRef.h:888
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::StringRef::split
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition StringRef.h:736
llvm::StringRef::npos
static constexpr size_t npos
Definition StringRef.h:58
llvm::StringRef::substr
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition StringRef.h:597
llvm::StringRef::starts_with
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
llvm::StringRef::empty
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
llvm::StringRef::drop_front
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition StringRef.h:635
llvm::StringRef::size
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
llvm::StringRef::find
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
Definition StringRef.h:290
llvm::StringRef::trim
StringRef trim(char Char) const
Return string with consecutive Char characters starting from the left and right removed.
Definition StringRef.h:850
llvm::StringRef::consume_front
bool consume_front(char Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
Definition StringRef.h:661
llvm::StringSwitch
A switch()-like statement whose cases are string literals.
Definition StringSwitch.h:47
llvm::StringSwitch::Case
StringSwitch & Case(StringLiteral S, T Value)
Definition StringSwitch.h:69
llvm::StringSwitch::StartsWith
StringSwitch & StartsWith(StringLiteral S, T Value)
Definition StringSwitch.h:81
llvm::StringSwitch::Cases
StringSwitch & Cases(std::initializer_list< StringLiteral > CaseStrings, T Value)
Definition StringSwitch.h:88
llvm::StructType
Class to represent struct types.
Definition DerivedTypes.h:278
llvm::StructType::get
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:467
llvm::StructType::getNumElements
unsigned getNumElements() const
Random access to the elements.
Definition DerivedTypes.h:423
llvm::StructType::getElementType
Type * getElementType(unsigned N) const
Definition DerivedTypes.h:424
llvm::TimeTraceScope
The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.
Definition TimeProfiler.h:194
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:48
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
llvm::Type::getInt64Ty
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:300
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
llvm::Type::getInt32Ty
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:299
llvm::Type::isFloatTy
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
llvm::Type::isBFloatTy
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
Definition Type.h:147
llvm::Type::getPointerAddressSpace
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition DerivedTypes.h:818
llvm::Type::getInt8Ty
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:297
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:363
llvm::Type::getPrimitiveSizeInBits
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
llvm::Type::getInt16Ty
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
Definition Type.cpp:298
llvm::Type::getScalarSizeInBits
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:222
llvm::Type::isPtrOrPtrVectorTy
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:280
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
llvm::Type::isFPOrFPVectorTy
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:222
llvm::Type::getFloatTy
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:276
llvm::Type::getBFloatTy
static LLVM_ABI Type * getBFloatTy(LLVMContext &C)
Definition Type.cpp:275
llvm::Type::getHalfTy
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:274
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition User.h:207
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition User.h:229
llvm::Value
LLVM Value Representation.
Definition Value.h:75
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
llvm::Value::print
LLVM_ABI void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
Definition AsmWriter.cpp:5354
llvm::Value::setMetadata
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set a particular kind of metadata attachment.
Definition Metadata.cpp:1670
llvm::Value::setName
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:389
llvm::Value::replaceAllUsesWith
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:548
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
llvm::Value::users
iterator_range< user_iterator > users()
Definition Value.h:428
llvm::Value::stripPointerCasts
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:707
llvm::Value::use_empty
bool use_empty() const
Definition Value.h:348
llvm::Value::hasName
bool hasName() const
Definition Value.h:263
llvm::Value::getName
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
llvm::Value::takeName
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:395
llvm::VectorType
Base class of all SIMD vector types.
Definition DerivedTypes.h:485
llvm::VectorType::getInteger
static VectorType * getInteger(VectorType *VTy)
This static method gets a VectorType with the same number of elements as the input type,...
Definition DerivedTypes.h:536
llvm::VectorType::get
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
llvm::details::FixedOrScalableQuantity::getFixedValue
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
llvm::ilist_detail::node_parent_access::getParent
const ParentTy * getParent() const
Definition ilist_node.h:34
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition ilist_node.h:123
llvm::raw_svector_ostream
A raw_ostream that writes to an SmallVector or SmallString.
Definition raw_ostream.h:694
llvm::raw_svector_ostream::str
StringRef str() const
Return a StringRef for the vector contents.
Definition raw_ostream.h:723
uint32_t
uint8_t
Call
CallInst * Call
Definition ObjCARCOpts.cpp:2458
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::AMDGPUAS::LOCAL_ADDRESS
@ LOCAL_ADDRESS
Address space for local memory.
Definition AMDGPUAddrSpace.h:35
llvm::AMDGPUAS::FLAT_ADDRESS
@ FLAT_ADDRESS
Address space for flat memory.
Definition AMDGPUAddrSpace.h:31
llvm::AMDGPUAS::PRIVATE_ADDRESS
@ PRIVATE_ADDRESS
Address space for private memory.
Definition AMDGPUAddrSpace.h:37
llvm::ARM_MB::ST
@ ST
Definition ARMBaseInfo.h:73
llvm::CallingConv::PTX_Kernel
@ PTX_Kernel
Call to a PTX kernel. Passes all arguments in parameter space.
Definition CallingConv.h:125
llvm::FloatABI::parseABIType
std::optional< ABIType > parseABIType(StringRef S)
Parse the string spelling used by the "float-abi" IR module flag into an ABIType.
Definition CodeGen.h:167
llvm::Intrinsic::remangleIntrinsicFunction
LLVM_ABI std::optional< Function * > remangleIntrinsicFunction(Function *F)
Definition Intrinsics.cpp:1451
llvm::Intrinsic::getOrInsertDeclaration
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
Definition Intrinsics.cpp:802
llvm::Intrinsic::getAttributes
LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)
Return the attributes for an intrinsic.
llvm::Intrinsic::getType
LLVM_ABI FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > OverloadTys={})
Return the function type for an intrinsic.
Definition Intrinsics.cpp:637
llvm::Intrinsic::isSignatureValid
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
Definition Intrinsics.cpp:1423
llvm::Intrinsic::hasStructReturnType
LLVM_ABI bool hasStructReturnType(ID id)
Returns true if id has a struct return type.
Definition Intrinsics.cpp:1416
llvm::Intrinsic::getAllDefaultArgValues
LLVM_ABI std::pair< unsigned, ArrayRef< uint64_t > > getAllDefaultArgValues(ID IID)
Returns the first default argument index and an ArrayRef of all default values for the trailing param...
llvm::NVVMAttr::GridConstant
constexpr StringLiteral GridConstant("nvvm.grid_constant")
llvm::NVVMAttr::MaxNTID
constexpr StringLiteral MaxNTID("nvvm.maxntid")
llvm::NVVMAttr::MaxNReg
constexpr StringLiteral MaxNReg("nvvm.maxnreg")
llvm::NVVMAttr::MinCTASm
constexpr StringLiteral MinCTASm("nvvm.minctasm")
llvm::NVVMAttr::ReqNTID
constexpr StringLiteral ReqNTID("nvvm.reqntid")
llvm::NVVMAttr::MaxClusterRank
constexpr StringLiteral MaxClusterRank("nvvm.maxclusterrank")
llvm::NVVMAttr::ClusterDim
constexpr StringLiteral ClusterDim("nvvm.cluster_dim")
llvm::mdconst::dyn_extract_or_null
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract_or_null(Y &&MD)
Extract a Value from Metadata, if any, allowing null.
Definition Metadata.h:720
llvm::mdconst::hasa
std::enable_if_t< detail::IsValidPointer< X, Y >::value, bool > hasa(Y &&MD)
Check whether Metadata has a Value.
Definition Metadata.h:662
llvm::mdconst::dyn_extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
Definition Metadata.h:707
llvm::mdconst::extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:679
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
Definition MicrosoftDemangle.h:132
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition DefaultTargetInfo.h:21
llvm::Offset
@ Offset
Definition DWP.cpp:577
llvm::Length
@ Length
Definition DWP.cpp:577
llvm::UpgradeIntrinsicCall
LLVM_ABI void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
Definition AutoUpgrade.cpp:5825
llvm::UpgradeSectionAttributes
LLVM_ABI void UpgradeSectionAttributes(Module &M)
Definition AutoUpgrade.cpp:7482
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
llvm::UpgradeInlineAsmString
LLVM_ABI void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
Definition AutoUpgrade.cpp:3282
llvm::isValidAtomicOrdering
bool isValidAtomicOrdering(Int I)
Definition AtomicOrdering.h:75
llvm::dyn_cast
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
llvm::VectorInstrContext::Load
@ Load
The value being inserted comes from a load (InsertElement only).
Definition TargetTransformInfo.h:201
llvm::getLongDoubleFormatName
StringRef getLongDoubleFormatName(LongDoubleFormat Format)
Returns the IR floating-point type name for a LongDoubleFormat.
Definition CodeGen.h:126
llvm::LongDoubleFormat
LongDoubleFormat
The floating-point format used for the target's "long double" type.
Definition CodeGen.h:117
llvm::UpgradeIntrinsicFunction
LLVM_ABI bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
Definition AutoUpgrade.cpp:2504
llvm::upgradeInstructionLoopAttachment
LLVM_ABI MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
Definition AutoUpgrade.cpp:7873
llvm::dyn_cast_if_present
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
llvm::UpgradeAttributes
LLVM_ABI void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
Definition AutoUpgrade.cpp:8062
llvm::UpgradeCallsToIntrinsic
LLVM_ABI void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
Definition AutoUpgrade.cpp:6708
llvm::UpgradeNVVMAnnotations
LLVM_ABI void UpgradeNVVMAnnotations(Module &M)
Convert legacy nvvm.annotations metadata to appropriate function attributes.
Definition AutoUpgrade.cpp:6969
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
llvm::UpgradeModuleFlags
LLVM_ABI bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
Definition AutoUpgrade.cpp:7264
llvm::utostr
std::string utostr(uint64_t X, bool isNeg=false)
Definition StringExtras.h:322
llvm::isPowerOf2_64
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
llvm::UpgradeCFIFunctionsMetadata
LLVM_ABI bool UpgradeCFIFunctionsMetadata(Module &M)
Upgrade the cfi.functions metadata node by calculating and inserting the GUID for each function entry...
Definition AutoUpgrade.cpp:7438
llvm::copyModuleAttrToFunctions
LLVM_ABI void copyModuleAttrToFunctions(Module &M)
Copies module attributes to the functions in the module.
Definition AutoUpgrade.cpp:7698
llvm::UpgradeOperandBundles
LLVM_ABI void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
Definition AutoUpgrade.cpp:8100
llvm::UpgradeBitCastExpr
LLVM_ABI Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
Definition AutoUpgrade.cpp:6797
llvm::dyn_cast_or_null
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
llvm::dbgs
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
llvm::UpgradeDataLayoutString
LLVM_ABI std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
Definition AutoUpgrade.cpp:7905
llvm::none_of
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1769
llvm::report_fatal_error
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
llvm::UpgradeTBAAStructNode
LLVM_ABI MDNode * UpgradeTBAAStructNode(MDNode &TBAAStructNode)
If the given !tbaa.struct node has old-style scalar field tags, return an equivalent node with each f...
Definition AutoUpgrade.cpp:6753
llvm::isa
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
llvm::UpgradeGlobalVariable
LLVM_ABI GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
Definition AutoUpgrade.cpp:2523
llvm::Key
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
Definition PassManager.h:695
llvm::errs
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition raw_ostream.cpp:890
llvm::StripDebugInfo
LLVM_ABI bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
Definition DebugInfo.cpp:718
llvm::drop_end
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Definition STLExtras.h:323
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition AtomicOrdering.h:56
llvm::ModRefInfo::Ref
@ Ref
The access may reference the value stored in memory.
Definition ModRef.h:32
llvm::join
std::string join(IteratorT Begin, IteratorT End, StringRef Separator)
Joins the strings in the range [Begin, End), adding Separator between the elements.
Definition StringExtras.h:492
llvm::findBooleanLoopTags
const BooleanLoopTags * findBooleanLoopTags(StringRef Name)
Return the replacement tags for the enable tag Name, or nullptr.
Definition AutoUpgrade.h:134
llvm::OperandBundleDef
OperandBundleDefT< Value * > OperandBundleDef
Definition AutoUpgrade.h:34
llvm::UpgradeBitCastInst
LLVM_ABI Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
Definition AutoUpgrade.cpp:6775
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:25
llvm::RoundingMode
RoundingMode
Rounding mode.
Definition FloatingPointMode.h:38
llvm::RoundingMode::TowardZero
@ TowardZero
roundTowardZero.
Definition FloatingPointMode.h:40
llvm::RoundingMode::NearestTiesToEven
@ NearestTiesToEven
roundTiesToEven.
Definition FloatingPointMode.h:41
llvm::RoundingMode::Dynamic
@ Dynamic
Denotes mode unknown at compile time.
Definition FloatingPointMode.h:47
llvm::RoundingMode::TowardPositive
@ TowardPositive
roundTowardPositive.
Definition FloatingPointMode.h:42
llvm::RoundingMode::TowardNegative
@ TowardNegative
roundTowardNegative.
Definition FloatingPointMode.h:43
llvm::ArrayRef
ArrayRef(const T &OneElt) -> ArrayRef< T >
llvm::getIROptions
const IROptions & getIROptions(const LLVMContext &C)
Definition IROptions.h:27
llvm::parseDenormalFPAttribute
DenormalMode parseDenormalFPAttribute(StringRef Str)
Returns the denormal mode to use for inputs and outputs.
Definition FloatingPointMode.h:254
llvm::cast
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
llvm::erase_if
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2208
llvm::UpgradeDebugInfo
LLVM_ABI bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
Definition AutoUpgrade.cpp:6827
llvm::UpgradeFunctionAttributes
LLVM_ABI void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
Definition AutoUpgrade.cpp:7555
llvm::UpgradeTBAANode
LLVM_ABI MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
Definition AutoUpgrade.cpp:6727
llvm::UpgradeARCRuntime
LLVM_ABI void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
Definition AutoUpgrade.cpp:7034
llvm::DEBUG_METADATA_VERSION
@ DEBUG_METADATA_VERSION
Definition Metadata.h:54
llvm::verifyModule
LLVM_ABI bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
Definition Verifier.cpp:8161
llvm::reportFatalUsageError
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:880
N
#define N
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
llvm::BooleanLoopTags
Single-operand tags replacing a removed two-operand form !
Definition AutoUpgrade.h:120
llvm::BooleanLoopTags::Disable
StringLiteral Disable
Definition AutoUpgrade.h:122
llvm::BooleanLoopTags::Enable
StringLiteral Enable
Definition AutoUpgrade.h:121
llvm::DenormalFPEnv
Represents the full denormal controls for a function, including the default mode and the f32 specific...
Definition FloatingPointMode.h:279
llvm::DenormalMode
Represent subnormal handling kind for floating point instruction inputs and outputs.
Definition FloatingPointMode.h:71
llvm::DenormalMode::getInvalid
static constexpr DenormalMode getInvalid()
Definition FloatingPointMode.h:106
llvm::DenormalMode::isValid
constexpr bool isValid() const
Definition FloatingPointMode.h:156
llvm::DenormalMode::getIEEE
static constexpr DenormalMode getIEEE()
Definition FloatingPointMode.h:115
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:111
Generated on for LLVM by doxygen 1.14.0