LLVM 24.0.0git
NVVMIntrinsicUtils.h
Go to the documentation of this file.
1//===--- NVVMIntrinsicUtils.h -----------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file contains the definitions of the enumerations and flags
11/// associated with NVVM Intrinsics, along with some helper functions.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_IR_NVVMINTRINSICUTILS_H
16#define LLVM_IR_NVVMINTRINSICUTILS_H
17
18#include <stdint.h>
19
20#include "llvm/ADT/APFloat.h"
21#include "llvm/ADT/APInt.h"
22#include "llvm/ADT/StringRef.h"
23#include "llvm/IR/Constants.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsNVPTX.h"
26#include "llvm/Support/raw_ostream.h"
27
28namespace llvm {
29namespace nvvm {
30
31// Reduction Ops supported with TMA Copy from Shared
32// to Global Memory for the "cp.reduce.async.bulk.tensor.*"
33// family of PTX instructions.
34enum class TMAReductionOp : uint8_t {
35 ADD = 0,
36 MIN = 1,
37 MAX = 2,
38 INC = 3,
39 DEC = 4,
40 AND = 5,
41 OR = 6,
42 XOR = 7,
43};
44
45inline StringRef getTMATensorReductionOpName(TMAReductionOp Op) {
46 switch (Op) {
47 case TMAReductionOp::ADD:
48 return "add";
49 case TMAReductionOp::MIN:
50 return "min";
51 case TMAReductionOp::MAX:
52 return "max";
53 case TMAReductionOp::INC:
54 return "inc";
55 case TMAReductionOp::DEC:
56 return "dec";
57 case TMAReductionOp::AND:
58 return "and";
59 case TMAReductionOp::OR:
60 return "or";
61 case TMAReductionOp::XOR:
62 return "xor";
63 }
64 llvm_unreachable("invalid TMA tensorreduction operation");
65}
66
67// Enum to represent the cta_group::1 and
68// cta_group::2 variants in TMA/TCGEN05 family of
69// PTX instructions.
70enum class CTAGroupKind : uint8_t {
71 CG_NONE = 0, // default with no cta_group modifier
72 CG_1 = 1, // cta_group::1 modifier
73 CG_2 = 2, // cta_group::2 modifier
74};
75
76// Validate data patterns supported with the TMA Copy from Global to Shared
77// memory (CTA and Cluster) intrinsics.
78enum class TMAValidateDataPattern : uint8_t {
79 DISABLED = 0, // Default; no validate_data qualifier is emitted.
80 PER_16BYTES_80000000 = 1,
81 PER_16BYTES_8000 = 2,
82 PER_16BYTES_80 = 3,
83 PER_16BYTES_8 = 4,
84 PER_ELEMENT_FF = 5,
85};
86
87inline StringRef getTMAValidateDataPatternName(TMAValidateDataPattern Pattern) {
88 using VDTy = TMAValidateDataPattern;
89 switch (Pattern) {
90 case VDTy::DISABLED:
91 return "disabled";
92 case VDTy::PER_16BYTES_80000000:
93 return "per_16bytes::80000000";
94 case VDTy::PER_16BYTES_8000:
95 return "per_16bytes::8000";
96 case VDTy::PER_16BYTES_80:
97 return "per_16bytes::80";
98 case VDTy::PER_16BYTES_8:
99 return "per_16bytes::8";
100 case VDTy::PER_ELEMENT_FF:
101 return "per_element::ff";
102 }
103 llvm_unreachable("invalid TMA validate data pattern");
104}
105
106// Eviction priorities applicable for prefetch and applypriority intrinsics.
107enum class EvictPolicyType : uint8_t {
108 EVICT_NORMAL = 0, // default
109 EVICT_LAST = 1,
110};
111
112inline StringRef getEvictPolicyName(EvictPolicyType Policy) {
113 switch (Policy) {
114 case EvictPolicyType::EVICT_NORMAL:
115 return "L2::evict_normal";
116 case EvictPolicyType::EVICT_LAST:
117 return "L2::evict_last";
118 }
119 llvm_unreachable("invalid evict policy");
120}
121
122enum class Tcgen05MMAKind : uint8_t {
123 F16 = 0,
124 TF32 = 1,
125 F8F6F4 = 2,
126 I8 = 3,
127 TI16 = 4,
128};
129
136
137enum class Tcgen05MMACollectorBBuffer : uint8_t {
138 B0 = 0,
139 B1 = 1,
140 B2 = 2,
141 B3 = 3,
142};
143
144enum class TensormapElemType : uint8_t {
145 U8 = 0,
146 U16 = 1,
147 U32 = 2,
148 S32 = 3,
149 U64 = 4,
150 S64 = 5,
151 F16 = 6,
152 F32 = 7,
153 F32_FTZ = 8,
154 F64 = 9,
155 BF16 = 10,
156 TF32 = 11,
157 TF32_FTZ = 12,
158 B4x16 = 13,
159 B4x16_p64 = 14,
160 B6x16_p32 = 15,
161};
162
168
176
183
188
189LLVM_ABI void printTcgen05MMAKind(raw_ostream &OS, const Constant *ImmArgVal);
190
191LLVM_ABI void printEvictPolicyType(raw_ostream &OS, const Constant *ImmArgVal);
192
193LLVM_ABI void printTMAReductionOp(raw_ostream &OS, const Constant *ImmArgVal);
194
195LLVM_ABI void printTMAValidateDataPattern(raw_ostream &OS,
196 const Constant *ImmArgVal);
197
198LLVM_ABI void printTcgen05CollectorUsageOp(raw_ostream &OS,
199 const Constant *ImmArgVal);
200
201LLVM_ABI void printTcgen05MMACollectorBBuffer(raw_ostream &OS,
202 const Constant *ImmArgVal);
203
204LLVM_ABI void printTensormapElemType(raw_ostream &OS,
205 const Constant *ImmArgVal);
206LLVM_ABI void printTensormapInterleaveLayout(raw_ostream &OS,
207 const Constant *ImmArgVal);
208LLVM_ABI void printTensormapSwizzleMode(raw_ostream &OS,
209 const Constant *ImmArgVal);
210LLVM_ABI void printTensormapSwizzleAtomicity(raw_ostream &OS,
211 const Constant *ImmArgVal);
212LLVM_ABI void printTensormapFillMode(raw_ostream &OS,
213 const Constant *ImmArgVal);
214LLVM_ABI void printFPRoundingMode(raw_ostream &OS, const Constant *ImmArgVal);
215
216inline bool FPToIntegerIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID) {
217 switch (IntrinsicID) {
218 case Intrinsic::nvvm_f2i_rm_ftz:
219 case Intrinsic::nvvm_f2i_rn_ftz:
220 case Intrinsic::nvvm_f2i_rp_ftz:
221 case Intrinsic::nvvm_f2i_rz_ftz:
222
223 case Intrinsic::nvvm_f2ui_rm_ftz:
224 case Intrinsic::nvvm_f2ui_rn_ftz:
225 case Intrinsic::nvvm_f2ui_rp_ftz:
226 case Intrinsic::nvvm_f2ui_rz_ftz:
227
228 case Intrinsic::nvvm_f2ll_rm_ftz:
229 case Intrinsic::nvvm_f2ll_rn_ftz:
230 case Intrinsic::nvvm_f2ll_rp_ftz:
231 case Intrinsic::nvvm_f2ll_rz_ftz:
232
233 case Intrinsic::nvvm_f2ull_rm_ftz:
234 case Intrinsic::nvvm_f2ull_rn_ftz:
235 case Intrinsic::nvvm_f2ull_rp_ftz:
236 case Intrinsic::nvvm_f2ull_rz_ftz:
237 return true;
238
239 case Intrinsic::nvvm_f2i_rm:
240 case Intrinsic::nvvm_f2i_rn:
241 case Intrinsic::nvvm_f2i_rp:
242 case Intrinsic::nvvm_f2i_rz:
243
244 case Intrinsic::nvvm_f2ui_rm:
245 case Intrinsic::nvvm_f2ui_rn:
246 case Intrinsic::nvvm_f2ui_rp:
247 case Intrinsic::nvvm_f2ui_rz:
248
249 case Intrinsic::nvvm_d2i_rm:
250 case Intrinsic::nvvm_d2i_rn:
251 case Intrinsic::nvvm_d2i_rp:
252 case Intrinsic::nvvm_d2i_rz:
253
254 case Intrinsic::nvvm_d2ui_rm:
255 case Intrinsic::nvvm_d2ui_rn:
256 case Intrinsic::nvvm_d2ui_rp:
257 case Intrinsic::nvvm_d2ui_rz:
258
259 case Intrinsic::nvvm_f2ll_rm:
260 case Intrinsic::nvvm_f2ll_rn:
261 case Intrinsic::nvvm_f2ll_rp:
262 case Intrinsic::nvvm_f2ll_rz:
263
264 case Intrinsic::nvvm_f2ull_rm:
265 case Intrinsic::nvvm_f2ull_rn:
266 case Intrinsic::nvvm_f2ull_rp:
267 case Intrinsic::nvvm_f2ull_rz:
268
269 case Intrinsic::nvvm_d2ll_rm:
270 case Intrinsic::nvvm_d2ll_rn:
271 case Intrinsic::nvvm_d2ll_rp:
272 case Intrinsic::nvvm_d2ll_rz:
273
274 case Intrinsic::nvvm_d2ull_rm:
275 case Intrinsic::nvvm_d2ull_rn:
276 case Intrinsic::nvvm_d2ull_rp:
277 case Intrinsic::nvvm_d2ull_rz:
278 return false;
279 }
280 llvm_unreachable("Checking FTZ flag for invalid f2i/d2i intrinsic");
281}
282
283inline bool FPToIntegerIntrinsicResultIsSigned(Intrinsic::ID IntrinsicID) {
284 switch (IntrinsicID) {
285 // f2i
286 case Intrinsic::nvvm_f2i_rm:
287 case Intrinsic::nvvm_f2i_rm_ftz:
288 case Intrinsic::nvvm_f2i_rn:
289 case Intrinsic::nvvm_f2i_rn_ftz:
290 case Intrinsic::nvvm_f2i_rp:
291 case Intrinsic::nvvm_f2i_rp_ftz:
292 case Intrinsic::nvvm_f2i_rz:
293 case Intrinsic::nvvm_f2i_rz_ftz:
294 // d2i
295 case Intrinsic::nvvm_d2i_rm:
296 case Intrinsic::nvvm_d2i_rn:
297 case Intrinsic::nvvm_d2i_rp:
298 case Intrinsic::nvvm_d2i_rz:
299 // f2ll
300 case Intrinsic::nvvm_f2ll_rm:
301 case Intrinsic::nvvm_f2ll_rm_ftz:
302 case Intrinsic::nvvm_f2ll_rn:
303 case Intrinsic::nvvm_f2ll_rn_ftz:
304 case Intrinsic::nvvm_f2ll_rp:
305 case Intrinsic::nvvm_f2ll_rp_ftz:
306 case Intrinsic::nvvm_f2ll_rz:
307 case Intrinsic::nvvm_f2ll_rz_ftz:
308 // d2ll
309 case Intrinsic::nvvm_d2ll_rm:
310 case Intrinsic::nvvm_d2ll_rn:
311 case Intrinsic::nvvm_d2ll_rp:
312 case Intrinsic::nvvm_d2ll_rz:
313 return true;
314
315 // f2ui
316 case Intrinsic::nvvm_f2ui_rm:
317 case Intrinsic::nvvm_f2ui_rm_ftz:
318 case Intrinsic::nvvm_f2ui_rn:
319 case Intrinsic::nvvm_f2ui_rn_ftz:
320 case Intrinsic::nvvm_f2ui_rp:
321 case Intrinsic::nvvm_f2ui_rp_ftz:
322 case Intrinsic::nvvm_f2ui_rz:
323 case Intrinsic::nvvm_f2ui_rz_ftz:
324 // d2ui
325 case Intrinsic::nvvm_d2ui_rm:
326 case Intrinsic::nvvm_d2ui_rn:
327 case Intrinsic::nvvm_d2ui_rp:
328 case Intrinsic::nvvm_d2ui_rz:
329 // f2ull
330 case Intrinsic::nvvm_f2ull_rm:
331 case Intrinsic::nvvm_f2ull_rm_ftz:
332 case Intrinsic::nvvm_f2ull_rn:
333 case Intrinsic::nvvm_f2ull_rn_ftz:
334 case Intrinsic::nvvm_f2ull_rp:
335 case Intrinsic::nvvm_f2ull_rp_ftz:
336 case Intrinsic::nvvm_f2ull_rz:
337 case Intrinsic::nvvm_f2ull_rz_ftz:
338 // d2ull
339 case Intrinsic::nvvm_d2ull_rm:
340 case Intrinsic::nvvm_d2ull_rn:
341 case Intrinsic::nvvm_d2ull_rp:
342 case Intrinsic::nvvm_d2ull_rz:
343 return false;
344 }
345 llvm_unreachable(
346 "Checking invalid f2i/d2i intrinsic for signed int conversion");
347}
348
349inline bool FPToIntegerIntrinsicNaNZero(Intrinsic::ID IntrinsicID) {
350 switch (IntrinsicID) {
351 // f2i
352 case Intrinsic::nvvm_f2i_rm:
353 case Intrinsic::nvvm_f2i_rn:
354 case Intrinsic::nvvm_f2i_rp:
355 case Intrinsic::nvvm_f2i_rz:
356 case Intrinsic::nvvm_f2i_rm_ftz:
357 case Intrinsic::nvvm_f2i_rn_ftz:
358 case Intrinsic::nvvm_f2i_rp_ftz:
359 case Intrinsic::nvvm_f2i_rz_ftz:
360 // f2ui
361 case Intrinsic::nvvm_f2ui_rm:
362 case Intrinsic::nvvm_f2ui_rn:
363 case Intrinsic::nvvm_f2ui_rp:
364 case Intrinsic::nvvm_f2ui_rz:
365 case Intrinsic::nvvm_f2ui_rm_ftz:
366 case Intrinsic::nvvm_f2ui_rn_ftz:
367 case Intrinsic::nvvm_f2ui_rp_ftz:
368 case Intrinsic::nvvm_f2ui_rz_ftz:
369 return true;
370 // d2i
371 case Intrinsic::nvvm_d2i_rm:
372 case Intrinsic::nvvm_d2i_rn:
373 case Intrinsic::nvvm_d2i_rp:
374 case Intrinsic::nvvm_d2i_rz:
375 // d2ui
376 case Intrinsic::nvvm_d2ui_rm:
377 case Intrinsic::nvvm_d2ui_rn:
378 case Intrinsic::nvvm_d2ui_rp:
379 case Intrinsic::nvvm_d2ui_rz:
380 // f2ll
381 case Intrinsic::nvvm_f2ll_rm:
382 case Intrinsic::nvvm_f2ll_rn:
383 case Intrinsic::nvvm_f2ll_rp:
384 case Intrinsic::nvvm_f2ll_rz:
385 case Intrinsic::nvvm_f2ll_rm_ftz:
386 case Intrinsic::nvvm_f2ll_rn_ftz:
387 case Intrinsic::nvvm_f2ll_rp_ftz:
388 case Intrinsic::nvvm_f2ll_rz_ftz:
389 // f2ull
390 case Intrinsic::nvvm_f2ull_rm:
391 case Intrinsic::nvvm_f2ull_rn:
392 case Intrinsic::nvvm_f2ull_rp:
393 case Intrinsic::nvvm_f2ull_rz:
394 case Intrinsic::nvvm_f2ull_rm_ftz:
395 case Intrinsic::nvvm_f2ull_rn_ftz:
396 case Intrinsic::nvvm_f2ull_rp_ftz:
397 case Intrinsic::nvvm_f2ull_rz_ftz:
398 // d2ll
399 case Intrinsic::nvvm_d2ll_rm:
400 case Intrinsic::nvvm_d2ll_rn:
401 case Intrinsic::nvvm_d2ll_rp:
402 case Intrinsic::nvvm_d2ll_rz:
403 // d2ull
404 case Intrinsic::nvvm_d2ull_rm:
405 case Intrinsic::nvvm_d2ull_rn:
406 case Intrinsic::nvvm_d2ull_rp:
407 case Intrinsic::nvvm_d2ull_rz:
408 return false;
409 }
410 llvm_unreachable("Checking NaN result for invalid f2i/d2i intrinsic");
411}
412
413inline APFloat::roundingMode
414GetFPToIntegerRoundingMode(Intrinsic::ID IntrinsicID) {
415 switch (IntrinsicID) {
416 // RM:
417 case Intrinsic::nvvm_f2i_rm:
418 case Intrinsic::nvvm_f2ui_rm:
419 case Intrinsic::nvvm_f2i_rm_ftz:
420 case Intrinsic::nvvm_f2ui_rm_ftz:
421 case Intrinsic::nvvm_d2i_rm:
422 case Intrinsic::nvvm_d2ui_rm:
423
424 case Intrinsic::nvvm_f2ll_rm:
425 case Intrinsic::nvvm_f2ull_rm:
426 case Intrinsic::nvvm_f2ll_rm_ftz:
427 case Intrinsic::nvvm_f2ull_rm_ftz:
428 case Intrinsic::nvvm_d2ll_rm:
429 case Intrinsic::nvvm_d2ull_rm:
430 return APFloat::rmTowardNegative;
431
432 // RN:
433 case Intrinsic::nvvm_f2i_rn:
434 case Intrinsic::nvvm_f2ui_rn:
435 case Intrinsic::nvvm_f2i_rn_ftz:
436 case Intrinsic::nvvm_f2ui_rn_ftz:
437 case Intrinsic::nvvm_d2i_rn:
438 case Intrinsic::nvvm_d2ui_rn:
439
440 case Intrinsic::nvvm_f2ll_rn:
441 case Intrinsic::nvvm_f2ull_rn:
442 case Intrinsic::nvvm_f2ll_rn_ftz:
443 case Intrinsic::nvvm_f2ull_rn_ftz:
444 case Intrinsic::nvvm_d2ll_rn:
445 case Intrinsic::nvvm_d2ull_rn:
446 return APFloat::rmNearestTiesToEven;
447
448 // RP:
449 case Intrinsic::nvvm_f2i_rp:
450 case Intrinsic::nvvm_f2ui_rp:
451 case Intrinsic::nvvm_f2i_rp_ftz:
452 case Intrinsic::nvvm_f2ui_rp_ftz:
453 case Intrinsic::nvvm_d2i_rp:
454 case Intrinsic::nvvm_d2ui_rp:
455
456 case Intrinsic::nvvm_f2ll_rp:
457 case Intrinsic::nvvm_f2ull_rp:
458 case Intrinsic::nvvm_f2ll_rp_ftz:
459 case Intrinsic::nvvm_f2ull_rp_ftz:
460 case Intrinsic::nvvm_d2ll_rp:
461 case Intrinsic::nvvm_d2ull_rp:
462 return APFloat::rmTowardPositive;
463
464 // RZ:
465 case Intrinsic::nvvm_f2i_rz:
466 case Intrinsic::nvvm_f2ui_rz:
467 case Intrinsic::nvvm_f2i_rz_ftz:
468 case Intrinsic::nvvm_f2ui_rz_ftz:
469 case Intrinsic::nvvm_d2i_rz:
470 case Intrinsic::nvvm_d2ui_rz:
471
472 case Intrinsic::nvvm_f2ll_rz:
473 case Intrinsic::nvvm_f2ull_rz:
474 case Intrinsic::nvvm_f2ll_rz_ftz:
475 case Intrinsic::nvvm_f2ull_rz_ftz:
476 case Intrinsic::nvvm_d2ll_rz:
477 case Intrinsic::nvvm_d2ull_rz:
478 return APFloat::rmTowardZero;
479 }
480 llvm_unreachable("Checking rounding mode for invalid f2i/d2i intrinsic");
481}
482
483inline bool FMinFMaxShouldFTZ(Intrinsic::ID IntrinsicID) {
484 switch (IntrinsicID) {
485 case Intrinsic::nvvm_fmax_ftz_f:
486 case Intrinsic::nvvm_fmax_ftz_nan_f:
487 case Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f:
488 case Intrinsic::nvvm_fmax_ftz_xorsign_abs_f:
489
490 case Intrinsic::nvvm_fmin_ftz_f:
491 case Intrinsic::nvvm_fmin_ftz_nan_f:
492 case Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f:
493 case Intrinsic::nvvm_fmin_ftz_xorsign_abs_f:
494 return true;
495
496 case Intrinsic::nvvm_fmax_d:
497 case Intrinsic::nvvm_fmax_f:
498 case Intrinsic::nvvm_fmax_nan_f:
499 case Intrinsic::nvvm_fmax_nan_xorsign_abs_f:
500 case Intrinsic::nvvm_fmax_xorsign_abs_f:
501
502 case Intrinsic::nvvm_fmin_d:
503 case Intrinsic::nvvm_fmin_f:
504 case Intrinsic::nvvm_fmin_nan_f:
505 case Intrinsic::nvvm_fmin_nan_xorsign_abs_f:
506 case Intrinsic::nvvm_fmin_xorsign_abs_f:
507 return false;
508 }
509 llvm_unreachable("Checking FTZ flag for invalid fmin/fmax intrinsic");
510}
511
512inline bool FMinFMaxPropagatesNaNs(Intrinsic::ID IntrinsicID) {
513 switch (IntrinsicID) {
514 case Intrinsic::nvvm_fmax_ftz_nan_f:
515 case Intrinsic::nvvm_fmax_nan_f:
516 case Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f:
517 case Intrinsic::nvvm_fmax_nan_xorsign_abs_f:
518
519 case Intrinsic::nvvm_fmin_ftz_nan_f:
520 case Intrinsic::nvvm_fmin_nan_f:
521 case Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f:
522 case Intrinsic::nvvm_fmin_nan_xorsign_abs_f:
523 return true;
524
525 case Intrinsic::nvvm_fmax_d:
526 case Intrinsic::nvvm_fmax_f:
527 case Intrinsic::nvvm_fmax_ftz_f:
528 case Intrinsic::nvvm_fmax_ftz_xorsign_abs_f:
529 case Intrinsic::nvvm_fmax_xorsign_abs_f:
530
531 case Intrinsic::nvvm_fmin_d:
532 case Intrinsic::nvvm_fmin_f:
533 case Intrinsic::nvvm_fmin_ftz_f:
534 case Intrinsic::nvvm_fmin_ftz_xorsign_abs_f:
535 case Intrinsic::nvvm_fmin_xorsign_abs_f:
536 return false;
537 }
538 llvm_unreachable("Checking NaN flag for invalid fmin/fmax intrinsic");
539}
540
541inline bool FMinFMaxIsXorSignAbs(Intrinsic::ID IntrinsicID) {
542 switch (IntrinsicID) {
543 case Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f:
544 case Intrinsic::nvvm_fmax_ftz_xorsign_abs_f:
545 case Intrinsic::nvvm_fmax_nan_xorsign_abs_f:
546 case Intrinsic::nvvm_fmax_xorsign_abs_f:
547
548 case Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f:
549 case Intrinsic::nvvm_fmin_ftz_xorsign_abs_f:
550 case Intrinsic::nvvm_fmin_nan_xorsign_abs_f:
551 case Intrinsic::nvvm_fmin_xorsign_abs_f:
552 return true;
553
554 case Intrinsic::nvvm_fmax_d:
555 case Intrinsic::nvvm_fmax_f:
556 case Intrinsic::nvvm_fmax_ftz_f:
557 case Intrinsic::nvvm_fmax_ftz_nan_f:
558 case Intrinsic::nvvm_fmax_nan_f:
559
560 case Intrinsic::nvvm_fmin_d:
561 case Intrinsic::nvvm_fmin_f:
562 case Intrinsic::nvvm_fmin_ftz_f:
563 case Intrinsic::nvvm_fmin_ftz_nan_f:
564 case Intrinsic::nvvm_fmin_nan_f:
565 return false;
566 }
567 llvm_unreachable("Checking XorSignAbs flag for invalid fmin/fmax intrinsic");
568}
569
570inline bool UnaryMathIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID) {
571 switch (IntrinsicID) {
572 case Intrinsic::nvvm_ceil_ftz_f:
573 case Intrinsic::nvvm_fabs_ftz:
574 case Intrinsic::nvvm_floor_ftz_f:
575 case Intrinsic::nvvm_round_ftz_f:
576 case Intrinsic::nvvm_saturate_ftz_f:
577 case Intrinsic::nvvm_sqrt_rn_ftz_f:
578 return true;
579 case Intrinsic::nvvm_ceil_f:
580 case Intrinsic::nvvm_ceil_d:
581 case Intrinsic::nvvm_fabs:
582 case Intrinsic::nvvm_floor_f:
583 case Intrinsic::nvvm_floor_d:
584 case Intrinsic::nvvm_round_f:
585 case Intrinsic::nvvm_round_d:
586 case Intrinsic::nvvm_saturate_d:
587 case Intrinsic::nvvm_saturate_f:
588 case Intrinsic::nvvm_sqrt_f:
589 case Intrinsic::nvvm_sqrt_rn_d:
590 case Intrinsic::nvvm_sqrt_rn_f:
591 return false;
592 }
593 llvm_unreachable("Checking FTZ flag for invalid unary intrinsic");
594}
595
596inline bool RCPShouldFTZ(Intrinsic::ID IntrinsicID) {
597 switch (IntrinsicID) {
598 case Intrinsic::nvvm_rcp_rm_ftz_f:
599 case Intrinsic::nvvm_rcp_rn_ftz_f:
600 case Intrinsic::nvvm_rcp_rp_ftz_f:
601 case Intrinsic::nvvm_rcp_rz_ftz_f:
602 return true;
603 case Intrinsic::nvvm_rcp_rm_d:
604 case Intrinsic::nvvm_rcp_rm_f:
605 case Intrinsic::nvvm_rcp_rn_d:
606 case Intrinsic::nvvm_rcp_rn_f:
607 case Intrinsic::nvvm_rcp_rp_d:
608 case Intrinsic::nvvm_rcp_rp_f:
609 case Intrinsic::nvvm_rcp_rz_d:
610 case Intrinsic::nvvm_rcp_rz_f:
611 return false;
612 }
613 llvm_unreachable("Checking FTZ flag for invalid rcp intrinsic");
614}
615
616inline APFloat::roundingMode GetRCPRoundingMode(Intrinsic::ID IntrinsicID) {
617 switch (IntrinsicID) {
618 case Intrinsic::nvvm_rcp_rm_f:
619 case Intrinsic::nvvm_rcp_rm_d:
620 case Intrinsic::nvvm_rcp_rm_ftz_f:
621 return APFloat::rmTowardNegative;
622
623 case Intrinsic::nvvm_rcp_rn_f:
624 case Intrinsic::nvvm_rcp_rn_d:
625 case Intrinsic::nvvm_rcp_rn_ftz_f:
626 return APFloat::rmNearestTiesToEven;
627
628 case Intrinsic::nvvm_rcp_rp_f:
629 case Intrinsic::nvvm_rcp_rp_d:
630 case Intrinsic::nvvm_rcp_rp_ftz_f:
631 return APFloat::rmTowardPositive;
632
633 case Intrinsic::nvvm_rcp_rz_f:
634 case Intrinsic::nvvm_rcp_rz_d:
635 case Intrinsic::nvvm_rcp_rz_ftz_f:
636 return APFloat::rmTowardZero;
637 }
638 llvm_unreachable("Checking rounding mode for invalid rcp intrinsic");
639}
640
641inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) {
642 if (ShouldFTZ)
643 return DenormalMode::getPreserveSign();
644 return DenormalMode::getIEEE();
645}
646
647inline APFloat::roundingMode GetRoundingModeFromImmArg(const Value *ImmArgVal) {
648 return static_cast<APFloat::roundingMode>(
649 cast<ConstantInt>(ImmArgVal)->getSExtValue());
650}
651
652inline StringRef GetRoundingModeName(APFloat::roundingMode RM) {
653 switch (RM) {
654 case APFloat::rmNearestTiesToEven:
655 return "rn";
656 case APFloat::rmTowardZero:
657 return "rz";
658 case APFloat::rmTowardNegative:
659 return "rm";
660 case APFloat::rmTowardPositive:
661 return "rp";
662 default:
663 return "";
664 }
665}
666
667inline bool FMulShouldFTZ(Intrinsic::ID IntrinsicID) {
668 switch (IntrinsicID) {
669 case Intrinsic::nvvm_mul_rm_ftz_f:
670 case Intrinsic::nvvm_mul_rn_ftz_f:
671 case Intrinsic::nvvm_mul_rp_ftz_f:
672 case Intrinsic::nvvm_mul_rz_ftz_f:
673 return true;
674
675 case Intrinsic::nvvm_mul_rm_f:
676 case Intrinsic::nvvm_mul_rn_f:
677 case Intrinsic::nvvm_mul_rp_f:
678 case Intrinsic::nvvm_mul_rz_f:
679 case Intrinsic::nvvm_mul_rm_d:
680 case Intrinsic::nvvm_mul_rn_d:
681 case Intrinsic::nvvm_mul_rp_d:
682 case Intrinsic::nvvm_mul_rz_d:
683 return false;
684 }
685 llvm_unreachable("Checking FTZ flag for invalid NVVM mul intrinsic");
686}
687
688inline APFloat::roundingMode GetFMulRoundingMode(Intrinsic::ID IntrinsicID) {
689 switch (IntrinsicID) {
690 case Intrinsic::nvvm_mul_rm_f:
691 case Intrinsic::nvvm_mul_rm_d:
692 case Intrinsic::nvvm_mul_rm_ftz_f:
693 return APFloat::rmTowardNegative;
694 case Intrinsic::nvvm_mul_rn_f:
695 case Intrinsic::nvvm_mul_rn_d:
696 case Intrinsic::nvvm_mul_rn_ftz_f:
697 return APFloat::rmNearestTiesToEven;
698 case Intrinsic::nvvm_mul_rp_f:
699 case Intrinsic::nvvm_mul_rp_d:
700 case Intrinsic::nvvm_mul_rp_ftz_f:
701 return APFloat::rmTowardPositive;
702 case Intrinsic::nvvm_mul_rz_f:
703 case Intrinsic::nvvm_mul_rz_d:
704 case Intrinsic::nvvm_mul_rz_ftz_f:
705 return APFloat::rmTowardZero;
706 }
707 llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM mul");
708}
709
710inline bool FDivShouldFTZ(Intrinsic::ID IntrinsicID) {
711 switch (IntrinsicID) {
712 case Intrinsic::nvvm_div_rm_ftz_f:
713 case Intrinsic::nvvm_div_rn_ftz_f:
714 case Intrinsic::nvvm_div_rp_ftz_f:
715 case Intrinsic::nvvm_div_rz_ftz_f:
716 return true;
717
718 case Intrinsic::nvvm_div_rm_f:
719 case Intrinsic::nvvm_div_rn_f:
720 case Intrinsic::nvvm_div_rp_f:
721 case Intrinsic::nvvm_div_rz_f:
722 case Intrinsic::nvvm_div_rm_d:
723 case Intrinsic::nvvm_div_rn_d:
724 case Intrinsic::nvvm_div_rp_d:
725 case Intrinsic::nvvm_div_rz_d:
726 return false;
727 }
728 llvm_unreachable("Checking FTZ flag for invalid NVVM div intrinsic");
729}
730
731inline APFloat::roundingMode GetFDivRoundingMode(Intrinsic::ID IntrinsicID) {
732 switch (IntrinsicID) {
733 case Intrinsic::nvvm_div_rm_f:
734 case Intrinsic::nvvm_div_rm_d:
735 case Intrinsic::nvvm_div_rm_ftz_f:
736 return APFloat::rmTowardNegative;
737 case Intrinsic::nvvm_div_rn_f:
738 case Intrinsic::nvvm_div_rn_d:
739 case Intrinsic::nvvm_div_rn_ftz_f:
740 return APFloat::rmNearestTiesToEven;
741 case Intrinsic::nvvm_div_rp_f:
742 case Intrinsic::nvvm_div_rp_d:
743 case Intrinsic::nvvm_div_rp_ftz_f:
744 return APFloat::rmTowardPositive;
745 case Intrinsic::nvvm_div_rz_f:
746 case Intrinsic::nvvm_div_rz_d:
747 case Intrinsic::nvvm_div_rz_ftz_f:
748 return APFloat::rmTowardZero;
749 }
750 llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM div");
751}
752
753inline bool FMAShouldFTZ(Intrinsic::ID IntrinsicID) {
754 switch (IntrinsicID) {
755 case Intrinsic::nvvm_fma_rm_ftz_f:
756 case Intrinsic::nvvm_fma_rn_ftz_f:
757 case Intrinsic::nvvm_fma_rp_ftz_f:
758 case Intrinsic::nvvm_fma_rz_ftz_f:
759 return true;
760
761 case Intrinsic::nvvm_fma_rm_f:
762 case Intrinsic::nvvm_fma_rn_f:
763 case Intrinsic::nvvm_fma_rp_f:
764 case Intrinsic::nvvm_fma_rz_f:
765 case Intrinsic::nvvm_fma_rm_d:
766 case Intrinsic::nvvm_fma_rn_d:
767 case Intrinsic::nvvm_fma_rp_d:
768 case Intrinsic::nvvm_fma_rz_d:
769 return false;
770 }
771 llvm_unreachable("Checking FTZ flag for invalid NVVM fma intrinsic");
772}
773
774inline APFloat::roundingMode GetFMARoundingMode(Intrinsic::ID IntrinsicID) {
775 switch (IntrinsicID) {
776 case Intrinsic::nvvm_fma_rm_f:
777 case Intrinsic::nvvm_fma_rm_d:
778 case Intrinsic::nvvm_fma_rm_ftz_f:
779 return APFloat::rmTowardNegative;
780 case Intrinsic::nvvm_fma_rn_f:
781 case Intrinsic::nvvm_fma_rn_d:
782 case Intrinsic::nvvm_fma_rn_ftz_f:
783 return APFloat::rmNearestTiesToEven;
784 case Intrinsic::nvvm_fma_rp_f:
785 case Intrinsic::nvvm_fma_rp_d:
786 case Intrinsic::nvvm_fma_rp_ftz_f:
787 return APFloat::rmTowardPositive;
788 case Intrinsic::nvvm_fma_rz_f:
789 case Intrinsic::nvvm_fma_rz_d:
790 case Intrinsic::nvvm_fma_rz_ftz_f:
791 return APFloat::rmTowardZero;
792 }
793 llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM fma");
794}
795
796} // namespace nvvm
797} // namespace llvm
798#endif // LLVM_IR_NVVMINTRINSICUTILS_H
APFloat.h
This file declares a class to represent arbitrary precision floating point values and provide a varie...
APInt.h
This file implements a class to represent arbitrary precision integral constant values and operations...
LLVM_ABI
#define LLVM_ABI
Definition Compiler.h:215
Constants.h
This file contains the declarations for the subclasses of Constant, which represent the different fla...
llvm::APFloatBase::rmTowardZero
static constexpr roundingMode rmTowardZero
Definition APFloat.h:365
llvm::APFloatBase::roundingMode
llvm::RoundingMode roundingMode
IEEE-754R 4.3: Rounding-direction attributes.
Definition APFloat.h:359
llvm::APFloatBase::rmTowardNegative
static constexpr roundingMode rmTowardNegative
Definition APFloat.h:364
llvm::APFloatBase::rmNearestTiesToEven
static constexpr roundingMode rmNearestTiesToEven
Definition APFloat.h:361
llvm::APFloatBase::rmTowardPositive
static constexpr roundingMode rmTowardPositive
Definition APFloat.h:363
llvm::Constant
This is an important base class in LLVM.
Definition Constant.h:43
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::Value
LLVM Value Representation.
Definition Value.h:75
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
uint8_t
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::nvvm::GetRoundingModeFromImmArg
APFloat::roundingMode GetRoundingModeFromImmArg(const Value *ImmArgVal)
Definition NVVMIntrinsicUtils.h:647
llvm::nvvm::GetFMARoundingMode
APFloat::roundingMode GetFMARoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:774
llvm::nvvm::printTcgen05MMACollectorBBuffer
LLVM_ABI void printTcgen05MMACollectorBBuffer(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:96
llvm::nvvm::GetRoundingModeName
StringRef GetRoundingModeName(APFloat::roundingMode RM)
Definition NVVMIntrinsicUtils.h:652
llvm::nvvm::printTensormapSwizzleMode
LLVM_ABI void printTensormapSwizzleMode(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:149
llvm::nvvm::GetNVVMDenormMode
DenormalMode GetNVVMDenormMode(bool ShouldFTZ)
Definition NVVMIntrinsicUtils.h:641
llvm::nvvm::getEvictPolicyName
StringRef getEvictPolicyName(EvictPolicyType Policy)
Definition NVVMIntrinsicUtils.h:112
llvm::nvvm::printFPRoundingMode
LLVM_ABI void printFPRoundingMode(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:177
llvm::nvvm::printTMAReductionOp
LLVM_ABI void printTMAReductionOp(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:29
llvm::nvvm::printTMAValidateDataPattern
LLVM_ABI void printTMAValidateDataPattern(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:39
llvm::nvvm::FPToIntegerIntrinsicNaNZero
bool FPToIntegerIntrinsicNaNZero(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:349
llvm::nvvm::printTensormapInterleaveLayout
LLVM_ABI void printTensormapInterleaveLayout(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:131
llvm::nvvm::GetFDivRoundingMode
APFloat::roundingMode GetFDivRoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:731
llvm::nvvm::printTensormapSwizzleAtomicity
LLVM_ABI void printTensormapSwizzleAtomicity(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:163
llvm::nvvm::FPToIntegerIntrinsicResultIsSigned
bool FPToIntegerIntrinsicResultIsSigned(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:283
llvm::nvvm::GetFPToIntegerRoundingMode
APFloat::roundingMode GetFPToIntegerRoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:414
llvm::nvvm::RCPShouldFTZ
bool RCPShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:596
llvm::nvvm::getTMATensorReductionOpName
StringRef getTMATensorReductionOpName(TMAReductionOp Op)
Definition NVVMIntrinsicUtils.h:45
llvm::nvvm::FPToIntegerIntrinsicShouldFTZ
bool FPToIntegerIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:216
llvm::nvvm::getTMAValidateDataPatternName
StringRef getTMAValidateDataPatternName(TMAValidateDataPattern Pattern)
Definition NVVMIntrinsicUtils.h:87
llvm::nvvm::FDivShouldFTZ
bool FDivShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:710
llvm::nvvm::FMinFMaxIsXorSignAbs
bool FMinFMaxIsXorSignAbs(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:541
llvm::nvvm::GetFMulRoundingMode
APFloat::roundingMode GetFMulRoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:688
llvm::nvvm::printTcgen05MMAKind
LLVM_ABI void printTcgen05MMAKind(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:52
llvm::nvvm::printTcgen05CollectorUsageOp
LLVM_ABI void printTcgen05CollectorUsageOp(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:75
llvm::nvvm::UnaryMathIntrinsicShouldFTZ
bool UnaryMathIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:570
llvm::nvvm::FMinFMaxShouldFTZ
bool FMinFMaxShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:483
llvm::nvvm::printEvictPolicyType
LLVM_ABI void printEvictPolicyType(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:19
llvm::nvvm::FMAShouldFTZ
bool FMAShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:753
llvm::nvvm::FMulShouldFTZ
bool FMulShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:667
llvm::nvvm::printTensormapFillMode
LLVM_ABI void printTensormapFillMode(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:182
llvm::nvvm::GetRCPRoundingMode
APFloat::roundingMode GetRCPRoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:616
llvm::nvvm::FMinFMaxPropagatesNaNs
bool FMinFMaxPropagatesNaNs(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:512
llvm::nvvm::printTensormapElemType
LLVM_ABI void printTensormapElemType(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:117
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition FunctionInfo.h:25
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:24
llvm::cast
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
raw_ostream.h
llvm::DenormalMode
Represent subnormal handling kind for floating point instruction inputs and outputs.
Definition FloatingPointMode.h:71
llvm::DenormalMode::getPreserveSign
static constexpr DenormalMode getPreserveSign()
Definition FloatingPointMode.h:119
llvm::DenormalMode::getIEEE
static constexpr DenormalMode getIEEE()
Definition FloatingPointMode.h:115
Generated on for LLVM by doxygen 1.14.0