|
| 1 | +//===----------------------------------------------------------------------===// |
| 2 | +// |
| 3 | +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | +// See https://llvm.org/LICENSE.txt for license information. |
| 5 | +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | +// |
| 7 | +//===----------------------------------------------------------------------===// |
| 8 | + |
| 9 | +#ifndef __OMPX_H |
| 10 | +#define __OMPX_H |
| 11 | + |
| 12 | +#ifdef __AMDGCN_WAVEFRONT_SIZE |
| 13 | +#define __WARP_SIZE __AMDGCN_WAVEFRONT_SIZE |
| 14 | +#else |
| 15 | +#define __WARP_SIZE 32 |
| 16 | +#endif |
| 17 | + |
| 18 | +typedef unsigned long uint64_t; |
| 19 | + |
| 20 | +#ifdef __cplusplus |
| 21 | +extern "C" { |
| 22 | +#endif |
| 23 | + |
| 24 | +int omp_get_ancestor_thread_num(int); |
| 25 | +int omp_get_team_size(int); |
| 26 | + |
| 27 | +#ifdef __cplusplus |
| 28 | +} |
| 29 | +#endif |
| 30 | + |
| 31 | +/// Target kernel language extensions |
| 32 | +/// |
| 33 | +/// These extensions exist for the host to allow fallback implementations, |
| 34 | +/// however, they cannot be arbitrarily composed with OpenMP. If the rules of |
| 35 | +/// the kernel language are followed, the host fallbacks should behave as |
| 36 | +/// expected since the kernel is represented as 3 sequential outer loops, one |
| 37 | +/// for each grid dimension, and three (nested) parallel loops, one for each |
| 38 | +/// block dimension. This fallback is not supposed to be optimal and should be |
| 39 | +/// configurable by the user. |
| 40 | +/// |
| 41 | +///{ |
| 42 | + |
| 43 | +#ifdef __cplusplus |
| 44 | +extern "C" { |
| 45 | +#endif |
| 46 | + |
| 47 | +enum { |
| 48 | + ompx_relaxed = __ATOMIC_RELAXED, |
| 49 | + ompx_aquire = __ATOMIC_ACQUIRE, |
| 50 | + ompx_release = __ATOMIC_RELEASE, |
| 51 | + ompx_acq_rel = __ATOMIC_ACQ_REL, |
| 52 | + ompx_seq_cst = __ATOMIC_SEQ_CST, |
| 53 | +}; |
| 54 | + |
| 55 | +enum { |
| 56 | + ompx_dim_x = 0, |
| 57 | + ompx_dim_y = 1, |
| 58 | + ompx_dim_z = 2, |
| 59 | +}; |
| 60 | + |
| 61 | +// TODO: The following implementation is for host fallback. We need to disable |
| 62 | +// generation of host fallback in kernel language mode. |
| 63 | +#pragma omp begin declare variant match(device = {kind(cpu)}) |
| 64 | + |
| 65 | +/// ompx_{thread,block}_{id,dim} |
| 66 | +///{ |
| 67 | +#define _TGT_KERNEL_LANGUAGE_HOST_IMPL_GRID_C(NAME, VALUE) \ |
| 68 | + static inline int ompx_##NAME(int Dim) { return VALUE; } |
| 69 | + |
| 70 | +_TGT_KERNEL_LANGUAGE_HOST_IMPL_GRID_C(thread_id, |
| 71 | + omp_get_ancestor_thread_num(Dim + 1)) |
| 72 | +_TGT_KERNEL_LANGUAGE_HOST_IMPL_GRID_C(block_dim, omp_get_team_size(Dim + 1)) |
| 73 | +_TGT_KERNEL_LANGUAGE_HOST_IMPL_GRID_C(block_id, 0) |
| 74 | +_TGT_KERNEL_LANGUAGE_HOST_IMPL_GRID_C(grid_dim, 1) |
| 75 | +#undef _TGT_KERNEL_LANGUAGE_HOST_IMPL_GRID_C |
| 76 | +///} |
| 77 | + |
| 78 | +/// ompx_{sync_block}_{,divergent} |
| 79 | +///{ |
| 80 | +#define _TGT_KERNEL_LANGUAGE_HOST_IMPL_SYNC_C(RETTY, NAME, ARGS, BODY) \ |
| 81 | + static inline RETTY ompx_##NAME(ARGS) { BODY; } |
| 82 | + |
| 83 | +_TGT_KERNEL_LANGUAGE_HOST_IMPL_SYNC_C(void, sync_block, int Ordering, |
| 84 | + _Pragma("omp barrier")) |
| 85 | +_TGT_KERNEL_LANGUAGE_HOST_IMPL_SYNC_C(void, sync_block_acq_rel, void, |
| 86 | + ompx_sync_block(ompx_acq_rel)) |
| 87 | +_TGT_KERNEL_LANGUAGE_HOST_IMPL_SYNC_C(void, sync_block_divergent, int Ordering, |
| 88 | + ompx_sync_block(Ordering)) |
| 89 | +#undef _TGT_KERNEL_LANGUAGE_HOST_IMPL_SYNC_C |
| 90 | +///} |
| 91 | + |
| 92 | +static inline uint64_t ompx_ballot_sync(uint64_t mask, int pred) { |
| 93 | + __builtin_trap(); |
| 94 | +} |
| 95 | + |
| 96 | +/// ompx_shfl_down_sync_{i,f,l,d} |
| 97 | +///{ |
| 98 | +#define _TGT_KERNEL_LANGUAGE_SHFL_DOWN_SYNC_HOST_IMPL(TYPE, TY) \ |
| 99 | + static inline TYPE ompx_shfl_down_sync_##TY(uint64_t mask, TYPE var, \ |
| 100 | + unsigned delta, int width) { \ |
| 101 | + __builtin_trap(); \ |
| 102 | + } |
| 103 | + |
| 104 | +_TGT_KERNEL_LANGUAGE_SHFL_DOWN_SYNC_HOST_IMPL(int, i) |
| 105 | +_TGT_KERNEL_LANGUAGE_SHFL_DOWN_SYNC_HOST_IMPL(float, f) |
| 106 | +_TGT_KERNEL_LANGUAGE_SHFL_DOWN_SYNC_HOST_IMPL(long, l) |
| 107 | +_TGT_KERNEL_LANGUAGE_SHFL_DOWN_SYNC_HOST_IMPL(double, d) |
| 108 | + |
| 109 | +#undef _TGT_KERNEL_LANGUAGE_SHFL_DOWN_SYNC_HOST_IMPL |
| 110 | +///} |
| 111 | + |
| 112 | +#pragma omp end declare variant |
| 113 | + |
| 114 | +/// ompx_{sync_block}_{,divergent} |
| 115 | +///{ |
| 116 | +#define _TGT_KERNEL_LANGUAGE_DECL_SYNC_C(RETTY, NAME, ARGS) \ |
| 117 | + RETTY ompx_##NAME(ARGS); |
| 118 | + |
| 119 | +_TGT_KERNEL_LANGUAGE_DECL_SYNC_C(void, sync_block, int Ordering) |
| 120 | +_TGT_KERNEL_LANGUAGE_DECL_SYNC_C(void, sync_block_acq_rel, void) |
| 121 | +_TGT_KERNEL_LANGUAGE_DECL_SYNC_C(void, sync_block_divergent, int Ordering) |
| 122 | +#undef _TGT_KERNEL_LANGUAGE_DECL_SYNC_C |
| 123 | +///} |
| 124 | + |
| 125 | +/// ompx_{thread,block}_{id,dim}_{x,y,z} |
| 126 | +///{ |
| 127 | +#define _TGT_KERNEL_LANGUAGE_DECL_GRID_C(NAME) \ |
| 128 | + int ompx_##NAME(int Dim); \ |
| 129 | + static inline int ompx_##NAME##_x() { return ompx_##NAME(ompx_dim_x); } \ |
| 130 | + static inline int ompx_##NAME##_y() { return ompx_##NAME(ompx_dim_y); } \ |
| 131 | + static inline int ompx_##NAME##_z() { return ompx_##NAME(ompx_dim_z); } |
| 132 | + |
| 133 | +_TGT_KERNEL_LANGUAGE_DECL_GRID_C(thread_id) |
| 134 | +_TGT_KERNEL_LANGUAGE_DECL_GRID_C(block_dim) |
| 135 | +_TGT_KERNEL_LANGUAGE_DECL_GRID_C(block_id) |
| 136 | +_TGT_KERNEL_LANGUAGE_DECL_GRID_C(grid_dim) |
| 137 | +#undef _TGT_KERNEL_LANGUAGE_DECL_GRID_C |
| 138 | +///} |
| 139 | + |
| 140 | +uint64_t ompx_ballot_sync(uint64_t mask, int pred); |
| 141 | + |
| 142 | +/// ompx_shfl_down_sync_{i,f,l,d} |
| 143 | +///{ |
| 144 | +#define _TGT_KERNEL_LANGUAGE_SHFL_DOWN_SYNC(TYPE, TY) \ |
| 145 | + TYPE ompx_shfl_down_sync_##TY(uint64_t mask, TYPE var, unsigned delta, \ |
| 146 | + int width); |
| 147 | + |
| 148 | +_TGT_KERNEL_LANGUAGE_SHFL_DOWN_SYNC(int, i) |
| 149 | +_TGT_KERNEL_LANGUAGE_SHFL_DOWN_SYNC(float, f) |
| 150 | +_TGT_KERNEL_LANGUAGE_SHFL_DOWN_SYNC(long, l) |
| 151 | +_TGT_KERNEL_LANGUAGE_SHFL_DOWN_SYNC(double, d) |
| 152 | + |
| 153 | +#undef _TGT_KERNEL_LANGUAGE_SHFL_DOWN_SYNC |
| 154 | +///} |
| 155 | + |
| 156 | +#ifdef __cplusplus |
| 157 | +} |
| 158 | +#endif |
| 159 | + |
| 160 | +#ifdef __cplusplus |
| 161 | + |
| 162 | +namespace ompx { |
| 163 | + |
| 164 | +enum { |
| 165 | + dim_x = ompx_dim_x, |
| 166 | + dim_y = ompx_dim_y, |
| 167 | + dim_z = ompx_dim_z, |
| 168 | +}; |
| 169 | + |
| 170 | +enum { |
| 171 | + relaxed = ompx_relaxed , |
| 172 | + aquire = ompx_aquire, |
| 173 | + release = ompx_release, |
| 174 | + acc_rel = ompx_acq_rel, |
| 175 | + seq_cst = ompx_seq_cst, |
| 176 | +}; |
| 177 | + |
| 178 | +/// ompx::{thread,block}_{id,dim}_{,x,y,z} |
| 179 | +///{ |
| 180 | +#define _TGT_KERNEL_LANGUAGE_HOST_IMPL_GRID_CXX(NAME) \ |
| 181 | + static inline int NAME(int Dim) noexcept { return ompx_##NAME(Dim); } \ |
| 182 | + static inline int NAME##_x() noexcept { return NAME(ompx_dim_x); } \ |
| 183 | + static inline int NAME##_y() noexcept { return NAME(ompx_dim_y); } \ |
| 184 | + static inline int NAME##_z() noexcept { return NAME(ompx_dim_z); } |
| 185 | + |
| 186 | +_TGT_KERNEL_LANGUAGE_HOST_IMPL_GRID_CXX(thread_id) |
| 187 | +_TGT_KERNEL_LANGUAGE_HOST_IMPL_GRID_CXX(block_dim) |
| 188 | +_TGT_KERNEL_LANGUAGE_HOST_IMPL_GRID_CXX(block_id) |
| 189 | +_TGT_KERNEL_LANGUAGE_HOST_IMPL_GRID_CXX(grid_dim) |
| 190 | +#undef _TGT_KERNEL_LANGUAGE_HOST_IMPL_GRID_CXX |
| 191 | +///} |
| 192 | + |
| 193 | +/// ompx_{sync_block}_{,divergent} |
| 194 | +///{ |
| 195 | +#define _TGT_KERNEL_LANGUAGE_HOST_IMPL_SYNC_CXX(RETTY, NAME, ARGS, CALL_ARGS) \ |
| 196 | + static inline RETTY NAME(ARGS) { \ |
| 197 | + return ompx_##NAME(CALL_ARGS); \ |
| 198 | + } |
| 199 | + |
| 200 | +_TGT_KERNEL_LANGUAGE_HOST_IMPL_SYNC_CXX(void, sync_block, int Ordering = acc_rel, |
| 201 | + Ordering) |
| 202 | +_TGT_KERNEL_LANGUAGE_HOST_IMPL_SYNC_CXX(void, sync_block_divergent, |
| 203 | + int Ordering = acc_rel, Ordering) |
| 204 | +#undef _TGT_KERNEL_LANGUAGE_HOST_IMPL_SYNC_CXX |
| 205 | +///} |
| 206 | + |
| 207 | +static inline uint64_t ballot_sync(uint64_t mask, int pred) { |
| 208 | + return ompx_ballot_sync(mask, pred); |
| 209 | +} |
| 210 | + |
| 211 | +/// shfl_down_sync |
| 212 | +///{ |
| 213 | +#define _TGT_KERNEL_LANGUAGE_SHFL_DOWN_SYNC(TYPE, TY) \ |
| 214 | + static inline TYPE shfl_down_sync(uint64_t mask, TYPE var, unsigned delta, \ |
| 215 | + int width = __WARP_SIZE) { \ |
| 216 | + return ompx_shfl_down_sync_##TY(mask, var, delta, width); \ |
| 217 | + } |
| 218 | + |
| 219 | +_TGT_KERNEL_LANGUAGE_SHFL_DOWN_SYNC(int, i) |
| 220 | +_TGT_KERNEL_LANGUAGE_SHFL_DOWN_SYNC(float, f) |
| 221 | +_TGT_KERNEL_LANGUAGE_SHFL_DOWN_SYNC(long, l) |
| 222 | +_TGT_KERNEL_LANGUAGE_SHFL_DOWN_SYNC(double, d) |
| 223 | + |
| 224 | +#undef _TGT_KERNEL_LANGUAGE_SHFL_DOWN_SYNC |
| 225 | +///} |
| 226 | + |
| 227 | +} // namespace ompx |
| 228 | +#endif |
| 229 | + |
| 230 | +///} |
| 231 | + |
| 232 | +#endif /* __OMPX_H */ |
![r-universe[bot]](https://avatars.githubusercontent.com/in/87942?v=4&size=40){kind=avatar}
0 commit comments