Kokkos Core Kernels Package
Version of the Day
core
src
Kokkos_Atomic.hpp
Go to the documentation of this file.
1
/*
2
//@HEADER
3
// ************************************************************************
4
//
5
// Kokkos v. 2.0
6
// Copyright (2014) Sandia Corporation
7
//
8
// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9
// the U.S. Government retains certain rights in this software.
10
//
11
// Redistribution and use in source and binary forms, with or without
12
// modification, are permitted provided that the following conditions are
13
// met:
14
//
15
// 1. Redistributions of source code must retain the above copyright
16
// notice, this list of conditions and the following disclaimer.
17
//
18
// 2. Redistributions in binary form must reproduce the above copyright
19
// notice, this list of conditions and the following disclaimer in the
20
// documentation and/or other materials provided with the distribution.
21
//
22
// 3. Neither the name of the Corporation nor the names of the
23
// contributors may be used to endorse or promote products derived from
24
// this software without specific prior written permission.
25
//
26
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37
//
38
// Questions? Contact Christian R. Trott (crtrott@sandia.gov)
39
//
40
// ************************************************************************
41
//@HEADER
42
*/
43
66
67
#ifndef KOKKOS_ATOMIC_HPP
68
#define KOKKOS_ATOMIC_HPP
69
70
#include <Kokkos_Macros.hpp>
71
#include <Kokkos_HostSpace.hpp>
72
#include <impl/Kokkos_Traits.hpp>
73
74
//----------------------------------------------------------------------------
75
#if defined(_WIN32)
76
#define KOKKOS_ENABLE_WINDOWS_ATOMICS
77
#else
78
#if defined( KOKKOS_ENABLE_CUDA )
79
80
// Compiling NVIDIA device code, must use Cuda atomics:
81
82
#define KOKKOS_ENABLE_CUDA_ATOMICS
83
84
#elif defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_ROCM_GPU)
85
86
#define KOKKOS_ENABLE_ROCM_ATOMICS
87
88
#endif
89
90
#if ! defined( KOKKOS_ENABLE_GNU_ATOMICS ) && \
91
! defined( KOKKOS_ENABLE_INTEL_ATOMICS ) && \
92
! defined( KOKKOS_ENABLE_OPENMP_ATOMICS ) && \
93
! defined( KOKKOS_ENABLE_SERIAL_ATOMICS )
94
95
// Compiling for non-Cuda atomic implementation has not been pre-selected.
96
// Choose the best implementation for the detected compiler.
97
// Preference: GCC, INTEL, OMP31
98
99
#if defined( KOKKOS_INTERNAL_NOT_PARALLEL )
100
101
#define KOKKOS_ENABLE_SERIAL_ATOMICS
102
103
#elif defined( KOKKOS_COMPILER_GNU ) || \
104
defined( KOKKOS_COMPILER_CLANG ) || \
105
( defined ( KOKKOS_COMPILER_NVCC ) )
106
107
#define KOKKOS_ENABLE_GNU_ATOMICS
108
109
#elif defined( KOKKOS_COMPILER_INTEL ) || \
110
defined( KOKKOS_COMPILER_CRAYC )
111
112
#define KOKKOS_ENABLE_INTEL_ATOMICS
113
114
#elif defined( _OPENMP ) && ( 201107 <= _OPENMP )
115
116
#define KOKKOS_ENABLE_OPENMP_ATOMICS
117
118
#else
119
120
#error "KOKKOS_ATOMICS_USE : Unsupported compiler"
121
122
#endif
123
124
#endif
/* Not pre-selected atomic implementation */
125
#endif
126
127
#ifdef KOKKOS_ENABLE_CUDA
128
#include <Cuda/Kokkos_Cuda_Locks.hpp>
129
#endif
130
131
namespace
Kokkos
{
132
template
<
typename
T>
133
KOKKOS_INLINE_FUNCTION
134
void
atomic_add(
volatile
T *
const
dest,
const
T src);
135
136
// Atomic increment
137
template
<
typename
T>
138
KOKKOS_INLINE_FUNCTION
139
void
atomic_increment(
volatile
T* a);
140
141
template
<
typename
T>
142
KOKKOS_INLINE_FUNCTION
143
void
atomic_decrement(
volatile
T* a);
144
}
145
146
namespace
Kokkos
{
147
148
149
inline
150
const
char
* atomic_query_version()
151
{
152
#if defined( KOKKOS_ENABLE_CUDA_ATOMICS )
153
return
"KOKKOS_ENABLE_CUDA_ATOMICS"
;
154
#elif defined( KOKKOS_ENABLE_GNU_ATOMICS )
155
return
"KOKKOS_ENABLE_GNU_ATOMICS"
;
156
#elif defined( KOKKOS_ENABLE_INTEL_ATOMICS )
157
return
"KOKKOS_ENABLE_INTEL_ATOMICS"
;
158
#elif defined( KOKKOS_ENABLE_OPENMP_ATOMICS )
159
return
"KOKKOS_ENABLE_OPENMP_ATOMICS"
;
160
#elif defined( KOKKOS_ENABLE_WINDOWS_ATOMICS )
161
return
"KOKKOS_ENABLE_WINDOWS_ATOMICS"
;
162
#elif defined( KOKKOS_ENABLE_SERIAL_ATOMICS )
163
return
"KOKKOS_ENABLE_SERIAL_ATOMICS"
;
164
#else
165
#error "No valid response for atomic_query_version!"
166
#endif
167
}
168
169
}
// namespace Kokkos
170
171
#if defined( KOKKOS_ENABLE_ROCM )
172
namespace
Kokkos
{
173
namespace
Impl {
174
extern
KOKKOS_INLINE_FUNCTION
175
bool
lock_address_rocm_space(
void
* ptr);
176
177
extern
KOKKOS_INLINE_FUNCTION
178
void
unlock_address_rocm_space(
void
* ptr);
179
}
180
}
181
#include <ROCm/Kokkos_ROCm_Atomic.hpp>
182
#endif
183
184
#ifdef _WIN32
185
#include "impl/Kokkos_Atomic_Windows.hpp"
186
#else
187
188
//----------------------------------------------------------------------------
189
// Atomic Assembly
190
//
191
// Implements CAS128-bit in assembly
192
193
#include "impl/Kokkos_Atomic_Assembly.hpp"
194
195
//----------------------------------------------------------------------------
196
// Atomic exchange
197
//
198
// template< typename T >
199
// T atomic_exchange( volatile T* const dest , const T val )
200
// { T tmp = *dest ; *dest = val ; return tmp ; }
201
202
#include "impl/Kokkos_Atomic_Exchange.hpp"
203
204
//----------------------------------------------------------------------------
205
// Atomic compare-and-exchange
206
//
207
// template<class T>
208
// bool atomic_compare_exchange_strong(volatile T* const dest, const T compare, const T val)
209
// { bool equal = compare == *dest ; if ( equal ) { *dest = val ; } return equal ; }
210
211
#include "impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp"
212
213
//----------------------------------------------------------------------------
214
// Atomic fetch and add
215
//
216
// template<class T>
217
// T atomic_fetch_add(volatile T* const dest, const T val)
218
// { T tmp = *dest ; *dest += val ; return tmp ; }
219
220
#include "impl/Kokkos_Atomic_Fetch_Add.hpp"
221
222
//----------------------------------------------------------------------------
223
// Atomic increment
224
//
225
// template<class T>
226
// T atomic_increment(volatile T* const dest)
227
// { dest++; }
228
229
#include "impl/Kokkos_Atomic_Increment.hpp"
230
231
//----------------------------------------------------------------------------
232
// Atomic Decrement
233
//
234
// template<class T>
235
// T atomic_decrement(volatile T* const dest)
236
// { dest--; }
237
238
#include "impl/Kokkos_Atomic_Decrement.hpp"
239
240
//----------------------------------------------------------------------------
241
// Atomic fetch and sub
242
//
243
// template<class T>
244
// T atomic_fetch_sub(volatile T* const dest, const T val)
245
// { T tmp = *dest ; *dest -= val ; return tmp ; }
246
247
#include "impl/Kokkos_Atomic_Fetch_Sub.hpp"
248
249
//----------------------------------------------------------------------------
250
// Atomic fetch and or
251
//
252
// template<class T>
253
// T atomic_fetch_or(volatile T* const dest, const T val)
254
// { T tmp = *dest ; *dest = tmp | val ; return tmp ; }
255
256
#include "impl/Kokkos_Atomic_Fetch_Or.hpp"
257
258
//----------------------------------------------------------------------------
259
// Atomic fetch and and
260
//
261
// template<class T>
262
// T atomic_fetch_and(volatile T* const dest, const T val)
263
// { T tmp = *dest ; *dest = tmp & val ; return tmp ; }
264
265
#include "impl/Kokkos_Atomic_Fetch_And.hpp"
266
#endif
/*Not _WIN32*/
267
268
//----------------------------------------------------------------------------
269
// Memory fence
270
//
271
// All loads and stores from this thread will be globally consistent before continuing
272
//
273
// void memory_fence() {...};
274
#include "impl/Kokkos_Memory_Fence.hpp"
275
276
//----------------------------------------------------------------------------
277
// Provide volatile_load and safe_load
278
//
279
// T volatile_load(T const volatile * const ptr);
280
//
281
// T const& safe_load(T const * const ptr);
282
// XEON PHI
283
// T safe_load(T const * const ptr
284
285
#include "impl/Kokkos_Volatile_Load.hpp"
286
287
#ifndef _WIN32
288
#include "impl/Kokkos_Atomic_Generic.hpp"
289
#endif
290
//----------------------------------------------------------------------------
291
// This atomic-style macro should be an inlined function, not a macro
292
293
#if defined( KOKKOS_COMPILER_GNU ) && !defined(__PGIC__) && !defined(__CUDA_ARCH__)
294
295
#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) __builtin_prefetch(addr,0,0)
296
#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) __builtin_prefetch(addr,1,0)
297
298
#else
299
300
#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) ((void)0)
301
#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) ((void)0)
302
303
#endif
304
305
//----------------------------------------------------------------------------
306
307
#endif
/* KOKKOS_ATOMIC_HPP */
308
Kokkos
Definition:
Kokkos_AnonymousSpace.hpp:53
Generated by
1.8.14