|
Template Numerical Library version\ main:c4e3186
|
Internal namespace for CUDA/HIP backend support. More...
Classes | |
| struct | LaunchConfiguration |
| Holds the parameters necessary to launch a CUDA or HIP kernel (i.e. schedule it for execution on some stream of some device). More... | |
| struct | SharedMemory |
| struct | SharedMemory< T, 16 > |
| struct | SharedMemory< T, 32 > |
| struct | SharedMemory< T, 64 > |
| struct | SharedMemory< T, 8 > |
| class | Stream |
| class | StreamPool |
| struct | Uninitialized |
| Storage wrapper for type-punning through an array of device words. More... | |
Typedefs | |
| using | error_t = cudaError_t |
| using | stream_t = cudaStream_t |
Enumerations | |
| enum | { StreamDefault = cudaStreamDefault , StreamNonBlocking = cudaStreamNonBlocking } |
| enum | FuncCache { FuncCachePreferNone = cudaFuncCachePreferNone , FuncCachePreferShared = cudaFuncCachePreferShared , FuncCachePreferL1 = cudaFuncCachePreferL1 , FuncCachePreferEqual = cudaFuncCachePreferEqual } |
| enum | MemcpyKind { MemcpyHostToHost = cudaMemcpyHostToHost , MemcpyHostToDevice = cudaMemcpyHostToDevice , MemcpyDeviceToHost = cudaMemcpyDeviceToHost , MemcpyDeviceToDevice = cudaMemcpyDeviceToDevice , MemcpyDefault = cudaMemcpyDefault } |
Functions | |
| template<typename Element, typename FillBuffer, typename PushBuffer> | |
| void | bufferedTransfer (std::size_t size, FillBuffer &fill, PushBuffer &push) |
| template<typename Element, typename FillBuffer> | |
| void | bufferedTransferToDevice (Element *destination, std::size_t size, FillBuffer &fill) |
| This function creates a buffer on the host, the fill handler fills it with data and this function transfers data from the buffer to the destination, which is a pointer to device memory. | |
| template<typename Element, typename PushBuffer> | |
| void | bufferedTransferToHost (const Element *source, std::size_t size, PushBuffer &push) |
| This function creates a buffer on the host, fills it with data transferred from source, which is a pointer to device memory, and the push handler processes the data in the buffer. | |
| void | checkErrorCode (const char *file_name, int line, error_t error) |
| void | checkLaunchConfiguration (const LaunchConfiguration &launch_configuration) |
| Checks that the launch configuration satisfies the limits of the device. | |
| void | deviceSynchronize () |
| template<class T> | |
| void | funcSetCacheConfig (T *func, enum FuncCache cacheConfig) |
| int | getArchitectureMajor (int deviceNum) |
| int | getArchitectureMinor (int deviceNum) |
| constexpr const char * | getBackendName () |
| Returns the name of the GPU backend: "CUDA", "HIP", or "none" when no GPU backend is available. | |
| int | getDevice () |
| Returns the ID of the active device. | |
| int | getDeviceCores (int deviceNum) |
| int | getDeviceCoresPerMultiprocessors (int deviceNum) |
| int | getDeviceCount () |
| Returns the number of devices available in the system. | |
| int | getDeviceMultiprocessors (int deviceNum) |
| std::string | getDeviceName (int deviceNum) |
| bool | getECCEnabled (int deviceNum) |
| std::size_t | getFreeGlobalMemory () |
| __device__ std::size_t | getGlobalBlockIdx_x (std::size_t blockOffset) |
| Returns the global index of the current block in the x-dimension. | |
| __device__ std::size_t | getGlobalBlockIdx_y (std::size_t blockOffset) |
| Returns the global index of the current block in the y-dimension, see getGlobalBlockIdx_x. | |
| __device__ std::size_t | getGlobalBlockIdx_z (std::size_t blockOffset) |
| Returns the global index of the current block in the z-dimension, see getGlobalBlockIdx_x. | |
| std::size_t | getGlobalMemorySize (int deviceNum) |
| __device__ std::size_t | getGlobalThreadIdx_x (std::size_t blockOffset) |
| Returns the global index of the current thread in the x-dimension, see getGlobalBlockIdx_x. | |
| __device__ std::size_t | getGlobalThreadIdx_y (std::size_t blockOffset) |
| Returns the global index of the current thread in the y-dimension, see getGlobalBlockIdx_x. | |
| __device__ std::size_t | getGlobalThreadIdx_z (std::size_t blockOffset) |
| Returns the global index of the current thread in the z-dimension, see getGlobalBlockIdx_x. | |
| unsigned int | getGridSize (std::size_t totalBlocksCount, std::size_t blockOffset, std::size_t maxGridSize) |
| Returns the number of blocks of the grid starting at the block blockOffset when totalBlocksCount blocks are split into grids of at most maxGridSize blocks. | |
| template<typename Index> | |
| __device__ Index | getInterleaving (const Index index) |
| constexpr int | getMaxBlockXSize () |
| constexpr int | getMaxBlockYSize () |
| constexpr int | getMaxBlockZSize () |
| constexpr std::size_t | getMaxGridXSize () |
| std::size_t | getMaxGridXSize (std::size_t threadsPerBlock) |
| Returns the maximal number of blocks of a one-dimensional grid with threadsPerBlock threads per block. | |
| constexpr std::size_t | getMaxGridYSize () |
| constexpr std::size_t | getMaxGridZSize () |
| constexpr int | getMaxThreadsPerBlock () |
| constexpr std::size_t | getMaxThreadsPerGrid () |
| constexpr int | getMaxWarpSize () |
| Returns the maximum possible warp/wavefront size for the current build. | |
| constexpr int | getMinWarpSize () |
| Returns the minimum possible warp/wavefront size across all platforms. | |
| template<typename Index> | |
| Index | getNumberOfBlocks (const Index threads, const unsigned int blockSize) |
| template<typename Index> | |
| Index | getNumberOfGrids (const Index blocks, const unsigned int gridSize) |
| constexpr int | getNumberOfSharedMemoryBanks () |
| int | getRegistersPerMultiprocessor (int deviceNum) |
| template<typename T> | |
| __device__ T * | getSharedMemory () |
| std::size_t | getSharedMemoryPerBlock (int deviceNum) |
| constexpr std::size_t | getTransferBufferSize () |
| constexpr auto | getWarpFullMask () |
| Returns the full mask for warp shuffle operations. | |
| constexpr int | getWarpSize () |
| Returns the warp size for the target device. | |
| int | getWarpSize (int deviceId) |
| Runtime query for the warp/wavefront size of a specific device. | |
| template<typename RawKernel, typename... KernelParameters> | |
| void | launchKernel (RawKernel kernel_function, LaunchConfiguration launch_configuration, KernelParameters &&... parameters) |
| template<typename RawKernel, typename... KernelParameters> | |
| void | launchKernelAsync (RawKernel kernel_function, LaunchConfiguration launch_configuration, KernelParameters &&... parameters) |
| template<typename RawKernel, typename... KernelParameters> | |
| void | launchKernelSync (RawKernel kernel_function, LaunchConfiguration launch_configuration, KernelParameters &&... parameters) |
| template<class T> | |
| __device__ T | ldg (const T &value) |
| Loads data from global memory using the __ldg() intrinsic. | |
| void | memcpy (void *dst, const void *src, std::size_t sizeBytes, MemcpyKind kind) |
| std::ostream & | operator<< (std::ostream &str, const dim3 &d) |
| void | printThreadsSetup (const dim3 &blockSize, const dim3 &blocksCount, const dim3 &gridSize, const dim3 &gridsCount, std::ostream &str=std::cout) |
| void | setDevice (int device) |
| Sets the active device. | |
| stream_t | streamCreateWithPriority (unsigned int flags, int priority) |
| void | streamDestroy (stream_t stream) |
| void | streamSynchronize (stream_t stream) |
| Blocks the host until all operations in the given stream are finished. | |
| __device__ std::uint64_t | warp_ballot (bool predicate) |
| Returns the mask of the lanes of the calling warp for which the predicate is true. | |
| template<typename T> | |
| __device__ T | warp_shuffle_down (T val, int delta) |
| Generic warp shuffle down for any type which satisfies TNL::is_nested_trivially_copyable. | |
| template<typename T> | |
| __device__ T | warp_shuffle_up (T val, int delta) |
| Generic warp shuffle up for any type which satisfies TNL::is_nested_trivially_copyable. | |
| template<typename T> | |
| __device__ T | warp_shuffle_xor (T val, int laneMask) |
| Generic warp shuffle xor for any type which satisfies TNL::is_nested_trivially_copyable. | |
Variables | |
| template<typename T> | |
| constexpr bool | is_warp_shuffle_native_v |
| Variable template listing the types with native __shfl_*_sync overloads. | |
Internal namespace for CUDA/HIP backend support.
|
inline |
Checks that the launch configuration satisfies the limits of the device.
Some violations of the limits are not reported by the runtime, e.g. a grid with too many threads silently runs fewer threads on AMD GPUs.
| std::invalid_argument | when the configuration violates some limit. |
|
inline |
Returns the global index of the current block in the x-dimension.
Kernels launched as several grids get the index of the first block of their grid, blockOffset, from the host as a kernel parameter.
|
inline |
Returns the number of blocks of the grid starting at the block blockOffset when totalBlocksCount blocks are split into grids of at most maxGridSize blocks.
Typical use for a one-dimensional launch of blocksCount blocks:
blockOffset must be smaller than totalBlocksCount.
|
inline |
Returns the maximal number of blocks of a one-dimensional grid with threadsPerBlock threads per block.
Besides the limit on the number of blocks, getMaxGridXSize(), it respects the limit on the number of threads per grid, getMaxThreadsPerGrid(), which AMD GPUs impose and HIP does not check. For a multidimensional grid, pass the number of threads of one slice of the grid in the x-dimension, i.e. the threads per block times the grid size in the other dimensions. Unit tests can force a smaller value by detail::MaxGridXSizeOverride.
|
constexpr |
Returns the maximum possible warp/wavefront size for the current build.
Unlike getWarpSize(), this is consistent between host and device compilation:
Use this for compile-time guards that must be consistent on the host side, e.g. deciding whether to instantiate TPS=64 kernel variants.
|
constexpr |
Returns the minimum possible warp/wavefront size across all platforms.
Always returns 32. Use this for compile-time checks that must allow configs valid on any architecture, e.g. column-major SlicedEllpack launch configs where TPS * SliceSize >= getMinWarpSize() ensures the group covers at least one warp on the smallest-warp-size device.
|
nodiscardconstexpr |
Returns the full mask for warp shuffle operations.
HIP shfl intrinsics require a 64-bit mask regardless of the wavefront size (unused upper bits are zero for wave32). CUDA uses a 32-bit mask.
|
constexpr |
Returns the warp size for the target device.
The warpSize variable is of type int and contains the warp size (in threads) for the target device. This should be used only from device code to develop portable wave-aware code.
https://rocm.docs.amd.com/projects/HIP/en/latest/reference/kernel_language.html#warpsize
|
inlinenodiscard |
Runtime query for the warp/wavefront size of a specific device.
Use this for host-side launch configuration decisions (TPS selection, etc.). For CUDA, always returns 32. For HIP, queries the runtime API for all devices on the first call and returns the cached value afterwards.
| deviceId | The device ID to query (default: current device). |
|
inline |
Returns the mask of the lanes of the calling warp for which the predicate is true.
All lanes of the warp must call it. The mask is 64-bit, since the wavefronts of AMD GPUs may have 64 lanes; lane i corresponds to bit i.
| __device__ T TNL::Backend::warp_shuffle_down | ( | T | val, |
| int | delta ) |
Generic warp shuffle down for any type which satisfies TNL::is_nested_trivially_copyable.
See warp_shuffle_xor for details.
| __device__ T TNL::Backend::warp_shuffle_up | ( | T | val, |
| int | delta ) |
Generic warp shuffle up for any type which satisfies TNL::is_nested_trivially_copyable.
See warp_shuffle_xor for details.
| __device__ T TNL::Backend::warp_shuffle_xor | ( | T | val, |
| int | laneMask ) |
Generic warp shuffle xor for any type which satisfies TNL::is_nested_trivially_copyable.
CUDA's __shfl_xor_sync only has overloads for a fixed set of primitive types (see is_warp_shuffle_native_v). This wrapper splits any other type into an array of words and shuffles each word separately, the same pattern used by NVIDIA's CUB and libcu++ libraries. Works for TNL::Arithmetics::Complex, StaticVector and any other type which satisfies TNL::is_nested_trivially_copyable.
|
inlineconstexpr |
Variable template listing the types with native __shfl_*_sync overloads.
CUDA/HIP __shfl_*_sync functions only accept int, unsigned int, long, unsigned long, long long, unsigned long long, float, and double. Other arithmetic types like bool, char, and short do not have overloads.