René's URL Explorer Experiment

    {
       "@context": "https://schema.org",
       "@type": "Article",
       "name": "CUDA Utility Functions",
       "headline": "CUDA Utility Functions",
       "description": "CUDA utility functions in PyTorch C++ \u2014 device properties, cuBLAS/cuDNN handles, and stream management.",
       "url": "/api/cuda/utilities.html",
       "articleBody": "CUDA Utility Functions# PyTorch provides utility functions for querying and managing CUDA devices, streams, and library handles. Device Management# c10::DeviceIndex torch::cuda::device_count()# Returns the number of CUDA devices available. int c10::cuda::current_device()# Returns the index of the current CUDA device. Example: #include \u003cc10/cuda/CUDAFunctions.h\u003e // Check available devices int num_devices = c10::cuda::device_count(); // Get current device int current = c10::cuda::current_device(); Device Properties# cudaDeviceProp *at::cuda::getCurrentDeviceProperties()# cudaDeviceProp *at::cuda::getDeviceProperties(c10::DeviceIndex device)# bool at::cuda::canDeviceAccessPeer(c10::DeviceIndex device, c10::DeviceIndex peer_device)# int at::cuda::warp_size()# Example: #include \u003cATen/cuda/CUDAContext.h\u003e // Query properties of the current device cudaDeviceProp* props = at::cuda::getCurrentDeviceProperties(); std::cout \u003c\u003c \"Device: \" \u003c\u003c props-\u003ename \u003c\u003c std::endl; std::cout \u003c\u003c \"Compute capability: \" \u003c\u003c props-\u003emajor \u003c\u003c \".\" \u003c\u003c props-\u003eminor \u003c\u003c std::endl; // Query a specific device cudaDeviceProp* dev1_props = at::cuda::getDeviceProperties(1); // Check peer access bool can_access = at::cuda::canDeviceAccessPeer(0, 1); Library Handles# These functions return handles for CUDA math libraries on the current device and stream. They are primarily useful when writing custom CUDA kernels that call cuBLAS, cuSPARSE, or cuSOLVER directly. cublasHandle_t at::cuda::getCurrentCUDABlasHandle(bool setup = true)# cublasLtHandle_t at::cuda::getCurrentCUDABlasLtHandle()# cusparseHandle_t at::cuda::getCurrentCUDASparseHandle()# cusolverDnHandle_t at::cuda::getCurrentCUDASolverDnHandle()# Example: #include \u003cATen/cuda/CUDAContext.h\u003e // Get cuBLAS handle for current device/stream cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); // Get cuSPARSE handle cusparseHandle_t sparse_handle = at::cuda::getCurrentCUDASparseHandle(); cuDNN Descriptors# When writing custom kernels that call cuDNN directly, PyTorch provides RAII wrapper classes for cuDNN descriptors. These are defined in ATen/cudnn/Descriptors.h. Descriptor (Base Class)# template\u003ctypename T, cudnnStatus_t (*ctor)(T**), cudnnStatus_t (*dtor)(T*)\u003eclass Descriptor# Public Functions inline T *desc() const# inline T *desc()# inline T *mut_desc()# A generic RAII wrapper for cuDNN descriptor types. Descriptors default construct to a nullptr and are initialized on first use via mut_desc(). Use desc() for read-only access. TensorDescriptor# class TensorDescriptor : public at::native::Descriptor\u003ccudnnTensorStruct, \u0026cudnnCreateTensorDescriptor, \u0026cudnnDestroyTensorDescriptor\u003e# Public Functions TensorDescriptor() = default# inline explicit TensorDescriptor(const at::Tensor \u0026t, size_t pad = 0)# void set(const at::Tensor \u0026t, size_t pad = 0)# void set(const at::Tensor \u0026t, at::MemoryFormat memory_format, size_t pad = 0)# void set(cudnnDataType_t dataType, IntArrayRef sizes, IntArrayRef strides, size_t pad = 0)# void print()# Wraps cudnnTensorDescriptor_t. Supports padding lower-dimensional tensors to meet cuDNN broadcasting requirements (see pad parameter). Example: #include \u003cATen/cudnn/Descriptors.h\u003e at::Tensor input = torch::randn({32, 3, 224, 224}, torch::kCUDA); at::native::TensorDescriptor desc(input); cudnnTensorDescriptor_t raw = desc.desc(); FilterDescriptor# class FilterDescriptor : public at::native::Descriptor\u003ccudnnFilterStruct, \u0026cudnnCreateFilterDescriptor, \u0026cudnnDestroyFilterDescriptor\u003e# Public Functions inline void set(const at::Tensor \u0026t, int64_t pad = 0)# void set(const at::Tensor \u0026t, const at::MemoryFormat memory_format, int64_t pad = 0)# void print()# Wraps cudnnFilterDescriptor_t for convolution filter weights. ConvolutionDescriptor# struct ConvolutionDescriptor : public at::native::Descriptor\u003ccudnnConvolutionStruct, \u0026cudnnCreateConvolutionDescriptor, \u0026cudnnDestroyConvolutionDescriptor\u003e# Public Functions inline void set(cudnnDataType_t dataType, int dim, int *pad, int *stride, int *upscale, int groups, bool allow_tf32)# Wraps cudnnConvolutionDescriptor_t. Configures padding, stride, dilation, groups, and math type (TF32, tensor ops) for convolution operations. RNNDataDescriptor# class RNNDataDescriptor : public at::native::Descriptor\u003ccudnnRNNDataStruct, \u0026cudnnCreateRNNDataDescriptor, \u0026cudnnDestroyRNNDataDescriptor\u003e# Public Functions void set(const at::Tensor \u0026t, cudnnRNNDataLayout_t layout, int maxSeqLength, int batchSize, int vectorSize, const int *seqLengthArray)# Wraps cudnnRNNDataDescriptor_t for variable-length sequence data. DropoutDescriptor# struct DropoutDescriptor : public at::native::Descriptor\u003ccudnnDropoutStruct, \u0026cudnnCreateDropoutDescriptor, \u0026cudnnDestroyDropoutDescriptor\u003e# Public Functions inline void initialize_rng(cudnnHandle_t handle, float dropout, long long int seed, const TensorOptions \u0026options)# inline void set(cudnnHandle_t handle, float dropout, const at::Tensor \u0026state)# inline void set_no_dropout(cudnnHandle_t handle)# Public Members at::Tensor state# Wraps cudnnDropoutDescriptor_t. Manages RNG state for cuDNN dropout. ActivationDescriptor# struct ActivationDescriptor : public at::native::Descriptor\u003ccudnnActivationStruct, \u0026cudnnCreateActivationDescriptor, \u0026cudnnDestroyActivationDescriptor\u003e# Public Functions inline void set(cudnnActivationMode_t mode)# Wraps cudnnActivationDescriptor_t. SpatialTransformerDescriptor# struct SpatialTransformerDescriptor : public at::native::Descriptor\u003ccudnnSpatialTransformerStruct, \u0026cudnnCreateSpatialTransformerDescriptor, \u0026cudnnDestroySpatialTransformerDescriptor\u003e# Public Functions inline void set(cudnnDataType_t dataType, int dim, int *size)# CTCLossDescriptor# struct CTCLossDescriptor : public at::native::Descriptor\u003ccudnnCTCLossStruct, \u0026cudnnCreateCTCLossDescriptor, \u0026cudnnDestroyCTCLossDescriptor\u003e# Public Functions inline void set(cudnnDataType_t datatype)# inline void setEx(cudnnDataType_t datatype, cudnnLossNormalizationMode_t normMode, cudnnNanPropagation_t gradMode)# inline void set_v8_v9(cudnnDataType_t datatype, cudnnLossNormalizationMode_t normMode, cudnnNanPropagation_t gradMode, int maxLabelLength)# Stream Management# CUDAStream c10::cuda::getDefaultCUDAStream(DeviceIndex device_index = -1)# Get the default CUDA stream, for the passed CUDA device, or for the current device if no device index is passed. The default stream is where most computation occurs when you aren\u2019t explicitly using streams. CUDAStream c10::cuda::getCurrentCUDAStream(DeviceIndex device_index = -1)# Get the current CUDA stream, for the passed CUDA device, or for the current device if no device index is passed. The current CUDA stream will usually be the default CUDA stream for the device, but it may be different if someone called \u2018setCurrentCUDAStream\u2019 or used \u2018StreamGuard\u2019 or \u2018CUDAStreamGuard\u2019. void c10::cuda::setCurrentCUDAStream(CUDAStream stream)# Set the current stream on the device of the passed in stream to be the passed in stream. Yes, you read that right: this function has nothing to do with the current device: it toggles the current stream of the device of the passed stream. Confused? Avoid using this function; prefer using \u2018CUDAStreamGuard\u2019 instead (which will switch both your current device and current stream in the way you expect, and reset it back to its original state afterwards). CUDAStream c10::cuda::getStreamFromPool(const bool isHighPriority = false, DeviceIndex device = -1)# Get a new stream from the CUDA stream pool. You can think of this as \u201ccreating\u201d a new stream, but no such creation actually happens; instead, streams are preallocated from the pool and returned in a round-robin fashion. You can request a stream from the high priority pool by setting isHighPriority to true, or a stream for a specific device by setting device (defaulting to the current CUDA stream.) CUDAStream c10::cuda::getStreamFromExternal(cudaStream_t ext_stream, DeviceIndex device_index)# Get a CUDAStream from a externally allocated one. This is mainly for interoperability with different libraries where we want to operate on a non-torch allocated stream for data exchange or similar purposes Example: #include \u003cc10/cuda/CUDAStream.h\u003e // Create and set custom stream auto stream = c10::cuda::getStreamFromPool(); c10::cuda::setCurrentCUDAStream(stream); // Get default stream auto default_stream = c10::cuda::getDefaultCUDAStream(); // Wrap an externally created CUDA stream cudaStream_t ext_stream; cudaStreamCreate(\u0026ext_stream); auto wrapped = c10::cuda::getStreamFromExternal(ext_stream, /*device_index=*/0);",
       "author": {
         "@type": "Organization",
         "name": "PyTorch Contributors",
         "url": "https://pytorch.org"
       },
       "image": "https://pytorch.org/docs/stable/_static/img/pytorch_seo.png",
       "mainEntityOfPage": {
         "@type": "WebPage",
         "@id": "/api/cuda/utilities.html"
       },
       "datePublished": "2023-01-01T00:00:00Z",
       "dateModified": "2023-01-01T00:00:00Z"
     }