3_ConvTranspose3d_Sum_LayerNorm_AvgPool_GELU • warp_optimized_layernorm_base_baseimport torch
import torch.nn as nn
import torch.nn.functional as F
def module_fn(
x: torch.Tensor,
conv_transpose_weight: torch.Tensor,
conv_transpose_bias: torch.Tensor,
sum_weight: torch.Tensor,
norm_weight: torch.Tensor,
norm_bias: torch.Tensor,
stride: tuple,
padding: tuple,
output_padding: tuple,
pool_kernel_size: tuple,
norm_shape: tuple,
) -> torch.Tensor:
"""
Functional implementation of a sequence of operations:
1. 3D transposed convolution
2. Addition with a learnable weight
3. Layer normalization
4. 3D average pooling
5. GELU activation
Args:
x (torch.Tensor): Input tensor of shape (batch_size, in_channels, depth, height, width)
conv_transpose_weight (torch.Tensor): Weight tensor for transposed convolution
conv_transpose_bias (torch.Tensor): Bias tensor for transposed convolution
sum_weight (torch.Tensor): Learnable weight for addition
norm_weight (torch.Tensor): Weight tensor for layer normalization
norm_bias (torch.Tensor): Bias tensor for layer normalization
stride (tuple): Stride for transposed convolution, as (depth_stride, height_stride, width_stride)
padding (tuple): Padding for transposed convolution, as (depth_pad, height_pad, width_pad)
output_padding (tuple): Output padding for transposed convolution, as (depth_pad, height_pad, width_pad)
pool_kernel_size (tuple): Kernel size for average pooling, as (depth_kernel, height_kernel, width_kernel)
norm_shape (tuple): Shape for layer normalization
Returns:
torch.Tensor: Output tensor after applying all operations
"""
x = F.conv_transpose3d(
x,
conv_transpose_weight,
bias=conv_transpose_bias,
stride=stride,
padding=padding,
output_padding=output_padding,
)
x = x + sum_weight
x = F.layer_norm(x, norm_shape, norm_weight, norm_bias)
x = F.avg_pool3d(x, kernel_size=pool_kernel_size)
x = F.gelu(x)
return x
class Model(nn.Module):
"""
Model that performs a 3D transposed convolution, followed by a sum, layer normalization, average pooling, and GELU activation.
"""
def __init__(
self,
in_channels,
out_channels,
kernel_size,
stride,
padding,
output_padding,
sum_weight,
norm_shape,
pool_kernel_size,
):
super(Model, self).__init__()
conv = nn.ConvTranspose3d(
in_channels,
out_channels,
kernel_size,
stride=stride,
padding=padding,
output_padding=output_padding,
)
self.conv_transpose_weight = nn.Parameter(conv.weight)
self.conv_transpose_bias = nn.Parameter(conv.bias)
self.sum_weight = nn.Parameter(torch.tensor(sum_weight))
norm = nn.LayerNorm(norm_shape)
self.norm_weight = nn.Parameter(norm.weight + torch.randn(norm_shape) * 0.02)
self.norm_bias = nn.Parameter(norm.bias + torch.randn(norm_shape) * 0.02)
def forward(
self,
x,
stride,
padding,
output_padding,
pool_kernel_size,
norm_shape,
fn=module_fn,
):
return fn(
x,
self.conv_transpose_weight,
self.conv_transpose_bias,
self.sum_weight,
self.norm_weight,
self.norm_bias,
stride,
padding,
output_padding,
pool_kernel_size,
norm_shape,
)
batch_size = 128
in_channels = 32
out_channels = 64
depth, height, width = 16, 32, 32
kernel_size = (3, 3, 3)
stride = (2, 2, 2)
padding = (1, 1, 1)
output_padding = (1, 1, 1)
sum_weight = 1.0
norm_shape = (out_channels,)
pool_kernel_size = (2, 2, 2)
def get_inputs():
return [
torch.randn(batch_size, in_channels, depth, height, width),
stride,
padding,
output_padding,
pool_kernel_size,
norm_shape,
]
def get_init_inputs():
return [
in_channels,
out_channels,
kernel_size,
stride,
padding,
output_padding,
sum_weight,
norm_shape,
pool_kernel_size,
]
import torch
import torch.nn as nn
class Model(nn.Module):
"""
Model that performs a 3D transposed convolution, followed by a sum, layer normalization, average pooling, and GELU activation.
"""
def __init__(self, in_channels, out_channels, kernel_size, stride, padding, output_padding, sum_weight, norm_shape, pool_kernel_size):
super(Model, self).__init__()
self.conv_transpose = nn.ConvTranspose3d(in_channels, out_channels, kernel_size, stride=stride, padding=padding, output_padding=output_padding)
self.sum_weight = nn.Parameter(torch.tensor(sum_weight))
self.norm = nn.LayerNorm(norm_shape)
self.norm.weight = nn.Parameter(self.norm.weight + torch.randn(norm_shape)*0.02)
self.norm.bias = nn.Parameter(self.norm.bias + torch.randn(norm_shape)*0.02)
self.avg_pool = nn.AvgPool3d(kernel_size=pool_kernel_size)
self.gelu = nn.GELU()
def forward(self, x):
x = self.conv_transpose(x)
x = x + self.sum_weight
x = self.norm(x)
x = self.avg_pool(x)
x = self.gelu(x)
return x
batch_size = 128
in_channels = 32
out_channels = 64
depth, height, width = 16, 32, 32
kernel_size = (3, 3, 3)
stride = (2, 2, 2)
padding = (1, 1, 1)
output_padding = (1, 1, 1)
sum_weight = 1.0
norm_shape = (out_channels,)
pool_kernel_size = (2, 2, 2)
def get_inputs():
return [torch.randn(batch_size, in_channels, depth, height, width)]
def get_init_inputs():
return [in_channels, out_channels, kernel_size, stride, padding, output_padding, sum_weight, norm_shape, pool_kernel_size]#include <torch/extension.h>
#include <ATen/ATen.h>
#include <cuda_runtime.h>
#include <vector>
#define BLOCK_SIZE 256
#define WARP_SIZE 32
#define WARPS_PER_BLOCK (BLOCK_SIZE/WARP_SIZE)
template <typename T>
__global__ void warp_optimized_layernorm_kernel(
const T* __restrict__ input,
const T* __restrict__ gamma,
const T* __restrict__ beta,
T* __restrict__ output,
const int n1,
const int n2
) {
__shared__ float s_mean[WARPS_PER_BLOCK];
__shared__ float s_var[WARPS_PER_BLOCK];
const unsigned int tid = threadIdx.x;
const unsigned int lane_id = tid % WARP_SIZE;
const unsigned int warp_id = tid / WARP_SIZE;
const unsigned int bid = blockIdx.x;
float local_sum = 0.0f;
float local_sq_sum = 0.0f;
const int offset = bid * n2;
#pragma unroll 4
for (int i = tid; i < n2; i += BLOCK_SIZE) {
float val = __ldg(&input[offset + i]);
local_sum += val;
local_sq_sum += val * val;
}
#pragma unroll
for (int delta = WARP_SIZE/2; delta > 0; delta >>= 1) {
local_sum += __shfl_xor_sync(0xffffffff, local_sum, delta);
local_sq_sum += __shfl_xor_sync(0xffffffff, local_sq_sum, delta);
}
if (lane_id == 0) {
s_mean[warp_id] = local_sum;
s_var[warp_id] = local_sq_sum;
}
__syncthreads();
if (warp_id == 0) {
local_sum = (lane_id < WARPS_PER_BLOCK) ? s_mean[lane_id] : 0.0f;
local_sq_sum = (lane_id < WARPS_PER_BLOCK) ? s_var[lane_id] : 0.0f;
#pragma unroll
for (int delta = WARP_SIZE/2; delta > 0; delta >>= 1) {
local_sum += __shfl_xor_sync(0xffffffff, local_sum, delta);
local_sq_sum += __shfl_xor_sync(0xffffffff, local_sq_sum, delta);
}
if (lane_id == 0) {
s_mean[0] = local_sum / n2;
s_var[0] = (local_sq_sum / n2) - (s_mean[0] * s_mean[0]);
}
}
__syncthreads();
const float mean = s_mean[0];
const float inv_std = rsqrtf(s_var[0] + 1e-5f);
#pragma unroll 4
for (int i = tid; i < n2; i += BLOCK_SIZE) {
const int idx = offset + i;
const float normalized = (input[idx] - mean) * inv_std;
output[idx] = gamma[i] * normalized + beta[i];
}
}
torch::Tensor forward(
torch::Tensor x,
torch::Tensor conv_transpose_weight,
torch::Tensor conv_transpose_bias,
torch::Tensor sum_weight,
torch::Tensor norm_weight,
torch::Tensor norm_bias,
std::vector<int64_t> stride,
std::vector<int64_t> padding,
std::vector<int64_t> output_padding,
std::vector<int64_t> pool_kernel_size,
std::vector<int64_t> norm_shape
) {
x = x.contiguous();
conv_transpose_weight = conv_transpose_weight.contiguous();
sum_weight = sum_weight.contiguous();
norm_weight = norm_weight.contiguous();
norm_bias = norm_bias.contiguous();
at::IntArrayRef strideRef(stride);
at::IntArrayRef paddingRef(padding);
at::IntArrayRef outputPaddingRef(output_padding);
at::IntArrayRef poolKernelRef(pool_kernel_size);
auto out = at::conv_transpose3d(
x,
conv_transpose_weight,
conv_transpose_bias,
strideRef,
paddingRef,
outputPaddingRef,
1,
1
);
out.add_(sum_weight);
auto out_size = out.sizes();
int64_t n1 = 1;
for (int i = 0; i < out.dim() - norm_shape.size(); ++i) {
n1 *= out_size[i];
}
int64_t n2 = 1;
for (size_t i = 0; i < norm_shape.size(); ++i) {
n2 *= norm_shape[i];
}
auto output = torch::empty_like(out);
dim3 grid(n1);
dim3 block(BLOCK_SIZE);
warp_optimized_layernorm_kernel<<<grid, block>>>(
out.data_ptr<float>(),
norm_weight.data_ptr<float>(),
norm_bias.data_ptr<float>(),
output.data_ptr<float>(),
n1,
n2
);
output = at::gelu(at::avg_pool3d(
output,
poolKernelRef,
poolKernelRef,
{0,0,0},
false,
true
));
return output;
}
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
m.def("forward", &forward, "Warp optimized module_fn forward (CUDA)");
}| Metric | Value | Unit | Variance | Samples |
|---|---|---|---|---|
| Executed Ipc Active | 2.290 | inst/cycle | 0.000 | 5 |
| Executed Ipc Elapsed | 2.290 | inst/cycle | 0.000 | 5 |
| Issue Slots Busy | 58.154 | % | 0.000 | 5 |
| Issued Ipc Active | 2.330 | inst/cycle | 0.000 | 5 |
| SM Busy | 58.154 | % | 0.000 | 5 |
| Memory Throughput | 382499820595.464 | byte/second | 925873378621357.000 | 5 |
| Mem Busy | 52.562 | % | 0.000 | 5 |
| Max Bandwidth | 47.062 | % | 0.000 | 5 |
| L1/TEX Hit Rate | 59.950 | % | 0.000 | 5 |
| L2 Hit Rate | 50.044 | % | 0.000 | 5 |
| Mem Pipes Busy | 49.918 | % | 0.000 | 5 |
| Warp Cycles Per Issued Instruction | 22.640 | cycle | 0.000 | 5 |
| Warp Cycles Per Executed Instruction | 22.960 | cycle | 0.000 | 5 |
| Avg. Active Threads Per Warp | 27.400 | 0.000 | 5 | |
| Avg. Not Predicated Off Threads Per Warp | 26.330 | 0.000 | 5 | |
| Max Active Clusters | 0.000 | cluster | 0.000 | 5 |
| Max Cluster Size | 8.000 | block | 0.000 | 5 |
| Overall GPU Occupancy | 0.000 | % | 0.000 | 5 |
| Cluster Occupancy | 0.000 | % | 0.000 | 5 |
| Block Limit SM | 32.000 | block | 0.000 | 5 |
| Block Limit Registers | 10.000 | block | 0.000 | 5 |
| Block Limit Shared Mem | 28.000 | block | 0.000 | 5 |
| Block Limit Warps | 8.000 | block | 0.000 | 5 |
| Theoretical Active Warps per SM | 64.000 | warp | 0.000 | 5 |
| Theoretical Occupancy | 100.000 | % | 0.000 | 5 |
| Achieved Occupancy | 83.140 | % | 0.000 | 5 |
| Achieved Active Warps Per SM | 53.210 | warp | 0.000 | 5 |
| Rule | Description |
|---|---|
| INF HighPipeUtilization | ALU is the highest-utilized pipeline (28.7%) based on active cycles, taking into account the rates of its different instructions. It executes integer and logic operations. It is well-utilized, but should not be a bottleneck. |
| INF CPIStall | Check the Warp Stall Sampling (All Cycles) table for the top stall locations in your source based on sampling data. The Kernel Profiling Guide (https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html#metrics-reference) provides more details on each stall reason. |
| WRN Occupancy | This kernel's theoretical occupancy is not impacted by any block limit. The difference between calculated theoretical (100.0%) and measured achieved occupancy (83.1%) can be the result of warp scheduling overheads or workload imbalances during the kernel execution. Load imbalances can occur between warps within a block as well as across blocks of the same kernel. See the CUDA Best Practices Guide (https://docs.nvidia.com/cuda/cuda-c-best-practices-guide/index.html#occupancy) for more details on optimizing occupancy. |
| Operation / Metric | Value | Unit |
|---|---|---|
| aten::conv_transpose3d | ||
| CPU Time | 1364391.02 | μs |
| Device Time | 2588067.38 | μs |
| Self CPU Time | 667.01 | μs |
| Self Device Time | 0.00 | μs |
| CPU Memory Usage | 0 | B |
| Device Memory Usage | 0 | B |
| Self CPU Memory Usage | 0 | B |
| Self Device Memory Usage | 0 | B |
| aten::convolution | ||
| CPU Time | 1363724.01 | μs |
| Device Time | 2588067.38 | μs |
| Self CPU Time | 894.85 | μs |
| Self Device Time | 0.00 | μs |
| CPU Memory Usage | 0 | B |
| Device Memory Usage | 0 | B |
| Self CPU Memory Usage | 0 | B |
| Self Device Memory Usage | 0 | B |
| aten::_convolution | ||
| CPU Time | 1362829.16 | μs |
| Device Time | 2588067.38 | μs |
| Self CPU Time | 1852.74 | μs |
| Self Device Time | 0.00 | μs |
| CPU Memory Usage | 0 | B |
| Device Memory Usage | 0 | B |
| Self CPU Memory Usage | 0 | B |
| Self Device Memory Usage | 0 | B |
| cudaLaunchKernel | ||
| CPU Time | 7603619.02 | μs |
| Device Time | 6466.66 | μs |
| Self CPU Time | 7603619.02 | μs |
| Self Device Time | 6466.66 | μs |
| CPU Memory Usage | 0 | B |
| Device Memory Usage | 0 | B |
| Self CPU Memory Usage | 0 | B |
| Self Device Memory Usage | 0 | B |
| aten::add_ | ||
| CPU Time | 1924685.43 | μs |
| Device Time | 1951129.88 | μs |
| Self CPU Time | 4611.63 | μs |
| Self Device Time | 1951129.88 | μs |
| CPU Memory Usage | 0 | B |
| Device Memory Usage | 0 | B |
| Self CPU Memory Usage | 0 | B |
| Self Device Memory Usage | 0 | B |
| void warp_optimized_layernorm_kernel<float>(float const*, float const*, float const*, float*, int, int) | ||
| CPU Time | 0.00 | μs |
| Device Time | 5597026.19 | μs |
| Self CPU Time | 0.00 | μs |
| Self Device Time | 5597026.19 | μs |
| CPU Memory Usage | 0 | B |
| Device Memory Usage | 0 | B |
| Self CPU Memory Usage | 0 | B |
| Self Device Memory Usage | 0 | B |
| aten::avg_pool3d | ||
| CPU Time | 4922730.33 | μs |
| Device Time | 801301.25 | μs |
| Self CPU Time | 3534.69 | μs |
| Self Device Time | 801301.25 | μs |
| CPU Memory Usage | 0 | B |
| Device Memory Usage | 0 | B |
| Self CPU Memory Usage | 0 | B |
| Self Device Memory Usage | 0 | B |
| cudaDeviceSynchronize | ||
| CPU Time | 2390095.57 | μs |
| Device Time | 3720.40 | μs |
| Self CPU Time | 2390095.57 | μs |
| Self Device Time | 3720.40 | μs |
| CPU Memory Usage | 0 | B |
| Device Memory Usage | 0 | B |
| Self CPU Memory Usage | 0 | B |
| Self Device Memory Usage | 0 | B |
45298 warnings generated when compiling for host. Suppressed 45331 warnings (45284 in non-user code, 47 NOLINT). Use -header-filter=.* to display errors from all non-system headers. Use -system-headers to display errors from system headers as well.