The AI CUDA Engineer 👷

import torch
import torch.nn as nn
import torch.nn.functional as F


def module_fn(
    x: torch.Tensor,
    weight: torch.Tensor,
    bias: torch.Tensor,
    scaling_factor: float,
) -> torch.Tensor:
    """
    Applies linear transformation, Swish activation, and scaling.

    Args:
        x (torch.Tensor): Input tensor of shape (batch_size, in_features)
        weight (torch.Tensor): Weight matrix of shape (out_features, in_features)
        bias (torch.Tensor): Bias vector of shape (out_features)
        scaling_factor (float): Factor to scale the output by

    Returns:
        torch.Tensor: Output tensor of shape (batch_size, out_features)
    """
    x = F.linear(x, weight, bias)
    x = x * torch.sigmoid(x)  # Swish activation
    x = x * scaling_factor
    return x


class Model(nn.Module):
    """
    Simple model that performs a matrix multiplication, applies Swish activation, and scales the result.
    """

    def __init__(self, in_features, out_features, scaling_factor):
        super(Model, self).__init__()
        gemm = nn.Linear(in_features, out_features)
        self.weight = nn.Parameter(gemm.weight)
        self.bias = nn.Parameter(gemm.bias)
        self.scaling_factor = scaling_factor

    def forward(self, x, fn=module_fn):
        return fn(x, self.weight, self.bias, self.scaling_factor)


batch_size = 128
in_features = 1024
out_features = 512
scaling_factor = 2.0


def get_inputs():
    return [torch.randn(batch_size, in_features)]


def get_init_inputs():
    return [in_features, out_features, scaling_factor]

#include <torch/extension.h>
#include <cuda.h>
#include <cuda_runtime.h>

// Kernel that evenly partitions the workload across all threads
__global__ void balanced_workload_swish_scaling_kernel(const float* __restrict__ input,
                                                        float* __restrict__ output,
                                                        float scaling_factor,
                                                        int N) {
    // Compute a unique thread id
    int tid = blockIdx.x * blockDim.x + threadIdx.x;
    int total_threads = gridDim.x * blockDim.x;
    
    // Determine how many elements each thread should process
    int work_per_thread = N / total_threads;
    int remainder = N % total_threads;

    // Calculate start index for this thread using a balanced partitioning
    int start = tid * work_per_thread + (tid < remainder ? tid : remainder);
    int num_elements = work_per_thread + (tid < remainder ? 1 : 0);
    int end = start + num_elements;

    // Process assigned elements
    for (int i = start; i < end; i++) {
        float x = input[i];
        float sigmoid = 1.0f / (1.0f + expf(-x));
        output[i] = x * sigmoid * scaling_factor;
    }
}

// Forward function: computes the linear transformation then applies the swish activation and scaling
torch::Tensor forward(
    torch::Tensor x,
    torch::Tensor weight,
    torch::Tensor bias,
    double scaling_factor) {

    x = x.contiguous();
    weight = weight.contiguous();
    bias = bias.contiguous();

    TORCH_CHECK(x.is_cuda(), "Input tensor 'x' must be a CUDA tensor.");
    TORCH_CHECK(weight.is_cuda(), "Weight tensor must be a CUDA tensor.");
    TORCH_CHECK(bias.is_cuda(), "Bias tensor must be a CUDA tensor.");

    TORCH_CHECK(x.scalar_type() == at::kFloat, "Input tensor 'x' must be of type torch.float32.");
    TORCH_CHECK(weight.scalar_type() == at::kFloat, "Weight tensor must be of type torch.float32.");
    TORCH_CHECK(bias.scalar_type() == at::kFloat, "Bias tensor must be of type torch.float32.");

    // Compute the linear transformation: y = x @ weight.T + bias
    auto y = at::addmm(bias, x, weight.t());
    auto output = at::empty_like(y);

    int N = y.numel();
   
    // Use a configuration with 256 threads per block and enough blocks to cover all elements
    int threads = 256;
    int blocks = (N + threads - 1) / threads;

    // Launch the balanced workload kernel
    balanced_workload_swish_scaling_kernel<<<blocks, threads>>>(
        y.data_ptr<float>(),
        output.data_ptr<float>(),
        static_cast<float>(scaling_factor),
        N
    );

    cudaError_t err = cudaGetLastError();
    TORCH_CHECK(err == cudaSuccess, "CUDA kernel failed : ", cudaGetErrorString(err));

    return output;
}

PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
    m.def("forward", &forward, "CUDA forward function with balanced workload distribution");
}