utils/hardware_utils.py

import subprocess
import psutil
import functools
from torch._inductor.runtime.hints import DeviceProperties
from torch._inductor.utils import get_gpu_type
from typing import Union
import torch


@functools.cache
def get_system_ram_gb():
    """
    Gets the total physical system RAM in Gigabytes.

    Returns:
        float: Total system RAM in GB, or None if it cannot be determined.
    """
    try:
        # Get virtual memory details
        virtual_memory = psutil.virtual_memory()
        # Total physical memory in bytes
        total_ram_bytes = virtual_memory.total
        # Convert bytes to gigabytes (1 GB = 1024^3 bytes)
        total_ram_gb = total_ram_bytes / (1024**3)
        return total_ram_gb
    except Exception as e:
        print(f"Error getting system RAM: {e}")
        return None


@functools.cache
def get_gpu_vram_gb():
    """
    Gets the total GPU VRAM in Gigabytes using the nvidia-smi command.
    This function is intended for NVIDIA GPUs.

    Returns:
        float: Total GPU VRAM in GB, or None if it cannot be determined.
    """
    try:
        # Execute the nvidia-smi command to get GPU memory info
        # The command queries for the total memory and outputs it in MiB
        result = subprocess.run(
            ["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"],
            capture_output=True,
            text=True,
            check=True,
        )
        # The output will be a string like "12288\n" for the first GPU
        # We take the first line in case there are multiple GPUs
        vram_mib = int(result.stdout.strip().split("\n")[0])
        # Convert MiB to Gigabytes (1 GB = 1024 MiB)
        vram_gb = vram_mib / 1024
        return vram_gb
    except FileNotFoundError:
        # This error occurs if nvidia-smi is not installed or not in the PATH
        print("INFO: 'nvidia-smi' command not found. Cannot determine GPU VRAM.")
        print("      This is expected if you don't have an NVIDIA GPU or drivers installed.")
        return None
    except (subprocess.CalledProcessError, ValueError, IndexError) as e:
        # Handles other potential errors like command failure or parsing issues
        print(f"Error getting GPU VRAM: {e}")
        return None


def categorize_ram(ram_gb):
    """
    Categorizes RAM into 'small', 'medium', or 'large'.

    Args:
        ram_gb (float): The amount of RAM in GB.

    Returns:
        str: The category ('small', 'medium', 'large') or 'unknown'.
    """
    if ram_gb is None:
        return "unknown"
    if ram_gb <= 20:
        return "small"
    elif 20 < ram_gb <= 40:
        return "medium"
    else:  # ram_gb > 40
        return "large"


def categorize_vram(vram_gb):
    """
    Categorizes VRAM into 'small', 'medium', or 'large'.

    Args:
        vram_gb (float): The amount of VRAM in GB.

    Returns:
        str: The category ('small', 'medium', 'large') or 'not applicable/unknown'.
    """
    if vram_gb is None:
        return "not applicable/unknown"
    if vram_gb <= 8:
        return "small"
    elif 8 < vram_gb <= 24:
        return "medium"
    else:  # vram_gb > 24
        return "large"


@functools.cache
def is_compile_friendly_gpu(index_or_device: Union[int, str, torch.device] = 0) -> bool:
    """Hand-coded rules from experiments. Don't take seriously."""
    if isinstance(index_or_device, torch.device):
        device = index_or_device
    elif isinstance(index_or_device, str):
        device = torch.device(index_or_device)
    else:
        device = torch.device(get_gpu_type(), index_or_device)

    prop = DeviceProperties.create(device)
    return prop.major >= 8


@functools.lru_cache()
def is_sm_version(major: int, minor: int) -> bool:
    """Check if the CUDA version is exactly major.minor"""
    is_cuda = torch.cuda.is_available() and torch.version.cuda
    return torch.cuda.get_device_capability() == (major, minor) if is_cuda else False


def is_fp8_friendly():
    return is_sm_version(8, 9)