NVIDIAGet GPU utilization-cpp.md

In deep learning inference, in order to utilize GPU more efficiently, you need to pay attention to creating new instances and allocating them to different GPU devices among multiple inference task instances. How much is left to the current GPU for better allocation?

Code Directory

.
├── CMakeLists.txt
├── src
│ └── main.cpp
├── ubuntu_build.sh
└── win10_vs2019_build.bat

windows

Prerequisites

Make sure you have installed the Nvidia driver and CUDA installation package

nvidia-smi.exe

It can be run, the screenshot is as follows:

nvcc --version

It can be run, the screenshot is as follows:

cuda installation directory

C:\Program Files\\
VIDIA GPU Computing Toolkit\CUDA\v11.8

Modify CUDA_ROOT in the CMakeLists.txt file to your own installation directory

NOTE: Note the slashes / and backslashes \

Build project

Open the console and execute win10_vs2019_build.bat, or double-click win10_vs2019_build.bat

NOTE: The premise is that vs2019, “Visual Studio 16 2019” is installed, other VS versions can be replaced simultaneously

Compile

Enter the win10_build directory, double-click get_gpu_info.sln, change the compilation type to Release, and generate get_gpu_info.exe after compilation. Just double-click to run.

linux

Build, compile, execute

bash ubuntu_build.sh

Code Appendix

src/main.cpp

/********************************************** *******************************\
|* *|
|* Copyright 2010-2016 NVIDIA Corporation. All rights reserved. *|
|* *|
|* NOTICE TO USER: *|
|* *|
|* This source code is subject to NVIDIA ownership rights under U.S. *|
|* and international Copyright laws. Users and possessors of this *|
|* source code are hereby granted a nonexclusive, royalty-free *|
|* license to use this code in individual and commercial software. *|
|* *|
|* NVIDIA MAKES NO REPRESENTATION ABOUT THE SUITABILITY OF THIS SOURCE *|
|* CODE FOR ANY PURPOSE. IT IS PROVIDED "AS IS" WITHOUT EXPRESS OR *|
|* IMPLIED WARRANTY OF ANY KIND. NVIDIA DISCLAIMS ALL WARRANTIES WITH *|
|* REGARD TO THIS SOURCE CODE, INCLUDING ALL IMPLIED WARRANTIES OF *|
|* MERCHANTABILITY, NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR *|
|* PURPOSE. IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, *|
|* INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES *|
|* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN *|
|* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING *|
|* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOURCE *|
|* CODE. *|
|* *|
|* U.S. Government End Users. This source code is a "commercial item" *|
|* as that term is defined at 48 C.F.R. 2.101 (OCT 1995), consisting of *|
|* of "commercial computer software" and "commercial computer software *|
|* documentation" as such terms are used in 48 C.F.R. 12.212 (SEPT 1995) *|
|* and is provided to the U.S. Government only as a commercial end item. *|
|* Consistent with 48 C.F.R.12.212 and 48 C.F.R. 227.7202-1 through *|
|* 227.7202-4 (JUNE 1995), all U.S. Government End Users acquire the *|
|* source code with only those rights set forth herein. *|
|* *|
|* Any use of this source code in individual and commercial software must *|
|* include, in the user documentation and internal comments to the code, *|
|* the above Disclaimer and U.S. Government End Users Notice. *|
|* *|
|* *|
\************************************************ ***************************/

#include <stdio.h>
#include <nvml.h>

static const char *convertToComputeModeString(nvmlComputeMode_t mode)
{<!-- -->
    switch (mode)
    {<!-- -->
    case NVML_COMPUTEMODE_DEFAULT:
        return "Default";
    case NVML_COMPUTEMODE_EXCLUSIVE_THREAD:
        return "Exclusive_Thread";
    case NVML_COMPUTEMODE_PROHIBITED:
        return "Prohibited";
    case NVML_COMPUTEMODE_EXCLUSIVE_PROCESS:
        return "Exclusive Process";
    default:
        return "Unknown";
    }
}


int main(int argc, char* argv[])
{<!-- -->
    nvmlReturn_t result;
    unsigned int device_count;

    //Initialize NVML
    result = nvmlInit();
    if (result != NVML_SUCCESS)
    {<!-- -->
        printf("Failed to initialize NVML: %s\\
", nvmlErrorString(result));
        printf("Press ENTER to continue...\\
");
        getchar();
        return (int)(result);
    }

    // Get the number of devices
    result = nvmlDeviceGetCount( & amp;device_count);
    if (result != NVML_SUCCESS)
    {<!-- -->
        printf("Failed to get device count: %s\\
", nvmlErrorString(result));
        printf("Press ENTER to continue...\\
");
        getchar();
        return (int)(result);
    }

    // Traverse the number of devices
    printf("\\
Found %u device%s, Listing devices:\\
", device_count,
        device_count != 1 ? "s" : "");
    printf("------------------------------------------------ ----------------\\
");
    printf("| Device ID | Device Name\t\t| pci.busId | GPU Util | Mem Util |\\
");
    for (int i = 0; i < device_count; + + i)
    {<!-- -->
        nvmlDevice_t device;
        char device_name[NVML_DEVICE_NAME_BUFFER_SIZE];
        nvmlPciInfo_t pci;
        nvmlComputeMode_t compute_mode;

        // Get the device, you can also use other methods to get the device
        // nvmlDeviceGetHandleBySerial
        // nvmlDeviceGetHandleByPciBusId
        result = nvmlDeviceGetHandleByIndex(i, & amp;device);
        if (result != NVML_SUCCESS)
        {<!-- -->
            printf("Failed to get handle for device %d: %s\\
", i,
                nvmlErrorString(result));
            continue;
        }

        // Get GPU device name
        result = nvmlDeviceGetName(device, device_name, NVML_DEVICE_NAME_BUFFER_SIZE);
        if (result != NVML_SUCCESS)
        {<!-- -->
            printf("Failed to get name for device %d: %s\\
", i,
                nvmlErrorString(result));
            continue;
        }

        // pci.busId is very useful to know which device physically you're talking to
        // Using PCI identifier you can also match nvmlDevice handle to CUDA device.
        result = nvmlDeviceGetPciInfo(device, & amp;pci);
        if (result != NVML_SUCCESS)
        {<!-- -->
            printf("Failed to get pci info for device %u: %s\\
", i,
                nvmlErrorString(result));
            continue;
        }

        // Get the utilization of the GPU device
        nvmlUtilization_st device_utilization;
        result = nvmlDeviceGetUtilizationRates(device, & amp;device_utilization);
        if (result != NVML_SUCCESS)
        {<!-- -->
            printf("Failed to get utilization for device %d: %s\\
", i,
                nvmlErrorString(result));
            continue;
        }

        printf("| %d | %s\t| [%s] | %u %% | %u %% | \\
",
               i, device_name, pci.busId, device_utilization.gpu, device_utilization.memory);
        printf("------------------------------------------------ ----------------\\
");

        // Simple example of changing GPU state
        result = nvmlDeviceGetComputeMode(device, & amp;compute_mode);
        if (NVML_ERROR_NOT_SUPPORTED == result)
        {<!-- -->
            printf("\t This is not CUDA capable device\\
");
        }
        else if (NVML_SUCCESS != result)
        {<!-- -->
            printf("Failed to get compute mode for device %u: %s\\
", i, nvmlErrorString(result));
            continue;
        }
        else
        {<!-- -->
            // try to change compute mode
            printf("\t Changing device's compute mode from '%s' to '%s'\\
",
                convertToComputeModeString(compute_mode),
                convertToComputeModeString(NVML_COMPUTEMODE_PROHIBITED));

            result = nvmlDeviceSetComputeMode(device, NVML_COMPUTEMODE_PROHIBITED);
            if (NVML_ERROR_NO_PERMISSION == result)
            {<!-- -->
                printf("\t\t Need root privileges to do that: %s\\
", nvmlErrorString(result));
            }
            else if (NVML_ERROR_NOT_SUPPORTED == result)
            {<!-- -->
                printf("\t\t Compute mode prohibited not supported. You might be running on\\
"
                    "\t\t windows in WDDM driver model or on non-CUDA capable GPU\\
");
            }
            else if (NVML_SUCCESS != result)
            {<!-- -->
                printf("\t\t Failed to set compute mode for device %u: %s\\
", i, nvmlErrorString(result));
                continue;
            }
            else
            {<!-- -->
                printf("\t Restoring device's compute mode back to '%s'\\
",
                    convertToComputeModeString(compute_mode));
                result = nvmlDeviceSetComputeMode(device, compute_mode);
                if (NVML_SUCCESS != result)
                {<!-- -->
                    printf("\t\t Failed to restore compute mode for device %u: %s\\
", i, nvmlErrorString(result));
                    continue;
                }
            }
        }
    }

    // Close NVML
    nvmlShutdown();
    if (NVML_SUCCESS != result)
    {<!-- -->
        printf("Failed to shutdown NVML: %s\\
", nvmlErrorString(result));
        printf("Press ENTER to continue...\\
");
        getchar();
        return (int)(result);
    }

    printf("All done.\\
");
    printf("Press ENTER to continue...\\
");
    getchar();
    return 0;
}

CMakeLists.txt

cmake_minimum_required(VERSION 3.0.0)
set(PROJECT_NAME get_gpu_info)
project(${PROJECT_NAME})

SET(CMAKE_CONFIGURATION_TYPES ${CMAKE_BUILD_TYPE} CACHE STRING "Release" FORCE)

add_executable(${PROJECT_NAME} src/main.cpp)

if(WIN32)
    set(CUDA_ROOT "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.8")
    include_directories("${CUDA_ROOT}/include/")
    target_link_libraries(${PROJECT_NAME} "${CUDA_ROOT}/lib/x64/nvml.lib")
endif()

if(UNIX)
    set(CUDA_ROOT "/usr/local/cuda")
    include_directories("${CUDA_ROOT}/include/")
    link_directories("${CUDA_ROOT}/lib64/stubs")
    target_link_libraries(${PROJECT_NAME} libnvidia-ml.so)
endif()

win10_vs2019_build.bat

::Set the paths of opencv and ncnn in the main CMakeLists.txt
set build_dir=win10_build

::delete compilation directory
rm -rf %build_dir%

::Recreate the compilation directory
mkdir %build_dir%

::Enter the compilation directory
cd %build_dir%

::Configuration, you can use -D to add compilation options here
cmake -G "Visual Studio 16 2019" -A x64 -DCMAKE_BUILD_TYPE=Release ..

::Exit directory
cd..

ubuntu_build.sh

#!/bin/bash

build_dir=ubuntu2204_build

# Delete the compilation directory
rm -rf ${build_dir}

# Recreate directory
mkdir ${build_dir}

# Enter directory
cd ${build_dir}

# Build project
cmake -DCMAKE_BUILD_TYPE=RELEASE ..

# compile
make -j8

# Copy it
cp get_gpu_info ../
cd..

# implement
./get_gpu_info