本文目录导读：
我来介绍几种监控GPU使用率的脚本方法：
使用 nvidia-smi 的基础监控脚本
Linux Bash 脚本
#!/bin/bash
# gpu_monitor.sh
while true; do
# 显示GPU使用率、温度、显存使用
nvidia-smi --query-gpu=index,utilization.gpu,utilization.memory,temperature.gpu,memory.used,memory.total --format=csv,noheader
# 或使用更简洁的输出
nvidia-smi stats -d gpuUtil
sleep 2
clear
done
实时监控（每秒刷新）
#!/bin/bash watch -n 1 nvidia-smi --query-gpu=index,utilization.gpu,utilization.memory,temperature.gpu --format=csv,noheader
Python 脚本
基础监控类
#!/usr/bin/env python3
# gpu_monitor.py
import subprocess
import json
import time
from datetime import datetime
def get_gpu_info():
"""获取GPU信息"""
try:
# 执行 nvidia-smi 命令
result = subprocess.run([
'nvidia-smi',
'--query-gpu=index,name,utilization.gpu,utilization.memory,memory.used,memory.total,temperature.gpu',
'--format=csv,noheader,nounits'
], capture_output=True, text=True, check=True)
gpu_info = []
for line in result.stdout.strip().split('\n'):
if line:
parts = [x.strip() for x in line.split(',')]
gpu_info.append({
'index': parts[0],
'name': parts[1],
'gpu_util': float(parts[2]),
'memory_util': float(parts[3]),
'memory_used': parts[4],
'memory_total': parts[5],
'temperature': parts[6]
})
return gpu_info
except (subprocess.CalledProcessError, FileNotFoundError):
print("Error: nvidia-smi not found or failed")
return []
def monitor_gpu(interval=2):
"""持续监控GPU"""
print(f"Starting GPU monitoring (interval: {interval}s)")
print("-" * 80)
while True:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
gpus = get_gpu_info()
if gpus:
print(f"\n[{timestamp}]")
for gpu in gpus:
print(f"GPU {gpu['index']} ({gpu['name']}):")
print(f" GPU使用率: {gpu['gpu_util']}%")
print(f" 显存使用率: {gpu['memory_util']}%")
print(f" 显存: {gpu['memory_used']}/{gpu['memory_total']} MB")
print(f" 温度: {gpu['temperature']}°C")
else:
print(f"[{timestamp}] No GPU information available")
time.sleep(interval)
if __name__ == "__main__":
try:
monitor_gpu()
except KeyboardInterrupt:
print("\nMonitoring stopped.")
带告警的监控脚本
#!/usr/bin/env python3
# gpu_monitor_alarm.py
import subprocess
import time
import smtplib
from email.mime.text import MIMEText
class GPUMonitor:
def __init__(self, threshold=90, interval=5):
self.threshold = threshold # GPU使用率阈值
self.interval = interval # 检查间隔
self.alerted = {} # 记录已告警的GPU
def get_gpu_usage(self):
"""获取GPU使用率"""
try:
result = subprocess.run([
'nvidia-smi',
'--query-gpu=index,utilization.gpu',
'--format=csv,noheader,nounits'
], capture_output=True, text=True, check=True)
usage = {}
for line in result.stdout.strip().split('\n'):
if line:
idx, util = line.strip().split(', ')
usage[int(idx)] = float(util)
return usage
except Exception as e:
print(f"Error getting GPU info: {e}")
return {}
def check_and_alert(self, gpu_id, usage):
"""检查是否需要告警"""
if usage > self.threshold and not self.alerted.get(gpu_id):
print(f"ALERT: GPU {gpu_id} usage at {usage}% (threshold: {self.threshold}%)")
# 这里可以添加发送邮件或消息的代码
self.alerted[gpu_id] = True
elif usage <= self.threshold and self.alerted.get(gpu_id):
print(f"INFO: GPU {gpu_id} usage back to normal at {usage}%")
self.alerted[gpu_id] = False
def run(self):
print(f"GPU Monitor Started")
print(f"Threshold: {self.threshold}%")
print(f"Check interval: {self.interval}s")
print("-" * 50)
while True:
usage = self.get_gpu_usage()
if usage:
for gpu_id, util in usage.items():
status = "⚠️" if util > self.threshold else "✓"
print(f"GPU {gpu_id}: {util}% {status}")
self.check_and_alert(gpu_id, util)
else:
print("No GPU information available")
time.sleep(self.interval)
if __name__ == "__main__":
monitor = GPUMonitor(threshold=90, interval=5)
try:
monitor.run()
except KeyboardInterrupt:
print("\nMonitoring stopped.")
日志记录版本
#!/usr/bin/env python3
# gpu_monitor_log.py
import subprocess
import csv
import time
from datetime import datetime
def log_gpu_usage(log_file="gpu_usage.csv", interval=60):
"""记录GPU使用率到CSV文件"""
# 创建CSV文件并写入表头
with open(log_file, 'w', newline='') as f:
writer = csv.writer(f)
writer.writerow(['Timestamp', 'GPU_ID', 'GPU_Util%', 'Memory_Util%', 'Temperature°C'])
print(f"Logging GPU usage to {log_file} (interval: {interval}s)")
while True:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
try:
result = subprocess.run([
'nvidia-smi',
'--query-gpu=index,utilization.gpu,utilization.memory,temperature.gpu',
'--format=csv,noheader,nounits'
], capture_output=True, text=True, check=True)
with open(log_file, 'a', newline='') as f:
writer = csv.writer(f)
for line in result.stdout.strip().split('\n'):
if line:
data = [timestamp] + [x.strip() for x in line.split(',')]
writer.writerow(data)
print(f"[{timestamp}] Data logged")
except (subprocess.CalledProcessError, FileNotFoundError) as e:
print(f"[{timestamp}] Error: {e}")
time.sleep(interval)
if __name__ == "__main__":
try:
log_gpu_usage("gpu_usage.csv", 60) # 每分钟记录一次
except KeyboardInterrupt:
print("\nLogging stopped.")
Web监控（Flask版本）
from flask import Flask, jsonify
import subprocess
import json
app = Flask(__name__)
@app.route('/gpu')
def get_gpu_info():
try:
result = subprocess.run([
'nvidia-smi',
'--query-gpu=index,name,utilization.gpu,utilization.memory,memory.used,memory.total,temperature.gpu',
'--format=csv,noheader,nounits'
], capture_output=True, text=True, check=True)
gpu_data = []
for line in result.stdout.strip().split('\n'):
if line:
parts = [x.strip() for x in line.split(',')]
gpu_data.append({
'index': int(parts[0]),
'name': parts[1],
'gpu_util': float(parts[2]),
'memory_util': float(parts[3]),
'memory_used': parts[4],
'memory_total': parts[5],
'temperature': parts[6]
})
return jsonify({
'status': 'success',
'data': gpu_data,
'count': len(gpu_data)
})
except Exception as e:
return jsonify({
'status': 'error',
'message': str(e)
}), 500
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5000, debug=True)
使用说明
-
安装依赖：
# Python脚本需要 pip install flask # Web监控需要
-
运行脚本：
# 基础监控 python3 gpu_monitor.py
带告警的监控
python3 gpu_monitor_alarm.py
日志记录
python3 gpu_monitor_log.py
3. **设置定时任务**（cron）：
```bash
# 每5分钟执行一次监控脚本
*/5 * * * * /path/to/gpu_monitor.sh >> /var/log/gpu_monitor.log 2>&1这些脚本可以根据你的具体需求进行修改和扩展。
