gpu: support read usage of multiple cards on linux (#79)

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/refraction-networking/utls v1.6.3
 	github.com/shirou/gopsutil/v4 v4.24.9
 	github.com/spf13/viper v1.19.0
+	github.com/tidwall/gjson v1.18.0
 	github.com/urfave/cli/v2 v2.27.5
 	golang.org/x/net v0.29.0
 	golang.org/x/sys v0.25.0
@@ -74,6 +75,8 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/tcnksm/go-gitconfig v0.1.2 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect

go.sum

@@ -167,6 +167,12 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
 github.com/tcnksm/go-gitconfig v0.1.2 h1:iiDhRitByXAEyjgBqsKi9QU4o2TNtv9kPP3RgPgXBPw=
 github.com/tcnksm/go-gitconfig v0.1.2/go.mod h1:/8EhP4H7oJZdIPyT+/UIsG87kTzrzM4UsLGSItWYCpE=
+github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
+github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
 github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
 github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=

model/host.go

@@ -26,7 +26,7 @@ type HostState struct {
 	UdpConnCount   uint64
 	ProcessCount   uint64
 	Temperatures   []SensorTemperature
-	GPU            float64
+	GPU            []float64
 }
 
 func (s *HostState) PB() *pb.State {

pkg/gpu/gpu.go

@@ -1,27 +0,0 @@
-//go:build !darwin
-// +build !darwin
-
-package gpu
-
-import (
-	"errors"
-
-	"github.com/jaypipes/ghw"
-)
-
-func GetGPUModel() ([]string, error) {
-	var gpuModel []string
-	gi, err := ghw.GPU(ghw.WithDisableWarnings())
-	if err != nil {
-		return nil, err
-	}
-
-	for _, card := range gi.GraphicsCards {
-		if card.DeviceInfo == nil {
-			return nil, errors.New("Cannot find device info")
-		}
-		gpuModel = append(gpuModel, card.DeviceInfo.Product.Name)
-	}
-
-	return gpuModel, nil
-}

pkg/gpu/gpu_darwin.go

@@ -125,8 +125,9 @@ func GetGPUModel() ([]string, error) {
 	return util.RemoveDuplicate(models), nil
 }
 
-func FindUtilization(key, dictKey string) (int, error) {
+func GetGPUStat() ([]float64, error) {
-	return findUtilization(key, dictKey)
+	usage, err := findUtilization("PerformanceStatistics", "Device Utilization %")
+	return []float64{float64(usage)}, err
 }
 
 func findDevices(key string) ([]string, error) {

pkg/gpu/gpu_fallback.go

@@ -0,0 +1,11 @@
+//go:build !darwin && !linux && !windows
+
+package gpu
+
+func GetGPUModel() ([]string, error) {
+	return nil, nil
+}
+
+func GetGPUStat() ([]float64, error) {
+	return nil, nil
+}

pkg/gpu/gpu_linux.go

@@ -0,0 +1,125 @@
+//go:build linux
+
+package gpu
+
+import (
+	"errors"
+
+	"github.com/nezhahq/agent/pkg/gpu/vendor"
+)
+
+const (
+	vendorAMD = iota + 1
+	vendorNVIDIA
+)
+
+var vendorType uint8
+
+func init() {
+	_, err := getNvidiaStat()
+	if err != nil {
+		vendorType = vendorAMD
+	} else {
+		vendorType = vendorNVIDIA
+	}
+}
+
+func getNvidiaStat() ([]float64, error) {
+	smi := &vendor.NvidiaSMI{
+		BinPath: "/usr/bin/nvidia-smi",
+	}
+	err1 := smi.Start()
+	if err1 != nil {
+		return nil, err1
+	}
+	data, err2 := smi.GatherUsage()
+	if err2 != nil {
+		return nil, err2
+	}
+	return data, nil
+}
+
+func getAMDStat() ([]float64, error) {
+	rsmi := &vendor.ROCmSMI{
+		BinPath: "/opt/rocm/bin/rocm-smi",
+	}
+	err := rsmi.Start()
+	if err != nil {
+		return nil, err
+	}
+	data, err := rsmi.GatherUsage()
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
+
+func getNvidiaHost() ([]string, error) {
+	smi := &vendor.NvidiaSMI{
+		BinPath: "/usr/bin/nvidia-smi",
+	}
+	err := smi.Start()
+	if err != nil {
+		return nil, err
+	}
+	data, err := smi.GatherModel()
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
+
+func getAMDHost() ([]string, error) {
+	rsmi := &vendor.ROCmSMI{
+		BinPath: "/opt/rocm/bin/rocm-smi",
+	}
+	err := rsmi.Start()
+	if err != nil {
+		return nil, err
+	}
+	data, err := rsmi.GatherModel()
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
+
+func GetGPUModel() ([]string, error) {
+	var gi []string
+	var err error
+
+	switch vendorType {
+	case vendorAMD:
+		gi, err = getAMDHost()
+	case vendorNVIDIA:
+		gi, err = getNvidiaHost()
+	default:
+		return nil, errors.New("invalid vendor")
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	return gi, nil
+}
+
+func GetGPUStat() ([]float64, error) {
+	var gs []float64
+	var err error
+
+	switch vendorType {
+	case vendorAMD:
+		gs, err = getAMDStat()
+	case vendorNVIDIA:
+		gs, err = getNvidiaStat()
+	default:
+		return nil, errors.New("invalid vendor")
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	return gs, nil
+}

pkg/gpu/gpu_windows.go

@@ -1,9 +1,6 @@
 //go:build windows
 
-// Modified from https://github.com/shirou/gopsutil/blob/master/internal/common/common_windows.go
+package gpu
-// Original License: BSD-3-Clause
-
-package stat
 
 import (
 	"errors"
@@ -11,6 +8,7 @@ import (
 	"time"
 	"unsafe"
 
+	"github.com/jaypipes/ghw"
 	"golang.org/x/sys/windows"
 )
 
@@ -43,6 +41,41 @@ type PDH_FMT_COUNTERVALUE_ITEM_DOUBLE struct {
 	FmtValue PDH_FMT_COUNTERVALUE_DOUBLE
 }
 
+func GetGPUModel() ([]string, error) {
+	var gpuModel []string
+	gi, err := ghw.GPU(ghw.WithDisableWarnings())
+	if err != nil {
+		return nil, err
+	}
+
+	for _, card := range gi.GraphicsCards {
+		if card.DeviceInfo == nil {
+			return nil, errors.New("Cannot find device info")
+		}
+		gpuModel = append(gpuModel, card.DeviceInfo.Product.Name)
+	}
+
+	return gpuModel, nil
+}
+
+func GetGPUStat() ([]float64, error) {
+	counter, err := newWin32PerformanceCounter("gpu_utilization", "\\GPU Engine(*engtype_3D)\\Utilization Percentage")
+	if err != nil {
+		return nil, err
+	}
+	defer pdhCloseQuery.Call(uintptr(counter.Query))
+
+	values, err := getValue(8192, counter)
+	if err != nil {
+		return nil, err
+	}
+	tot := sumArray(values)
+	if tot > 100 {
+		tot = 100
+	}
+	return []float64{tot}, nil
+}
+
 // https://github.com/influxdata/telegraf/blob/master/plugins/inputs/win_perf_counters/performance_query.go
 func getCounterArrayValue(initialBufSize uint32, counter *win32PerformanceCounter) ([]float64, error) {
 	for buflen := initialBufSize; buflen <= 100*1024*1024; buflen *= 2 {
@@ -127,24 +160,6 @@ func getValue(initialBufSize uint32, counter *win32PerformanceCounter) ([]float6
 	return getCounterArrayValue(initialBufSize, counter)
 }
 
-func GetGPUStat() (float64, error) {
-	counter, err := newWin32PerformanceCounter("gpu_utilization", "\\GPU Engine(*engtype_3D)\\Utilization Percentage")
-	if err != nil {
-		return 0, err
-	}
-	defer pdhCloseQuery.Call(uintptr(counter.Query))
-
-	values, err := getValue(8192, counter)
-	if err != nil {
-		return 0, err
-	}
-	tot := sumArray(values)
-	if tot > 100 {
-		tot = 100
-	}
-	return tot, nil
-}
-
 func sumArray(arr []float64) float64 {
 	var sum float64
 	for _, value := range arr {

pkg/gpu/stat/amd_rocm_smi.go

@@ -1,67 +0,0 @@
-package stat
-
-// Modified from https://github.com/influxdata/telegraf/blob/master/plugins/inputs/amd_rocm_smi/amd_rocm_smi.go
-// Original License: MIT
-
-import (
-	"errors"
-	"os"
-	"os/exec"
-	"strconv"
-
-	"github.com/nezhahq/agent/pkg/util"
-)
-
-type ROCmSMI struct {
-	BinPath string
-}
-
-func (rsmi *ROCmSMI) Gather() ([]float64, error) {
-	data := rsmi.pollROCmSMI()
-
-	return gatherROCmSMI(data)
-}
-
-func (rsmi *ROCmSMI) Start() error {
-	if _, err := os.Stat(rsmi.BinPath); os.IsNotExist(err) {
-		binPath, err := exec.LookPath("rocm-smi")
-		if err != nil {
-			return errors.New("didn't find the adequate tool to query GPU utilization")
-		}
-		rsmi.BinPath = binPath
-	}
-	return nil
-}
-
-func (rsmi *ROCmSMI) pollROCmSMI() []byte {
-	cmd := exec.Command(rsmi.BinPath,
-		"-u",
-		"--json",
-	)
-	gs, err := cmd.CombinedOutput()
-	if err != nil {
-		return nil
-	}
-	return gs
-}
-
-func gatherROCmSMI(ret []byte) ([]float64, error) {
-	var gpus map[string]GPU
-	var percentage []float64
-
-	err := util.Json.Unmarshal(ret, &gpus)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, gpu := range gpus {
-		gp, _ := strconv.ParseFloat(gpu.GpuUsePercentage, 64)
-		percentage = append(percentage, gp)
-	}
-
-	return percentage, nil
-}
-
-type GPU struct {
-	GpuUsePercentage string `json:"GPU use (%)"`
-}

pkg/gpu/stat/stat_darwin.go

@@ -1,12 +0,0 @@
-//go:build darwin
-
-package stat
-
-import (
-	"github.com/nezhahq/agent/pkg/gpu"
-)
-
-func GetGPUStat() (float64, error) {
-	usage, err := gpu.FindUtilization("PerformanceStatistics", "Device Utilization %")
-	return float64(usage), err
-}

pkg/gpu/stat/stat_freebsd.go

@@ -1,7 +0,0 @@
-//go:build freebsd
-
-package stat
-
-func GetGPUStat() (float64, error) {
-	return 0, nil
-}

pkg/gpu/stat/stat_linux.go

@@ -1,44 +0,0 @@
-//go:build linux
-
-package stat
-
-func getNvidiaStat() ([]float64, error) {
-	smi := &NvidiaSMI{
-		BinPath: "/usr/bin/nvidia-smi",
-	}
-	err1 := smi.Start()
-	if err1 != nil {
-		return nil, err1
-	}
-	data, err2 := smi.Gather()
-	if err2 != nil {
-		return nil, err2
-	}
-	return data, nil
-}
-
-func getAMDStat() ([]float64, error) {
-	rsmi := &ROCmSMI{
-		BinPath: "/opt/rocm/bin/rocm-smi",
-	}
-	err1 := rsmi.Start()
-	if err1 != nil {
-		return nil, err1
-	}
-	data, err2 := rsmi.Gather()
-	if err2 != nil {
-		return nil, err2
-	}
-	return data, nil
-}
-
-func GetGPUStat() (float64, error) {
-	gs, err := getNvidiaStat()
-	if err != nil {
-		gs, err = getAMDStat()
-	}
-	if err != nil || len(gs) == 0 {
-		return 0, err
-	}
-	return gs[0], nil
-}

pkg/gpu/vendor/amd_rocm_smi.go

@@ -0,0 +1,107 @@
+package vendor
+
+// Modified from https://github.com/influxdata/telegraf/blob/master/plugins/inputs/amd_rocm_smi/amd_rocm_smi.go
+// Original License: MIT
+
+import (
+	"errors"
+	"os"
+	"os/exec"
+
+	"github.com/tidwall/gjson"
+)
+
+type ROCmSMI struct {
+	BinPath string
+	data    []byte
+}
+
+func (rsmi *ROCmSMI) GatherModel() ([]string, error) {
+	return rsmi.gatherModel()
+}
+
+func (rsmi *ROCmSMI) GatherUsage() ([]float64, error) {
+	return rsmi.gatherUsage()
+}
+
+func (rsmi *ROCmSMI) Start() error {
+	if _, err := os.Stat(rsmi.BinPath); os.IsNotExist(err) {
+		binPath, err := exec.LookPath("rocm-smi")
+		if err != nil {
+			return errors.New("didn't find the adequate tool to query GPU utilization")
+		}
+		rsmi.BinPath = binPath
+	}
+
+	rsmi.data = rsmi.pollROCmSMI()
+	return nil
+}
+
+func (rsmi *ROCmSMI) pollROCmSMI() []byte {
+	cmd := exec.Command(rsmi.BinPath,
+		"-u",
+		"--showproductname",
+		"--json",
+	)
+	gs, err := cmd.CombinedOutput()
+	if err != nil {
+		return nil
+	}
+	return gs
+}
+
+func (rsmi *ROCmSMI) gatherModel() ([]string, error) {
+	m, err := parseModel(rsmi.data)
+	if err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}
+
+func (rsmi *ROCmSMI) gatherUsage() ([]float64, error) {
+	u, err := parseUsage(rsmi.data)
+	if err != nil {
+		return nil, err
+	}
+
+	return u, nil
+}
+
+func parseModel(jsonObject []byte) ([]string, error) {
+	if jsonObject == nil {
+		return nil, nil
+	}
+
+	result := gjson.ParseBytes(jsonObject)
+	if !result.IsObject() {
+		return nil, errors.New("invalid JSON")
+	}
+
+	ret := make([]string, 0)
+	result.ForEach(func(_, value gjson.Result) bool {
+		ret = append(ret, value.Get("Card series").String())
+		return true
+	})
+
+	return ret, nil
+}
+
+func parseUsage(jsonObject []byte) ([]float64, error) {
+	if jsonObject == nil {
+		return nil, nil
+	}
+
+	result := gjson.ParseBytes(jsonObject)
+	if !result.IsObject() {
+		return nil, errors.New("invalid JSON")
+	}
+
+	ret := make([]float64, 0)
+	result.ForEach(func(_, value gjson.Result) bool {
+		ret = append(ret, value.Get("GPU use (%)").Float())
+		return true
+	})
+
+	return ret, nil
+}

pkg/gpu/vendor/nvidia_smi.go

@@ -1,4 +1,4 @@
-package stat
+package vendor
 
 // Modified from https://github.com/influxdata/telegraf/blob/master/plugins/inputs/nvidia_smi/nvidia_smi.go
 // Original License: MIT
@@ -14,12 +14,15 @@ import (
 
 type NvidiaSMI struct {
 	BinPath string
+	data    []byte
 }
 
-func (smi *NvidiaSMI) Gather() ([]float64, error) {
+func (smi *NvidiaSMI) GatherModel() ([]string, error) {
-	data := smi.pollNvidiaSMI()
+	return smi.gatherModel()
+}
 
-	return smi.parse(data)
+func (smi *NvidiaSMI) GatherUsage() ([]float64, error) {
+	return smi.gatherUsage()
 }
 
 func (smi *NvidiaSMI) Start() error {
@@ -30,6 +33,7 @@ func (smi *NvidiaSMI) Start() error {
 		}
 		smi.BinPath = binPath
 	}
+	smi.data = smi.pollNvidiaSMI()
 	return nil
 }
 
@@ -45,11 +49,27 @@ func (smi *NvidiaSMI) pollNvidiaSMI() []byte {
 	return gs
 }
 
-func (smi *NvidiaSMI) parse(data []byte) ([]float64, error) {
+func (smi *NvidiaSMI) gatherModel() ([]string, error) {
+	var s smistat
+	var models []string
+
+	err := xml.Unmarshal(smi.data, &s)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, gpu := range s.GPUs {
+		models = append(models, gpu.ProductName)
+	}
+
+	return models, nil
+}
+
+func (smi *NvidiaSMI) gatherUsage() ([]float64, error) {
 	var s smistat
 	var percentage []float64
 
-	err := xml.Unmarshal(data, &s)
+	err := xml.Unmarshal(smi.data, &s)
 	if err != nil {
 		return nil, err
 	}
@@ -75,11 +95,12 @@ func parsePercentage(p string) (float64, error) {
 	return value, nil
 }
 
-type nGPU struct {
+type gpu struct {
+	ProductName string `xml:"product_name"`
 	Utilization struct {
 		GpuUtil string `xml:"gpu_util"`
 	} `xml:"utilization"`
 }
 type smistat struct {
-	GPUs []nGPU `xml:"gpu"`
+	GPUs []gpu `xml:"gpu"`
 }

pkg/monitor/monitor.go

@@ -23,7 +23,6 @@ import (
 
 	"github.com/nezhahq/agent/model"
 	"github.com/nezhahq/agent/pkg/gpu"
-	gpustat "github.com/nezhahq/agent/pkg/gpu/stat"
 	"github.com/nezhahq/agent/pkg/util"
 )
 
@@ -349,21 +348,21 @@ func getConns(skipConnectionCount bool) (tcpConnCount, udpConnCount uint64) {
 	return tcpConnCount, udpConnCount
 }
 
-func updateGPUStat() float64 {
+func updateGPUStat() []float64 {
 	if agentConfig.GPU {
 		if statDataFetchAttempts["GPU"] < maxDeviceDataFetchAttempts {
-			gs, err := gpustat.GetGPUStat()
+			gs, err := gpu.GetGPUStat()
 			if err != nil {
 				statDataFetchAttempts["GPU"]++
 				printf("gpustat.GetGPUStat error: %v, attempt: %d", err, statDataFetchAttempts["GPU"])
-				return 0
+				return nil
 			} else {
 				statDataFetchAttempts["GPU"] = 0
 				return gs
 			}
 		}
 	}
-	return 0
+	return nil
 }
 
 func updateTemperatureStat() {

proto/nezha.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.34.2
+// 	protoc-gen-go v1.34.1
-// 	protoc        v5.28.1
+// 	protoc        v5.28.2
 // source: proto/nezha.proto
 
 package proto
@@ -184,7 +184,7 @@ type State struct {
 	UdpConnCount   uint64                     `protobuf:"varint,15,opt,name=udp_conn_count,json=udpConnCount,proto3" json:"udp_conn_count,omitempty"`
 	ProcessCount   uint64                     `protobuf:"varint,16,opt,name=process_count,json=processCount,proto3" json:"process_count,omitempty"`
 	Temperatures   []*State_SensorTemperature `protobuf:"bytes,17,rep,name=temperatures,proto3" json:"temperatures,omitempty"`
-	Gpu            float64                    `protobuf:"fixed64,18,opt,name=gpu,proto3" json:"gpu,omitempty"`
+	Gpu            []float64                  `protobuf:"fixed64,18,rep,packed,name=gpu,proto3" json:"gpu,omitempty"`
 }
 
 func (x *State) Reset() {
@@ -331,11 +331,11 @@ func (x *State) GetTemperatures() []*State_SensorTemperature {
 	return nil
 }
 
-func (x *State) GetGpu() float64 {
+func (x *State) GetGpu() []float64 {
 	if x != nil {
 		return x.Gpu
 	}
-	return 0
+	return nil
 }
 
 type State_SensorTemperature struct {
@@ -746,7 +746,7 @@ var file_proto_nezha_proto_rawDesc = []byte{
 	0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x53, 0x65, 0x6e, 0x73, 0x6f, 0x72,
 	0x54, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x0c, 0x74, 0x65, 0x6d,
 	0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x70, 0x75,
-	0x18, 0x12, 0x20, 0x01, 0x28, 0x01, 0x52, 0x03, 0x67, 0x70, 0x75, 0x22, 0x4f, 0x0a, 0x17, 0x53,
+	0x18, 0x12, 0x20, 0x03, 0x28, 0x01, 0x52, 0x03, 0x67, 0x70, 0x75, 0x22, 0x4f, 0x0a, 0x17, 0x53,
 	0x74, 0x61, 0x74, 0x65, 0x5f, 0x53, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x65,
 	0x72, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01,
 	0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x74, 0x65,
@@ -808,7 +808,7 @@ func file_proto_nezha_proto_rawDescGZIP() []byte {
 }
 
 var file_proto_nezha_proto_msgTypes = make([]protoimpl.MessageInfo, 8)
-var file_proto_nezha_proto_goTypes = []any{
+var file_proto_nezha_proto_goTypes = []interface{}{
 	(*Host)(nil),                    // 0: proto.Host
 	(*State)(nil),                   // 1: proto.State
 	(*State_SensorTemperature)(nil), // 2: proto.State_SensorTemperature
@@ -845,7 +845,7 @@ func file_proto_nezha_proto_init() {
 		return
 	}
 	if !protoimpl.UnsafeEnabled {
-		file_proto_nezha_proto_msgTypes[0].Exporter = func(v any, i int) any {
+		file_proto_nezha_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*Host); i {
 			case 0:
 				return &v.state
@@ -857,7 +857,7 @@ func file_proto_nezha_proto_init() {
 				return nil
 			}
 		}
-		file_proto_nezha_proto_msgTypes[1].Exporter = func(v any, i int) any {
+		file_proto_nezha_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*State); i {
 			case 0:
 				return &v.state
@@ -869,7 +869,7 @@ func file_proto_nezha_proto_init() {
 				return nil
 			}
 		}
-		file_proto_nezha_proto_msgTypes[2].Exporter = func(v any, i int) any {
+		file_proto_nezha_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*State_SensorTemperature); i {
 			case 0:
 				return &v.state
@@ -881,7 +881,7 @@ func file_proto_nezha_proto_init() {
 				return nil
 			}
 		}
-		file_proto_nezha_proto_msgTypes[3].Exporter = func(v any, i int) any {
+		file_proto_nezha_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*Task); i {
 			case 0:
 				return &v.state
@@ -893,7 +893,7 @@ func file_proto_nezha_proto_init() {
 				return nil
 			}
 		}
-		file_proto_nezha_proto_msgTypes[4].Exporter = func(v any, i int) any {
+		file_proto_nezha_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*TaskResult); i {
 			case 0:
 				return &v.state
@@ -905,7 +905,7 @@ func file_proto_nezha_proto_init() {
 				return nil
 			}
 		}
-		file_proto_nezha_proto_msgTypes[5].Exporter = func(v any, i int) any {
+		file_proto_nezha_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*Receipt); i {
 			case 0:
 				return &v.state
@@ -917,7 +917,7 @@ func file_proto_nezha_proto_init() {
 				return nil
 			}
 		}
-		file_proto_nezha_proto_msgTypes[6].Exporter = func(v any, i int) any {
+		file_proto_nezha_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*IOStreamData); i {
 			case 0:
 				return &v.state
@@ -929,7 +929,7 @@ func file_proto_nezha_proto_init() {
 				return nil
 			}
 		}
-		file_proto_nezha_proto_msgTypes[7].Exporter = func(v any, i int) any {
+		file_proto_nezha_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*GeoIP); i {
 			case 0:
 				return &v.state

proto/nezha.proto

@@ -45,7 +45,7 @@ message State {
     uint64 udp_conn_count = 15;
     uint64 process_count = 16;
     repeated State_SensorTemperature temperatures = 17;
-    double gpu = 18;
+    repeated double gpu = 18;
 }
 
 message State_SensorTemperature {

proto/nezha_grpc.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
 // versions:
-// - protoc-gen-go-grpc v1.3.0
+// - protoc-gen-go-grpc v1.5.1
-// - protoc             v5.28.1
+// - protoc             v5.28.2
 // source: proto/nezha.proto
 
 package proto
@@ -15,8 +15,8 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
-// Requires gRPC-Go v1.32.0 or later.
+// Requires gRPC-Go v1.64.0 or later.
-const _ = grpc.SupportPackageIsVersion7
+const _ = grpc.SupportPackageIsVersion9
 
 const (
 	NezhaService_ReportSystemState_FullMethodName = "/proto.NezhaService/ReportSystemState"
@@ -34,8 +34,8 @@ type NezhaServiceClient interface {
 	ReportSystemState(ctx context.Context, in *State, opts ...grpc.CallOption) (*Receipt, error)
 	ReportSystemInfo(ctx context.Context, in *Host, opts ...grpc.CallOption) (*Receipt, error)
 	ReportTask(ctx context.Context, in *TaskResult, opts ...grpc.CallOption) (*Receipt, error)
-	RequestTask(ctx context.Context, in *Host, opts ...grpc.CallOption) (NezhaService_RequestTaskClient, error)
+	RequestTask(ctx context.Context, in *Host, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Task], error)
-	IOStream(ctx context.Context, opts ...grpc.CallOption) (NezhaService_IOStreamClient, error)
+	IOStream(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[IOStreamData, IOStreamData], error)
 	LookupGeoIP(ctx context.Context, in *GeoIP, opts ...grpc.CallOption) (*GeoIP, error)
 }
 
@@ -48,8 +48,9 @@ func NewNezhaServiceClient(cc grpc.ClientConnInterface) NezhaServiceClient {
 }
 
 func (c *nezhaServiceClient) ReportSystemState(ctx context.Context, in *State, opts ...grpc.CallOption) (*Receipt, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
 	out := new(Receipt)
-	err := c.cc.Invoke(ctx, NezhaService_ReportSystemState_FullMethodName, in, out, opts...)
+	err := c.cc.Invoke(ctx, NezhaService_ReportSystemState_FullMethodName, in, out, cOpts...)
 	if err != nil {
 		return nil, err
 	}
@@ -57,8 +58,9 @@ func (c *nezhaServiceClient) ReportSystemState(ctx context.Context, in *State, o
 }
 
 func (c *nezhaServiceClient) ReportSystemInfo(ctx context.Context, in *Host, opts ...grpc.CallOption) (*Receipt, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
 	out := new(Receipt)
-	err := c.cc.Invoke(ctx, NezhaService_ReportSystemInfo_FullMethodName, in, out, opts...)
+	err := c.cc.Invoke(ctx, NezhaService_ReportSystemInfo_FullMethodName, in, out, cOpts...)
 	if err != nil {
 		return nil, err
 	}
@@ -66,20 +68,22 @@ func (c *nezhaServiceClient) ReportSystemInfo(ctx context.Context, in *Host, opt
 }
 
 func (c *nezhaServiceClient) ReportTask(ctx context.Context, in *TaskResult, opts ...grpc.CallOption) (*Receipt, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
 	out := new(Receipt)
-	err := c.cc.Invoke(ctx, NezhaService_ReportTask_FullMethodName, in, out, opts...)
+	err := c.cc.Invoke(ctx, NezhaService_ReportTask_FullMethodName, in, out, cOpts...)
 	if err != nil {
 		return nil, err
 	}
 	return out, nil
 }
 
-func (c *nezhaServiceClient) RequestTask(ctx context.Context, in *Host, opts ...grpc.CallOption) (NezhaService_RequestTaskClient, error) {
+func (c *nezhaServiceClient) RequestTask(ctx context.Context, in *Host, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Task], error) {
-	stream, err := c.cc.NewStream(ctx, &NezhaService_ServiceDesc.Streams[0], NezhaService_RequestTask_FullMethodName, opts...)
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &NezhaService_ServiceDesc.Streams[0], NezhaService_RequestTask_FullMethodName, cOpts...)
 	if err != nil {
 		return nil, err
 	}
-	x := &nezhaServiceRequestTaskClient{stream}
+	x := &grpc.GenericClientStream[Host, Task]{ClientStream: stream}
 	if err := x.ClientStream.SendMsg(in); err != nil {
 		return nil, err
 	}
@@ -89,57 +93,26 @@ func (c *nezhaServiceClient) RequestTask(ctx context.Context, in *Host, opts ...
 	return x, nil
 }
 
-type NezhaService_RequestTaskClient interface {
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
-	Recv() (*Task, error)
+type NezhaService_RequestTaskClient = grpc.ServerStreamingClient[Task]
-	grpc.ClientStream
-}
 
-type nezhaServiceRequestTaskClient struct {
+func (c *nezhaServiceClient) IOStream(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[IOStreamData, IOStreamData], error) {
-	grpc.ClientStream
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
-}
+	stream, err := c.cc.NewStream(ctx, &NezhaService_ServiceDesc.Streams[1], NezhaService_IOStream_FullMethodName, cOpts...)
-
-func (x *nezhaServiceRequestTaskClient) Recv() (*Task, error) {
-	m := new(Task)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *nezhaServiceClient) IOStream(ctx context.Context, opts ...grpc.CallOption) (NezhaService_IOStreamClient, error) {
-	stream, err := c.cc.NewStream(ctx, &NezhaService_ServiceDesc.Streams[1], NezhaService_IOStream_FullMethodName, opts...)
 	if err != nil {
 		return nil, err
 	}
-	x := &nezhaServiceIOStreamClient{stream}
+	x := &grpc.GenericClientStream[IOStreamData, IOStreamData]{ClientStream: stream}
 	return x, nil
 }
 
-type NezhaService_IOStreamClient interface {
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
-	Send(*IOStreamData) error
+type NezhaService_IOStreamClient = grpc.BidiStreamingClient[IOStreamData, IOStreamData]
-	Recv() (*IOStreamData, error)
-	grpc.ClientStream
-}
-
-type nezhaServiceIOStreamClient struct {
-	grpc.ClientStream
-}
-
-func (x *nezhaServiceIOStreamClient) Send(m *IOStreamData) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *nezhaServiceIOStreamClient) Recv() (*IOStreamData, error) {
-	m := new(IOStreamData)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
 
 func (c *nezhaServiceClient) LookupGeoIP(ctx context.Context, in *GeoIP, opts ...grpc.CallOption) (*GeoIP, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
 	out := new(GeoIP)
-	err := c.cc.Invoke(ctx, NezhaService_LookupGeoIP_FullMethodName, in, out, opts...)
+	err := c.cc.Invoke(ctx, NezhaService_LookupGeoIP_FullMethodName, in, out, cOpts...)
 	if err != nil {
 		return nil, err
 	}
@@ -148,19 +121,22 @@ func (c *nezhaServiceClient) LookupGeoIP(ctx context.Context, in *GeoIP, opts ..
 
 // NezhaServiceServer is the server API for NezhaService service.
 // All implementations should embed UnimplementedNezhaServiceServer
-// for forward compatibility
+// for forward compatibility.
 type NezhaServiceServer interface {
 	ReportSystemState(context.Context, *State) (*Receipt, error)
 	ReportSystemInfo(context.Context, *Host) (*Receipt, error)
 	ReportTask(context.Context, *TaskResult) (*Receipt, error)
-	RequestTask(*Host, NezhaService_RequestTaskServer) error
+	RequestTask(*Host, grpc.ServerStreamingServer[Task]) error
-	IOStream(NezhaService_IOStreamServer) error
+	IOStream(grpc.BidiStreamingServer[IOStreamData, IOStreamData]) error
 	LookupGeoIP(context.Context, *GeoIP) (*GeoIP, error)
 }
 
-// UnimplementedNezhaServiceServer should be embedded to have forward compatible implementations.
+// UnimplementedNezhaServiceServer should be embedded to have
-type UnimplementedNezhaServiceServer struct {
+// forward compatible implementations.
-}
+//
+// NOTE: this should be embedded by value instead of pointer to avoid a nil
+// pointer dereference when methods are called.
+type UnimplementedNezhaServiceServer struct{}
 
 func (UnimplementedNezhaServiceServer) ReportSystemState(context.Context, *State) (*Receipt, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method ReportSystemState not implemented")
@@ -171,15 +147,16 @@ func (UnimplementedNezhaServiceServer) ReportSystemInfo(context.Context, *Host)
 func (UnimplementedNezhaServiceServer) ReportTask(context.Context, *TaskResult) (*Receipt, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method ReportTask not implemented")
 }
-func (UnimplementedNezhaServiceServer) RequestTask(*Host, NezhaService_RequestTaskServer) error {
+func (UnimplementedNezhaServiceServer) RequestTask(*Host, grpc.ServerStreamingServer[Task]) error {
 	return status.Errorf(codes.Unimplemented, "method RequestTask not implemented")
 }
-func (UnimplementedNezhaServiceServer) IOStream(NezhaService_IOStreamServer) error {
+func (UnimplementedNezhaServiceServer) IOStream(grpc.BidiStreamingServer[IOStreamData, IOStreamData]) error {
 	return status.Errorf(codes.Unimplemented, "method IOStream not implemented")
 }
 func (UnimplementedNezhaServiceServer) LookupGeoIP(context.Context, *GeoIP) (*GeoIP, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method LookupGeoIP not implemented")
 }
+func (UnimplementedNezhaServiceServer) testEmbeddedByValue() {}
 
 // UnsafeNezhaServiceServer may be embedded to opt out of forward compatibility for this service.
 // Use of this interface is not recommended, as added methods to NezhaServiceServer will
@@ -189,6 +166,13 @@ type UnsafeNezhaServiceServer interface {
 }
 
 func RegisterNezhaServiceServer(s grpc.ServiceRegistrar, srv NezhaServiceServer) {
+	// If the following call panics, it indicates UnimplementedNezhaServiceServer was
+	// embedded by pointer and is nil.  This will cause panics if an
+	// unimplemented method is ever invoked, so we test this at initialization
+	// time to prevent it from happening at runtime later due to I/O.
+	if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
+		t.testEmbeddedByValue()
+	}
 	s.RegisterService(&NezhaService_ServiceDesc, srv)
 }
 
@@ -251,47 +235,18 @@ func _NezhaService_RequestTask_Handler(srv interface{}, stream grpc.ServerStream
 	if err := stream.RecvMsg(m); err != nil {
 		return err
 	}
-	return srv.(NezhaServiceServer).RequestTask(m, &nezhaServiceRequestTaskServer{stream})
+	return srv.(NezhaServiceServer).RequestTask(m, &grpc.GenericServerStream[Host, Task]{ServerStream: stream})
 }
 
-type NezhaService_RequestTaskServer interface {
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
-	Send(*Task) error
+type NezhaService_RequestTaskServer = grpc.ServerStreamingServer[Task]
-	grpc.ServerStream
-}
-
-type nezhaServiceRequestTaskServer struct {
-	grpc.ServerStream
-}
-
-func (x *nezhaServiceRequestTaskServer) Send(m *Task) error {
-	return x.ServerStream.SendMsg(m)
-}
 
 func _NezhaService_IOStream_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(NezhaServiceServer).IOStream(&nezhaServiceIOStreamServer{stream})
+	return srv.(NezhaServiceServer).IOStream(&grpc.GenericServerStream[IOStreamData, IOStreamData]{ServerStream: stream})
 }
 
-type NezhaService_IOStreamServer interface {
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
-	Send(*IOStreamData) error
+type NezhaService_IOStreamServer = grpc.BidiStreamingServer[IOStreamData, IOStreamData]
-	Recv() (*IOStreamData, error)
-	grpc.ServerStream
-}
-
-type nezhaServiceIOStreamServer struct {
-	grpc.ServerStream
-}
-
-func (x *nezhaServiceIOStreamServer) Send(m *IOStreamData) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *nezhaServiceIOStreamServer) Recv() (*IOStreamData, error) {
-	m := new(IOStreamData)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
 
 func _NezhaService_LookupGeoIP_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(GeoIP)