-Fonu/internal/monitor/monitor.go

// Package monitor 周期巡检「资源占用」与「隧道连通性」这两类必须持续观察才会出现的问题。
// 事件型通知（DDNS 失败、证书到期）由各自模块在发生时直接发出，不需要巡检。
//
// 判定在这里，通知只走 notify.Service：开关、阈值、冷却都读设置项，
// 所以本包不需要知道通知通道的任何细节。
package monitor

import (
	"context"
	"fmt"
	"log/slog"
	"strings"
	"sync"
	"time"

	"github.com/fonu/fonu/internal/notify"
	"github.com/fonu/fonu/internal/settings"
	"github.com/fonu/fonu/internal/sysinfo"
)

// RouteStatus 是巡检关心的隧道状态；由调用方注入（避免 monitor 依赖 frp 包）。
type RouteStatus struct {
	Domain   string
	TunnelOK bool
	Detail   string
}

// Checker 每轮都重新读设置项：改阈值或开关不需要重启。
type Checker struct {
	settings *settings.Store
	notify   *notify.Service
	logger   *slog.Logger
	routes   func(ctx context.Context) ([]RouteStatus, error)

	mu             sync.Mutex
	cpuPrev        sysinfo.CPUSample
	resourceAlerts map[string]time.Time
	tunnelFailures map[string]int
}

func New(settingsStore *settings.Store, notifySvc *notify.Service, logger *slog.Logger, routes func(context.Context) ([]RouteStatus, error)) *Checker {
	if logger == nil {
		logger = slog.Default()
	}
	return &Checker{
		settings:       settingsStore,
		notify:         notifySvc,
		logger:         logger,
		routes:         routes,
		resourceAlerts: map[string]time.Time{},
		tunnelFailures: map[string]int{},
	}
}

// Run 跑一轮巡检。scheduler 已对任务包了 recover，这里不重复兜底。
func (c *Checker) Run(ctx context.Context) {
	if c == nil || c.settings == nil {
		return
	}
	c.checkResources(ctx)
	c.checkTunnels(ctx)
}

func (c *Checker) checkResources(ctx context.Context) {
	// 开关关闭时连采集都省掉（关掉的功能不该有后台开销）
	if enabled, _ := c.settings.GetBool(ctx, settings.KeyNotifyOnResourceThreshold); !enabled {
		return
	}
	cpuLimit, _ := c.settings.GetInt(ctx, settings.KeyNotifyCPUThreshold)
	memoryLimit, _ := c.settings.GetInt(ctx, settings.KeyNotifyMemoryThreshold)
	diskLimit, _ := c.settings.GetInt(ctx, settings.KeyNotifyDiskThreshold)

	c.mu.Lock()
	stats, sample := sysinfo.Read(c.cpuPrev)
	c.cpuPrev = sample
	c.mu.Unlock()

	limits := []struct {
		metric string
		label  string
		limit  int
		value  float64
	}{
		{"cpu", "CPU", cpuLimit, stats.CPUPercent},
		{"memory", "内存", memoryLimit, percent(stats.MemUsedBytes, stats.MemTotalBytes)},
		{"disk", "磁盘", diskLimit, percent(stats.DiskUsedBytes, stats.DiskTotalBytes)},
	}
	now := time.Now()
	for _, item := range limits {
		// 阈值 <= 0 表示这一项不告警（CPU 首次采样时占用率为 0，也不会误报）
		if item.limit <= 0 {
			continue
		}
		if item.value < float64(item.limit) {
			// 回落到阈值以下就清冷却：再次超标可以立刻提醒
			c.mu.Lock()
			delete(c.resourceAlerts, item.metric)
			c.mu.Unlock()
			continue
		}
		if !c.allowResourceAlert(item.metric, now) {
			continue
		}
		c.notify.Alert(ctx, notify.EventResourceThreshold, "资源占用过高",
			fmt.Sprintf("%s 使用率 %.1f%%，已超过阈值 %d%%", item.label, item.value, item.limit))
	}
}

// allowResourceAlert 按静默期限流：持续高占用每 DefaultResourceAlertCooldown 提醒一次。
func (c *Checker) allowResourceAlert(metric string, now time.Time) bool {
	cooldown := time.Duration(notify.DefaultResourceAlertCooldown) * time.Second
	c.mu.Lock()
	defer c.mu.Unlock()
	if last, ok := c.resourceAlerts[metric]; ok && now.Sub(last) < cooldown {
		return false
	}
	c.resourceAlerts[metric] = now
	return true
}

func (c *Checker) checkTunnels(ctx context.Context) {
	if c.routes == nil {
		return
	}
	if enabled, _ := c.settings.GetBool(ctx, settings.KeyNotifyOnTunnelDown); !enabled {
		return
	}
	limit, _ := c.settings.GetInt(ctx, settings.KeyNotifyTunnelFailThreshold)
	if limit <= 0 {
		return
	}

	routes, err := c.routes(ctx)
	if err != nil {
		c.logger.Warn("巡检隧道状态失败", "module", "MONITOR", "error", err.Error())
		return
	}
	for _, route := range routes {
		if strings.TrimSpace(route.Domain) == "" {
			continue
		}
		c.mu.Lock()
		failures, event := tunnelTransition(c.tunnelFailures[route.Domain], route.TunnelOK, limit)
		c.tunnelFailures[route.Domain] = failures
		c.mu.Unlock()

		switch event {
		case tunnelEventDown:
			c.notify.Alert(ctx, notify.EventTunnelDown, "隧道连续不通",
				fmt.Sprintf("%s 连续 %d 次巡检未在 frps 上注册：%s", route.Domain, limit, detailOr(route.Detail)))
		case tunnelEventRecovered:
			c.notify.Alert(ctx, notify.EventTunnelDown, "隧道已恢复",
				fmt.Sprintf("%s 已重新在 frps 上注册", route.Domain))
		}
	}
}

type tunnelEvent string

const (
	tunnelEventNone      tunnelEvent = ""
	tunnelEventDown      tunnelEvent = "down"
	tunnelEventRecovered tunnelEvent = "recovered"
)

// tunnelTransition 推进某个域名的连续失败计数，返回新计数与本轮要发的事件。
// 只在「刚好达到阈值」与「从失败中恢复」两个时刻发，避免每轮巡检都刷通知。
func tunnelTransition(previous int, tunnelOK bool, limit int) (int, tunnelEvent) {
	if tunnelOK {
		if previous >= limit {
			return 0, tunnelEventRecovered
		}
		return 0, tunnelEventNone
	}
	next := previous + 1
	if next == limit {
		return next, tunnelEventDown
	}
	return next, tunnelEventNone
}

func percent(used, total uint64) float64 {
	if total == 0 {
		return 0
	}
	return float64(used) / float64(total) * 100
}

func detailOr(detail string) string {
	if strings.TrimSpace(detail) == "" {
		return "未返回具体原因"
	}
	return detail
}