- 配置版本：fsutil 版本化备份（内容相同自动去重），主 nginx.conf 与 VPS vhost 的版本列表/逐行差异/一键回滚，回滚先校验语法与 nginx -t 再写回 - agent 0.12.2：写 vhost 前留版本；nginx -t 校验失败改为恢复上一个可用版本（此前直接删文件），新增域名 vhost 版本端点，双架构二进制已重编 - 通知：补企业微信 / 钉钉 / 飞书机器人，切换预设时服务地址跟随该预设默认值 - 阈值告警：新增 internal/monitor 巡检（CPU/内存/磁盘持续超阈值、隧道连续 N 次未注册），默认关闭，带静默期与恢复通知 - API Token：api_tokens 表 + Authorization Bearer 认证，只读/读写两种范围；令牌管理与改密/退出仅限登录会话
This commit is contained in:
@@ -0,0 +1,194 @@
|
||||
// Package monitor 周期巡检「资源占用」与「隧道连通性」这两类必须持续观察才会出现的问题。
|
||||
// 事件型通知（DDNS 失败、证书到期）由各自模块在发生时直接发出，不需要巡检。
|
||||
//
|
||||
// 判定在这里，通知只走 notify.Service：开关、阈值、冷却都读设置项，
|
||||
// 所以本包不需要知道通知通道的任何细节。
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/fonu/fonu/internal/notify"
|
||||
"github.com/fonu/fonu/internal/settings"
|
||||
"github.com/fonu/fonu/internal/sysinfo"
|
||||
)
|
||||
|
||||
// RouteStatus 是巡检关心的隧道状态；由调用方注入（避免 monitor 依赖 frp 包）。
|
||||
type RouteStatus struct {
|
||||
Domain string
|
||||
TunnelOK bool
|
||||
Detail string
|
||||
}
|
||||
|
||||
// Checker 每轮都重新读设置项：改阈值或开关不需要重启。
|
||||
type Checker struct {
|
||||
settings *settings.Store
|
||||
notify *notify.Service
|
||||
logger *slog.Logger
|
||||
routes func(ctx context.Context) ([]RouteStatus, error)
|
||||
|
||||
mu sync.Mutex
|
||||
cpuPrev sysinfo.CPUSample
|
||||
resourceAlerts map[string]time.Time
|
||||
tunnelFailures map[string]int
|
||||
}
|
||||
|
||||
func New(settingsStore *settings.Store, notifySvc *notify.Service, logger *slog.Logger, routes func(context.Context) ([]RouteStatus, error)) *Checker {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Checker{
|
||||
settings: settingsStore,
|
||||
notify: notifySvc,
|
||||
logger: logger,
|
||||
routes: routes,
|
||||
resourceAlerts: map[string]time.Time{},
|
||||
tunnelFailures: map[string]int{},
|
||||
}
|
||||
}
|
||||
|
||||
// Run 跑一轮巡检。scheduler 已对任务包了 recover，这里不重复兜底。
|
||||
func (c *Checker) Run(ctx context.Context) {
|
||||
if c == nil || c.settings == nil {
|
||||
return
|
||||
}
|
||||
c.checkResources(ctx)
|
||||
c.checkTunnels(ctx)
|
||||
}
|
||||
|
||||
func (c *Checker) checkResources(ctx context.Context) {
|
||||
// 开关关闭时连采集都省掉（关掉的功能不该有后台开销）
|
||||
if enabled, _ := c.settings.GetBool(ctx, settings.KeyNotifyOnResourceThreshold); !enabled {
|
||||
return
|
||||
}
|
||||
cpuLimit, _ := c.settings.GetInt(ctx, settings.KeyNotifyCPUThreshold)
|
||||
memoryLimit, _ := c.settings.GetInt(ctx, settings.KeyNotifyMemoryThreshold)
|
||||
diskLimit, _ := c.settings.GetInt(ctx, settings.KeyNotifyDiskThreshold)
|
||||
|
||||
c.mu.Lock()
|
||||
stats, sample := sysinfo.Read(c.cpuPrev)
|
||||
c.cpuPrev = sample
|
||||
c.mu.Unlock()
|
||||
|
||||
limits := []struct {
|
||||
metric string
|
||||
label string
|
||||
limit int
|
||||
value float64
|
||||
}{
|
||||
{"cpu", "CPU", cpuLimit, stats.CPUPercent},
|
||||
{"memory", "内存", memoryLimit, percent(stats.MemUsedBytes, stats.MemTotalBytes)},
|
||||
{"disk", "磁盘", diskLimit, percent(stats.DiskUsedBytes, stats.DiskTotalBytes)},
|
||||
}
|
||||
now := time.Now()
|
||||
for _, item := range limits {
|
||||
// 阈值 <= 0 表示这一项不告警（CPU 首次采样时占用率为 0，也不会误报）
|
||||
if item.limit <= 0 {
|
||||
continue
|
||||
}
|
||||
if item.value < float64(item.limit) {
|
||||
// 回落到阈值以下就清冷却：再次超标可以立刻提醒
|
||||
c.mu.Lock()
|
||||
delete(c.resourceAlerts, item.metric)
|
||||
c.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
if !c.allowResourceAlert(item.metric, now) {
|
||||
continue
|
||||
}
|
||||
c.notify.Alert(ctx, notify.EventResourceThreshold, "资源占用过高",
|
||||
fmt.Sprintf("%s 使用率 %.1f%%，已超过阈值 %d%%", item.label, item.value, item.limit))
|
||||
}
|
||||
}
|
||||
|
||||
// allowResourceAlert 按静默期限流：持续高占用每 DefaultResourceAlertCooldown 提醒一次。
|
||||
func (c *Checker) allowResourceAlert(metric string, now time.Time) bool {
|
||||
cooldown := time.Duration(notify.DefaultResourceAlertCooldown) * time.Second
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if last, ok := c.resourceAlerts[metric]; ok && now.Sub(last) < cooldown {
|
||||
return false
|
||||
}
|
||||
c.resourceAlerts[metric] = now
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *Checker) checkTunnels(ctx context.Context) {
|
||||
if c.routes == nil {
|
||||
return
|
||||
}
|
||||
if enabled, _ := c.settings.GetBool(ctx, settings.KeyNotifyOnTunnelDown); !enabled {
|
||||
return
|
||||
}
|
||||
limit, _ := c.settings.GetInt(ctx, settings.KeyNotifyTunnelFailThreshold)
|
||||
if limit <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
routes, err := c.routes(ctx)
|
||||
if err != nil {
|
||||
c.logger.Warn("巡检隧道状态失败", "module", "MONITOR", "error", err.Error())
|
||||
return
|
||||
}
|
||||
for _, route := range routes {
|
||||
if strings.TrimSpace(route.Domain) == "" {
|
||||
continue
|
||||
}
|
||||
c.mu.Lock()
|
||||
failures, event := tunnelTransition(c.tunnelFailures[route.Domain], route.TunnelOK, limit)
|
||||
c.tunnelFailures[route.Domain] = failures
|
||||
c.mu.Unlock()
|
||||
|
||||
switch event {
|
||||
case tunnelEventDown:
|
||||
c.notify.Alert(ctx, notify.EventTunnelDown, "隧道连续不通",
|
||||
fmt.Sprintf("%s 连续 %d 次巡检未在 frps 上注册：%s", route.Domain, limit, detailOr(route.Detail)))
|
||||
case tunnelEventRecovered:
|
||||
c.notify.Alert(ctx, notify.EventTunnelDown, "隧道已恢复",
|
||||
fmt.Sprintf("%s 已重新在 frps 上注册", route.Domain))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type tunnelEvent string
|
||||
|
||||
const (
|
||||
tunnelEventNone tunnelEvent = ""
|
||||
tunnelEventDown tunnelEvent = "down"
|
||||
tunnelEventRecovered tunnelEvent = "recovered"
|
||||
)
|
||||
|
||||
// tunnelTransition 推进某个域名的连续失败计数，返回新计数与本轮要发的事件。
|
||||
// 只在「刚好达到阈值」与「从失败中恢复」两个时刻发，避免每轮巡检都刷通知。
|
||||
func tunnelTransition(previous int, tunnelOK bool, limit int) (int, tunnelEvent) {
|
||||
if tunnelOK {
|
||||
if previous >= limit {
|
||||
return 0, tunnelEventRecovered
|
||||
}
|
||||
return 0, tunnelEventNone
|
||||
}
|
||||
next := previous + 1
|
||||
if next == limit {
|
||||
return next, tunnelEventDown
|
||||
}
|
||||
return next, tunnelEventNone
|
||||
}
|
||||
|
||||
func percent(used, total uint64) float64 {
|
||||
if total == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(used) / float64(total) * 100
|
||||
}
|
||||
|
||||
func detailOr(detail string) string {
|
||||
if strings.TrimSpace(detail) == "" {
|
||||
return "未返回具体原因"
|
||||
}
|
||||
return detail
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package monitor
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestTunnelTransitionAlertsOncePerOutage(t *testing.T) {
|
||||
const limit = 3
|
||||
failures := 0
|
||||
var events []tunnelEvent
|
||||
|
||||
// 连续三轮不通：只在第 3 轮（刚好达到阈值）发一次
|
||||
for i := 0; i < 5; i++ {
|
||||
var event tunnelEvent
|
||||
failures, event = tunnelTransition(failures, false, limit)
|
||||
if event != tunnelEventNone {
|
||||
events = append(events, event)
|
||||
}
|
||||
}
|
||||
if len(events) != 1 || events[0] != tunnelEventDown {
|
||||
t.Fatalf("连续不通应只告警一次，实际 %v", events)
|
||||
}
|
||||
if failures != 5 {
|
||||
t.Fatalf("失败计数应继续累加，实际 %d", failures)
|
||||
}
|
||||
|
||||
// 恢复：发一次恢复通知并清零
|
||||
failures, event := tunnelTransition(failures, true, limit)
|
||||
if event != tunnelEventRecovered || failures != 0 {
|
||||
t.Fatalf("恢复应通知一次并清零：event=%v failures=%d", event, failures)
|
||||
}
|
||||
|
||||
// 恢复后再次不通：还能再报一次
|
||||
failures, event = tunnelTransition(failures, false, limit)
|
||||
if event != tunnelEventNone || failures != 1 {
|
||||
t.Fatalf("恢复后重新计数：event=%v failures=%d", event, failures)
|
||||
}
|
||||
failures, event = tunnelTransition(failures, false, limit)
|
||||
if event != tunnelEventNone {
|
||||
t.Fatalf("未达阈值不应告警：%v", event)
|
||||
}
|
||||
if _, event = tunnelTransition(failures, false, limit); event != tunnelEventDown {
|
||||
t.Fatalf("第二次中断达到阈值应再告警一次：%v", event)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTunnelTransitionBelowThresholdIsSilent(t *testing.T) {
|
||||
// 单次抖动（未达阈值）不打扰用户，也不算作恢复
|
||||
if failures, event := tunnelTransition(0, true, 3); failures != 0 || event != tunnelEventNone {
|
||||
t.Fatalf("健康状态不应产生事件：%d %v", failures, event)
|
||||
}
|
||||
if failures, event := tunnelTransition(1, true, 3); failures != 0 || event != tunnelEventNone {
|
||||
t.Fatalf("未达阈值的恢复不应通知：%d %v", failures, event)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPercent(t *testing.T) {
|
||||
if got := percent(0, 0); got != 0 {
|
||||
t.Fatalf("总数为 0 时应返回 0，实际 %v", got)
|
||||
}
|
||||
if got := percent(50, 200); got != 25 {
|
||||
t.Fatalf("期望 25，实际 %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetailOrFallsBack(t *testing.T) {
|
||||
if got := detailOr(" "); got != "未返回具体原因" {
|
||||
t.Fatalf("空详情应给出兜底文案，实际 %q", got)
|
||||
}
|
||||
if got := detailOr("dashboard 超时"); got != "dashboard 超时" {
|
||||
t.Fatalf("非空详情应原样返回，实际 %q", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user