306 lines
10 KiB
Go
306 lines
10 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"net/netip"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"gorm.io/gorm"
|
|
"gorm.io/gorm/clause"
|
|
|
|
"oci-portal/internal/model"
|
|
)
|
|
|
|
// 告警规则约束与命中记录保留期(窗口计数之外多留几天便于排查)。
|
|
const (
|
|
alertMaxThreshold = 100
|
|
alertMaxWindowMin = 1440
|
|
alertHitRetention = 7 * 24 * time.Hour
|
|
alertSourceIPIn = "in"
|
|
alertSourceIPNotIn = "notin"
|
|
)
|
|
|
|
// ErrInvalidAlertRule 标记规则字段非法,api 层映射 400。
|
|
var ErrInvalidAlertRule = fmt.Errorf("告警规则字段非法")
|
|
|
|
// ListAlertRules 返回全部告警规则(创建顺序)。
|
|
func (s *LogEventService) ListAlertRules(ctx context.Context) ([]model.AlertRule, error) {
|
|
var rules []model.AlertRule
|
|
if err := s.db.WithContext(ctx).Order("id").Find(&rules).Error; err != nil {
|
|
return nil, fmt.Errorf("list alert rules: %w", err)
|
|
}
|
|
return rules, nil
|
|
}
|
|
|
|
// CreateAlertRule 校验并创建规则。
|
|
func (s *LogEventService) CreateAlertRule(ctx context.Context, rule model.AlertRule) (model.AlertRule, error) {
|
|
if err := validateAlertRule(&rule); err != nil {
|
|
return model.AlertRule{}, err
|
|
}
|
|
rule.ID = 0
|
|
if err := s.db.WithContext(ctx).Create(&rule).Error; err != nil {
|
|
return model.AlertRule{}, fmt.Errorf("create alert rule: %w", err)
|
|
}
|
|
return rule, nil
|
|
}
|
|
|
|
// UpdateAlertRule 校验并整体覆盖规则(含启停)。
|
|
func (s *LogEventService) UpdateAlertRule(ctx context.Context, id uint, rule model.AlertRule) (model.AlertRule, error) {
|
|
if err := validateAlertRule(&rule); err != nil {
|
|
return model.AlertRule{}, err
|
|
}
|
|
var cur model.AlertRule
|
|
if err := s.db.WithContext(ctx).First(&cur, id).Error; err != nil {
|
|
return model.AlertRule{}, fmt.Errorf("find alert rule %d: %w", id, err)
|
|
}
|
|
rule.ID, rule.CreatedAt = cur.ID, cur.CreatedAt
|
|
if err := s.db.WithContext(ctx).Save(&rule).Error; err != nil {
|
|
return model.AlertRule{}, fmt.Errorf("update alert rule: %w", err)
|
|
}
|
|
return rule, nil
|
|
}
|
|
|
|
// DeleteAlertRule 删除规则及其命中记录。
|
|
func (s *LogEventService) DeleteAlertRule(ctx context.Context, id uint) error {
|
|
if err := s.db.WithContext(ctx).Delete(&model.AlertRule{}, id).Error; err != nil {
|
|
return fmt.Errorf("delete alert rule: %w", err)
|
|
}
|
|
if err := s.db.WithContext(ctx).Where("rule_id = ?", id).Delete(&model.AlertRuleHit{}).Error; err != nil {
|
|
return fmt.Errorf("delete alert rule hits: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// validateAlertRule 校验字段并归一化;非法时返回含具体原因的 ErrInvalidAlertRule 包装。
|
|
func validateAlertRule(rule *model.AlertRule) error {
|
|
rule.Name = strings.TrimSpace(rule.Name)
|
|
if rule.Name == "" {
|
|
return fmt.Errorf("%w: 名称必填", ErrInvalidAlertRule)
|
|
}
|
|
if rule.SourceIPMode == "" {
|
|
rule.SourceIPMode = alertSourceIPIn
|
|
}
|
|
if rule.SourceIPMode != alertSourceIPIn && rule.SourceIPMode != alertSourceIPNotIn {
|
|
return fmt.Errorf("%w: 来源 IP 模式须为 in/notin", ErrInvalidAlertRule)
|
|
}
|
|
if rule.Threshold < 1 || rule.Threshold > alertMaxThreshold {
|
|
return fmt.Errorf("%w: 阈值须在 1-%d 之间", ErrInvalidAlertRule, alertMaxThreshold)
|
|
}
|
|
if rule.Threshold > 1 && (rule.WindowMinutes < 1 || rule.WindowMinutes > alertMaxWindowMin) {
|
|
return fmt.Errorf("%w: 阈值>1 时窗口须在 1-%d 分钟之间", ErrInvalidAlertRule, alertMaxWindowMin)
|
|
}
|
|
if rule.EventTypes != "" {
|
|
rule.EventTypes = normalizeCSV(rule.EventTypes)
|
|
}
|
|
return validateAlertRuleIPs(rule)
|
|
}
|
|
|
|
// validateAlertRuleIPs 归一化并校验来源 IP 列表(裸 IP 或 CIDR)。
|
|
func validateAlertRuleIPs(rule *model.AlertRule) error {
|
|
if rule.SourceIPs == "" {
|
|
return nil
|
|
}
|
|
rule.SourceIPs = normalizeCSV(rule.SourceIPs)
|
|
for _, item := range strings.Split(rule.SourceIPs, ",") {
|
|
if _, err := parseIPMatcher(item); err != nil {
|
|
return fmt.Errorf("%w: 来源 IP %q 不是合法的 IP 或 CIDR", ErrInvalidAlertRule, item)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// normalizeCSV 去除各项空白与空项后重组逗号分隔串。
|
|
func normalizeCSV(s string) string {
|
|
parts := strings.Split(s, ",")
|
|
out := parts[:0]
|
|
for _, p := range parts {
|
|
if p = strings.TrimSpace(p); p != "" {
|
|
out = append(out, p)
|
|
}
|
|
}
|
|
return strings.Join(out, ",")
|
|
}
|
|
|
|
// parseIPMatcher 把裸 IP 或 CIDR 解析为前缀(裸 IP 视为单地址前缀)。
|
|
func parseIPMatcher(item string) (netip.Prefix, error) {
|
|
if strings.Contains(item, "/") {
|
|
return netip.ParsePrefix(item)
|
|
}
|
|
addr, err := netip.ParseAddr(item)
|
|
if err != nil {
|
|
return netip.Prefix{}, err
|
|
}
|
|
return netip.PrefixFrom(addr, addr.BitLen()), nil
|
|
}
|
|
|
|
// ipListMatch 报告 ip 是否命中列表中的任一前缀;ip 解析失败视为未命中。
|
|
func ipListMatch(list, ip string) bool {
|
|
addr, err := netip.ParseAddr(ip)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
for _, item := range strings.Split(list, ",") {
|
|
if p, err := parseIPMatcher(item); err == nil && p.Contains(addr) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// ruleHits 报告事件是否命中规则的全部条件(AND 语义,空条件视为任意)。
|
|
func ruleHits(rule model.AlertRule, e *model.LogEvent, p parsedEvent) bool {
|
|
if rule.OciConfigID != 0 && rule.OciConfigID != e.OciConfigID {
|
|
return false
|
|
}
|
|
name := relayEventShortName(p.EventType)
|
|
if rule.EventTypes != "" && !slices.Contains(strings.Split(rule.EventTypes, ","), name) {
|
|
return false
|
|
}
|
|
if rule.ResourceMatch != "" && !strings.Contains(p.ResourceName, rule.ResourceMatch) {
|
|
return false
|
|
}
|
|
return ruleIPHits(rule, p.SourceIP)
|
|
}
|
|
|
|
// ruleIPHits 按模式判定来源 IP 条件:in 命中列表告警;notin 不在列表才告警,
|
|
// 事件缺 IP 字段时 notin 不告警(避免解析缺字段导致白名单误报)。
|
|
func ruleIPHits(rule model.AlertRule, ip string) bool {
|
|
if rule.SourceIPs == "" {
|
|
return true
|
|
}
|
|
if rule.SourceIPMode == alertSourceIPNotIn {
|
|
return ip != "" && !ipListMatch(rule.SourceIPs, ip)
|
|
}
|
|
return ipListMatch(rule.SourceIPs, ip)
|
|
}
|
|
|
|
// matchAlertRules 对一条已解析事件执行全部启用规则;任何内部错误只记日志,不影响解析主流程。
|
|
func (s *LogEventService) matchAlertRules(ctx context.Context, rules []model.AlertRule, e *model.LogEvent, p parsedEvent) {
|
|
if s.notifier == nil {
|
|
return
|
|
}
|
|
for _, rule := range rules {
|
|
if !rule.Enabled || !ruleHits(rule, e, p) {
|
|
continue
|
|
}
|
|
count, ok := s.recordAlertHit(ctx, rule, e)
|
|
if !ok || count < rule.Threshold || !s.alertCooldownPass(rule) {
|
|
continue
|
|
}
|
|
s.notifier.SendTemplateAsync("audit_alert", map[string]string{
|
|
"rule": rule.Name, "tenant": s.configAlias(ctx, e.OciConfigID),
|
|
"event": relayEventShortName(p.EventType), "resource": p.ResourceName,
|
|
"ip": p.SourceIP, "count": fmt.Sprint(count),
|
|
})
|
|
}
|
|
}
|
|
|
|
// recordAlertHit 落一条命中并返回窗口内累计次数;阈值 1 的规则免计数直接触发。
|
|
func (s *LogEventService) recordAlertHit(ctx context.Context, rule model.AlertRule, e *model.LogEvent) (int, bool) {
|
|
count, err := s.recordAlertHitTx(ctx, rule, e)
|
|
if err != nil {
|
|
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
|
log.Printf("alert rule hit record: %v", err)
|
|
}
|
|
return 0, false
|
|
}
|
|
return count, true
|
|
}
|
|
|
|
func (s *LogEventService) recordAlertHitTx(ctx context.Context, rule model.AlertRule, event *model.LogEvent) (int, error) {
|
|
count := 0
|
|
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
|
var err error
|
|
count, err = insertAlertHit(tx, rule, event)
|
|
return err
|
|
})
|
|
return count, err
|
|
}
|
|
|
|
// insertAlertHit 按 rule→event 锁顺序确认引用存在后插入并统计窗口命中。
|
|
func insertAlertHit(tx *gorm.DB, rule model.AlertRule, event *model.LogEvent) (int, error) {
|
|
if err := lockAlertRefs(tx, rule.ID, event.ID); err != nil {
|
|
return 0, err
|
|
}
|
|
if rule.Threshold <= 1 {
|
|
return 1, nil
|
|
}
|
|
now := time.Now()
|
|
hit := model.AlertRuleHit{RuleID: rule.ID, LogEventID: event.ID, HitAt: now}
|
|
if err := tx.Create(&hit).Error; err != nil {
|
|
return 0, fmt.Errorf("create alert rule hit: %w", err)
|
|
}
|
|
var count int64
|
|
cutoff := now.Add(-time.Duration(rule.WindowMinutes) * time.Minute)
|
|
err := tx.Model(&model.AlertRuleHit{}).
|
|
Where("rule_id = ? AND hit_at >= ?", rule.ID, cutoff).Count(&count).Error
|
|
if err != nil {
|
|
return 0, fmt.Errorf("count alert rule hits: %w", err)
|
|
}
|
|
return int(count), nil
|
|
}
|
|
|
|
func lockAlertRefs(tx *gorm.DB, ruleID, eventID uint) error {
|
|
var rule model.AlertRule
|
|
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Select("id").First(&rule, ruleID).Error; err != nil {
|
|
return fmt.Errorf("lock alert rule %d: %w", ruleID, err)
|
|
}
|
|
var event model.LogEvent
|
|
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Select("id").First(&event, eventID).Error; err != nil {
|
|
return fmt.Errorf("lock log event %d: %w", eventID, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// alertCooldownPass 报告规则是否已过冷却窗口;通过即记录本次发送时刻。
|
|
// 阈值 1 的规则无冷却(每次命中即时告警,与既有云端事件通知一致)。
|
|
func (s *LogEventService) alertCooldownPass(rule model.AlertRule) bool {
|
|
if rule.Threshold <= 1 {
|
|
return true
|
|
}
|
|
s.alertMu.Lock()
|
|
defer s.alertMu.Unlock()
|
|
window := time.Duration(rule.WindowMinutes) * time.Minute
|
|
if last, ok := s.alertSentAt[rule.ID]; ok && time.Since(last) < window {
|
|
return false
|
|
}
|
|
if s.alertSentAt == nil {
|
|
s.alertSentAt = map[uint]time.Time{}
|
|
}
|
|
s.alertSentAt[rule.ID] = time.Now()
|
|
return true
|
|
}
|
|
|
|
// ClearAlertCooldown 清除已删除租户规则的进程内冷却状态。
|
|
func (s *LogEventService) ClearAlertCooldown(ruleIDs []uint) {
|
|
s.alertMu.Lock()
|
|
defer s.alertMu.Unlock()
|
|
for _, id := range ruleIDs {
|
|
delete(s.alertSentAt, id)
|
|
}
|
|
}
|
|
|
|
// loadEnabledAlertRules 载入启用中的规则;失败时返回空集并记日志(解析主流程照常)。
|
|
func (s *LogEventService) loadEnabledAlertRules(ctx context.Context) []model.AlertRule {
|
|
var rules []model.AlertRule
|
|
err := s.db.WithContext(ctx).Where("enabled = ?", true).Order("id").Find(&rules).Error
|
|
if err != nil {
|
|
log.Printf("load alert rules: %v", err)
|
|
return nil
|
|
}
|
|
return rules
|
|
}
|
|
|
|
// cleanupAlertHits 删除保留期外的命中记录(随 cleanupOnce 周期执行)。
|
|
func (s *LogEventService) cleanupAlertHits(ctx context.Context) {
|
|
cutoff := time.Now().Add(-alertHitRetention)
|
|
if err := s.db.WithContext(ctx).Where("hit_at < ?", cutoff).Delete(&model.AlertRuleHit{}).Error; err != nil {
|
|
log.Printf("cleanup alert hits: %v", err)
|
|
}
|
|
}
|