发布 0.1.0:通知渠道、告警规则、令牌版本与安全加固
This commit is contained in:
Wang Defa
+240
-12
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -22,6 +23,13 @@ const taskRunTimeout = 10 * time.Minute
|
||||
// taskLogKeep 是每个任务保留的执行日志条数。
|
||||
const taskLogKeep = 100
|
||||
|
||||
// defaultSnatchIPWait 是抢机成功后等待公网 IPv4 分配的总预算(全部实例共享,
|
||||
// 实例从创建到 VNIC 就绪通常 1~2 分钟);taskRunTimeout 余量内,超时降级不阻塞通知。
|
||||
const defaultSnatchIPWait = 3 * time.Minute
|
||||
|
||||
// snatchIPPollInterval 是等待公网 IP 的轮询间隔。
|
||||
const snatchIPPollInterval = 10 * time.Second
|
||||
|
||||
// TaskService 管理后台任务的存储、cron 调度与执行。
|
||||
type TaskService struct {
|
||||
db *gorm.DB
|
||||
@@ -31,21 +39,32 @@ type TaskService struct {
|
||||
cron *cron.Cron
|
||||
// aiGateway 供 AI 探测任务执行渠道探测,由 main 装配(可为 nil)
|
||||
aiGateway *AiGatewayService
|
||||
// snatchIPWait 是抢机成功后等待公网 IP 的预算,测试置 0 跳过等待
|
||||
snatchIPWait time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
entries map[uint]cron.EntryID
|
||||
// runMu 让租户删除等待在途执行结束,并阻止新执行读取删除前的任务快照。
|
||||
runMu sync.RWMutex
|
||||
|
||||
// snatchVarsMu 保护 snatchVars:runSnatch 抢满时写入成功通知变量,
|
||||
// execute 组装执行后快照时取走(一次性)
|
||||
snatchVarsMu sync.Mutex
|
||||
snatchVars map[uint]map[string]string
|
||||
}
|
||||
|
||||
// NewTaskService 组装依赖；notifier 传 nil 表示整体关闭通知，
|
||||
// settings 供发送前按事件类型过滤（nil 视为全开）。调用 Start 后开始调度。
|
||||
func NewTaskService(db *gorm.DB, configs *OciConfigService, notifier *Notifier, settings *SettingService) *TaskService {
|
||||
return &TaskService{
|
||||
db: db,
|
||||
configs: configs,
|
||||
notifier: notifier,
|
||||
settings: settings,
|
||||
cron: cron.New(),
|
||||
entries: map[uint]cron.EntryID{},
|
||||
db: db,
|
||||
configs: configs,
|
||||
notifier: notifier,
|
||||
settings: settings,
|
||||
cron: cron.New(),
|
||||
entries: map[uint]cron.EntryID{},
|
||||
snatchIPWait: defaultSnatchIPWait,
|
||||
snatchVars: map[uint]map[string]string{},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -371,9 +390,37 @@ func (s *TaskService) unschedule(taskID uint) {
|
||||
}
|
||||
}
|
||||
|
||||
// ApplyTenantCleanup 在租户删除事务提交后注销已删除任务并对齐 AI 探测任务。
|
||||
func (s *TaskService) ApplyTenantCleanup(ctx context.Context, taskIDs []uint, channelsGone bool) {
|
||||
for _, id := range taskIDs {
|
||||
s.unschedule(id)
|
||||
s.forgetSnatchVars(id)
|
||||
}
|
||||
if channelsGone {
|
||||
s.SyncAiProbeTask(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TaskService) forgetSnatchVars(taskID uint) {
|
||||
s.snatchVarsMu.Lock()
|
||||
delete(s.snatchVars, taskID)
|
||||
s.snatchVarsMu.Unlock()
|
||||
}
|
||||
|
||||
func (s *TaskService) lockTenantCleanup() func() {
|
||||
s.runMu.Lock()
|
||||
return s.runMu.Unlock
|
||||
}
|
||||
|
||||
// execute 执行一次任务：加载 → 分派 → 更新任务状态并写日志，
|
||||
// 前后状态交给通知判定，只在状态变化时推送。
|
||||
func (s *TaskService) execute(taskID uint) *model.TaskLog {
|
||||
s.runMu.RLock()
|
||||
defer s.runMu.RUnlock()
|
||||
return s.executeLocked(taskID)
|
||||
}
|
||||
|
||||
func (s *TaskService) executeLocked(taskID uint) *model.TaskLog {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), taskRunTimeout)
|
||||
defer cancel()
|
||||
task, err := s.GetTask(ctx, taskID)
|
||||
@@ -391,12 +438,40 @@ func (s *TaskService) execute(taskID uint) *model.TaskLog {
|
||||
task.LastError = oci.CompactError(runErr)
|
||||
message = task.LastError
|
||||
}
|
||||
s.db.Save(task)
|
||||
cur := taskSnapshot{Name: task.Name, Status: task.Status, LastError: task.LastError, Message: message}
|
||||
if !s.storeTaskRun(ctx, task) {
|
||||
return nil
|
||||
}
|
||||
cur := taskSnapshot{Name: task.Name, Status: task.Status, LastError: task.LastError,
|
||||
Message: message, SnatchVars: s.takeSnatchVars(task.ID)}
|
||||
s.notify(notifyEvents(prev, cur))
|
||||
return s.appendLog(task.ID, runErr == nil, message, time.Since(start))
|
||||
}
|
||||
|
||||
func (s *TaskService) storeTaskRun(ctx context.Context, task *model.Task) bool {
|
||||
stored, err := s.persistTaskRun(ctx, task)
|
||||
if err != nil {
|
||||
log.Printf("persist task run: %v", err)
|
||||
return false
|
||||
}
|
||||
return stored
|
||||
}
|
||||
|
||||
func (s *TaskService) persistTaskRun(ctx context.Context, task *model.Task) (bool, error) {
|
||||
updates := map[string]any{
|
||||
"status": task.Status, "last_run_at": task.LastRunAt,
|
||||
"last_error": task.LastError, "run_count": task.RunCount,
|
||||
}
|
||||
if task.Type == model.TaskTypeSnatch {
|
||||
updates["payload"] = task.Payload
|
||||
}
|
||||
res := s.db.WithContext(ctx).Model(&model.Task{}).
|
||||
Where("id = ? AND updated_at = ?", task.ID, task.UpdatedAt).Updates(updates)
|
||||
if res.Error != nil {
|
||||
return false, fmt.Errorf("persist task %d run: %w", task.ID, res.Error)
|
||||
}
|
||||
return res.RowsAffected == 1, nil
|
||||
}
|
||||
|
||||
// run 按类型分派任务执行。
|
||||
func (s *TaskService) run(ctx context.Context, task *model.Task) (string, error) {
|
||||
switch task.Type {
|
||||
@@ -650,12 +725,158 @@ func (s *TaskService) runSnatch(ctx context.Context, task *model.Task) (string,
|
||||
writeSnatchPayload(task, &p)
|
||||
return fmt.Sprintf("created %d (%s)%s, %d remaining", len(instances), strings.Join(ids, ","), adNote, remaining), nil
|
||||
}
|
||||
task.Status = model.TaskStatusSucceeded
|
||||
writeSnatchPayload(task, &p)
|
||||
s.unschedule(task.ID)
|
||||
s.snatchSucceed(ctx, task, &p, in, instances)
|
||||
return fmt.Sprintf("created %d: %s%s", len(instances), strings.Join(ids, ","), adNote), nil
|
||||
}
|
||||
|
||||
// snatchSucceed 处理抢满收尾:标记成功、停止调度,并组装成功通知变量暂存,
|
||||
// 供 execute 发通知时合并进 snatch_success 事件。
|
||||
func (s *TaskService) snatchSucceed(ctx context.Context, task *model.Task, p *snatchPayload, in oci.CreateInstanceInput, instances []oci.Instance) {
|
||||
task.Status = model.TaskStatusSucceeded
|
||||
writeSnatchPayload(task, p)
|
||||
s.unschedule(task.ID)
|
||||
s.storeSnatchVars(task.ID, s.snatchSuccessVars(ctx, p, in, instances))
|
||||
}
|
||||
|
||||
// snatchSuccessVars 组装抢机成功通知的模板变量;单项查询失败以占位值回退,
|
||||
// 绝不阻塞通知发送。密码只进通知变量,不落任务日志。
|
||||
func (s *TaskService) snatchSuccessVars(ctx context.Context, p *snatchPayload, in oci.CreateInstanceInput, instances []oci.Instance) map[string]string {
|
||||
tenant, region := "-", in.Region
|
||||
if cfg, err := s.configs.Get(ctx, p.OciConfigID); err == nil {
|
||||
tenant = cfg.Alias
|
||||
if region == "" {
|
||||
region = cfg.Region
|
||||
}
|
||||
}
|
||||
ips, pwds := s.snatchInstanceNet(ctx, p.OciConfigID, region, in, instances)
|
||||
return map[string]string{
|
||||
"tenant": tenant,
|
||||
"region": regionLabel(region),
|
||||
"shape": in.Shape,
|
||||
"spec": specLabel(in),
|
||||
"image": s.snatchImageLabel(ctx, p.OciConfigID, region, in),
|
||||
"ip": ips,
|
||||
"root_password": pwds,
|
||||
}
|
||||
}
|
||||
|
||||
// snatchInstanceNet 逐台等待公网 IPv4 并取回 root 密码;全部实例共享
|
||||
// snatchIPWait 总预算,多台结果按创建顺序以「、」连接。
|
||||
func (s *TaskService) snatchInstanceNet(ctx context.Context, cfgID uint, region string, in oci.CreateInstanceInput, instances []oci.Instance) (string, string) {
|
||||
deadline := time.Now().Add(s.snatchIPWait)
|
||||
ips := make([]string, 0, len(instances))
|
||||
pwds := make([]string, 0, len(instances))
|
||||
for _, inst := range instances {
|
||||
ips = append(ips, s.waitPublicIPv4(ctx, cfgID, region, inst.ID, deadline))
|
||||
pwds = append(pwds, rootPasswordOf(in, inst))
|
||||
}
|
||||
return strings.Join(ips, "、"), strings.Join(pwds, "、")
|
||||
}
|
||||
|
||||
// waitPublicIPv4 轮询实例公网 IPv4 直至分配或超过 deadline;
|
||||
// 超时回退内网 IPv4(标注内网),再无则「待分配」。
|
||||
func (s *TaskService) waitPublicIPv4(ctx context.Context, cfgID uint, region, instanceID string, deadline time.Time) string {
|
||||
private := ""
|
||||
for {
|
||||
inst, err := s.configs.Instance(ctx, cfgID, region, instanceID)
|
||||
if err == nil {
|
||||
if inst.PublicIP != "" {
|
||||
return inst.PublicIP
|
||||
}
|
||||
if inst.PrivateIP != "" {
|
||||
private = inst.PrivateIP
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return waitIPFallback(private)
|
||||
case <-time.After(snatchIPPollInterval):
|
||||
}
|
||||
}
|
||||
return waitIPFallback(private)
|
||||
}
|
||||
|
||||
// waitIPFallback 是公网 IP 等待超时后的展示回退。
|
||||
func waitIPFallback(private string) string {
|
||||
if private != "" {
|
||||
return private + "(内网)"
|
||||
}
|
||||
return "待分配"
|
||||
}
|
||||
|
||||
// rootPasswordOf 取单台实例的 root 密码:随机生成模式从创建时写入的
|
||||
// FreeformTags 回读(每台独立),显式密码模式各台相同;SSH 密钥模式无密码。
|
||||
func rootPasswordOf(in oci.CreateInstanceInput, inst oci.Instance) string {
|
||||
if pwd := inst.FreeformTags["RootPassword"]; pwd != "" {
|
||||
return pwd
|
||||
}
|
||||
if in.RootPassword != "" {
|
||||
return in.RootPassword
|
||||
}
|
||||
return "—(SSH 密钥登录)"
|
||||
}
|
||||
|
||||
// specLabel 拼 Flex 规格,如 4C/24G;固定规格 shape(无 ocpus 参数)为 -。
|
||||
func specLabel(in oci.CreateInstanceInput) string {
|
||||
if in.Ocpus <= 0 {
|
||||
return "-"
|
||||
}
|
||||
return fmt.Sprintf("%gC/%gG", in.Ocpus, in.MemoryInGBs)
|
||||
}
|
||||
|
||||
// regionLabel 把区域标识换成控制台风格别名展示,如「Japan Central (Osaka)」;
|
||||
// 区域名与三字码均可匹配,表中未收录时回退原值。
|
||||
func regionLabel(region string) string {
|
||||
if region == "" {
|
||||
return "-"
|
||||
}
|
||||
regions, err := oci.AllRegions()
|
||||
if err != nil {
|
||||
return region
|
||||
}
|
||||
for _, r := range regions {
|
||||
if strings.EqualFold(r.Name, region) || strings.EqualFold(r.Key, region) {
|
||||
return r.Alias
|
||||
}
|
||||
}
|
||||
return region
|
||||
}
|
||||
|
||||
// snatchImageLabel 返回启动源展示名:镜像查显示名(失败回退 OCID),
|
||||
// 引导卷启动源固定文案。
|
||||
func (s *TaskService) snatchImageLabel(ctx context.Context, cfgID uint, region string, in oci.CreateInstanceInput) string {
|
||||
if in.BootVolumeID != "" {
|
||||
return "引导卷"
|
||||
}
|
||||
if in.ImageID == "" {
|
||||
return "-"
|
||||
}
|
||||
img, err := s.configs.Image(ctx, cfgID, region, in.ImageID)
|
||||
if err != nil || img.DisplayName == "" {
|
||||
return in.ImageID
|
||||
}
|
||||
return img.DisplayName
|
||||
}
|
||||
|
||||
// storeSnatchVars 暂存抢机成功通知变量,由 takeSnatchVars 取走。
|
||||
func (s *TaskService) storeSnatchVars(taskID uint, vars map[string]string) {
|
||||
s.snatchVarsMu.Lock()
|
||||
defer s.snatchVarsMu.Unlock()
|
||||
s.snatchVars[taskID] = vars
|
||||
}
|
||||
|
||||
// takeSnatchVars 取走并清除暂存的成功通知变量(无则 nil)。
|
||||
func (s *TaskService) takeSnatchVars(taskID uint) map[string]string {
|
||||
s.snatchVarsMu.Lock()
|
||||
defer s.snatchVarsMu.Unlock()
|
||||
vars := s.snatchVars[taskID]
|
||||
delete(s.snatchVars, taskID)
|
||||
return vars
|
||||
}
|
||||
|
||||
// snatchInstanceInput 组装本次创建参数:可用域显式指定时原样使用;
|
||||
// 留空(自动)时按执行序号轮询区域全部可用域——ad-1、ad-2、ad-3 依次循环,
|
||||
// 分摊单可用域容量不足。附加说明串供执行日志展示本次所用可用域。
|
||||
@@ -735,6 +956,9 @@ type taskSnapshot struct {
|
||||
Status string
|
||||
LastError string
|
||||
Message string // 本次执行结果摘要，仅执行后快照填写
|
||||
// SnatchVars 是抢机成功通知的补充变量(租户/区域/实例明细),
|
||||
// 仅抢满那次执行后快照携带
|
||||
SnatchVars map[string]string
|
||||
}
|
||||
|
||||
// notifyKind 是通知事件类型，与设置页「通知管理」开关一一对应。
|
||||
@@ -760,7 +984,11 @@ type notifyEvent struct {
|
||||
func notifyEvents(prev, cur taskSnapshot) []notifyEvent {
|
||||
var events []notifyEvent
|
||||
if prev.Status != model.TaskStatusSucceeded && cur.Status == model.TaskStatusSucceeded {
|
||||
events = append(events, notifyEvent{notifySnatchSuccess, map[string]string{"task_name": cur.Name, "message": cur.Message}})
|
||||
vars := map[string]string{"task_name": cur.Name, "message": cur.Message}
|
||||
for k, v := range cur.SnatchVars {
|
||||
vars[k] = v
|
||||
}
|
||||
events = append(events, notifyEvent{notifySnatchSuccess, vars})
|
||||
}
|
||||
switch {
|
||||
case prev.Status != model.TaskStatusFailed && cur.Status == model.TaskStatusFailed:
|
||||
|
||||
Reference in New Issue
Block a user