702 lines
24 KiB
Go
702 lines
24 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"github.com/glebarez/sqlite"
|
|
"gorm.io/gorm"
|
|
"gorm.io/gorm/logger"
|
|
|
|
"oci-portal/internal/crypto"
|
|
"oci-portal/internal/model"
|
|
"oci-portal/internal/oci"
|
|
)
|
|
|
|
// snatchFake 可控 LaunchInstance 结果：前 failFirst 次失败模拟容量不足。
|
|
type snatchFake struct {
|
|
fakeClient
|
|
launches int
|
|
failFirst int
|
|
}
|
|
|
|
func (f *snatchFake) LaunchInstance(ctx context.Context, cred oci.Credentials, in oci.CreateInstanceInput) (oci.Instance, error) {
|
|
f.launches++
|
|
if f.launches <= f.failFirst {
|
|
return oci.Instance{}, fmt.Errorf("Out of host capacity")
|
|
}
|
|
return oci.Instance{ID: fmt.Sprintf("ocid1.instance.%d", f.launches), DisplayName: in.DisplayName}, nil
|
|
}
|
|
|
|
// scriptedSnatchFake 按脚本逐次控制 LaunchInstance 结果：nil 表示成功；
|
|
// 脚本耗尽后一律成功。
|
|
type scriptedSnatchFake struct {
|
|
fakeClient
|
|
mu sync.Mutex
|
|
script []error
|
|
launches int
|
|
}
|
|
|
|
func (f *scriptedSnatchFake) LaunchInstance(ctx context.Context, cred oci.Credentials, in oci.CreateInstanceInput) (oci.Instance, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.launches++
|
|
if len(f.script) > 0 {
|
|
err := f.script[0]
|
|
f.script = f.script[1:]
|
|
if err != nil {
|
|
return oci.Instance{}, err
|
|
}
|
|
}
|
|
return oci.Instance{ID: fmt.Sprintf("ocid1.instance.%d", f.launches), DisplayName: in.DisplayName}, nil
|
|
}
|
|
|
|
func newTaskEnv(t *testing.T, client oci.Client) (*TaskService, *OciConfigService, *gorm.DB) {
|
|
t.Helper()
|
|
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
|
|
Logger: logger.Default.LogMode(logger.Silent),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("open in-memory sqlite: %v", err)
|
|
}
|
|
// :memory: 库每个连接彼此独立，锁单连接避免池内新连接拿到空库
|
|
sqlDB, err := db.DB()
|
|
if err != nil {
|
|
t.Fatalf("db handle: %v", err)
|
|
}
|
|
sqlDB.SetMaxOpenConns(1)
|
|
if err := db.AutoMigrate(
|
|
&model.OciConfig{}, &model.Task{}, &model.TaskLog{},
|
|
&model.CheckSnapshot{}, &model.CostSnapshot{}, &model.Setting{}, &model.Proxy{},
|
|
); err != nil {
|
|
t.Fatalf("auto migrate: %v", err)
|
|
}
|
|
cipher, err := crypto.NewCipher("test-data-key")
|
|
if err != nil {
|
|
t.Fatalf("new cipher: %v", err)
|
|
}
|
|
configs := NewOciConfigService(db, cipher, client)
|
|
return NewTaskService(db, configs, nil, NewSettingService(db, cipher)), configs, db
|
|
}
|
|
|
|
func TestCreateTaskValidation(t *testing.T) {
|
|
tasks, _, _ := newTaskEnv(t, &fakeClient{})
|
|
tests := []struct {
|
|
name string
|
|
in CreateTaskInput
|
|
}{
|
|
{name: "非法 cron", in: CreateTaskInput{Name: "t", Type: model.TaskTypeHealthCheck, CronExpr: "not-cron"}},
|
|
{name: "未知类型", in: CreateTaskInput{Name: "t", Type: "nope", CronExpr: "* * * * *"}},
|
|
{name: "缺名称", in: CreateTaskInput{Type: model.TaskTypeHealthCheck, CronExpr: "* * * * *"}},
|
|
{
|
|
name: "抢机缺配置ID",
|
|
in: CreateTaskInput{Name: "t", Type: model.TaskTypeSnatch, CronExpr: "* * * * *",
|
|
Payload: json.RawMessage(`{"instance":{"displayName":"a","shape":"s","imageId":"i"}}`)},
|
|
},
|
|
{
|
|
name: "抢机实例参数不全",
|
|
in: CreateTaskInput{Name: "t", Type: model.TaskTypeSnatch, CronExpr: "* * * * *",
|
|
Payload: json.RawMessage(`{"ociConfigId":1,"instance":{"shape":"s"}}`)},
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
if _, err := tasks.CreateTask(context.Background(), tt.in); err == nil {
|
|
t.Error("CreateTask succeeded, want error")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunHealthCheckTask(t *testing.T) {
|
|
client := &fakeClient{tenancy: oci.TenancyInfo{Name: "acme", HomeRegionKey: "FRA"}}
|
|
tasks, configs, _ := newTaskEnv(t, client)
|
|
ctx := context.Background()
|
|
if _, err := configs.Import(ctx, trialImportInput()); err != nil {
|
|
t.Fatalf("import: %v", err)
|
|
}
|
|
second := trialImportInput()
|
|
second.Alias = "第二个"
|
|
if _, err := configs.Import(ctx, second); err != nil {
|
|
t.Fatalf("import second: %v", err)
|
|
}
|
|
|
|
task, err := tasks.CreateTask(ctx, CreateTaskInput{
|
|
Name: "测活", Type: model.TaskTypeHealthCheck, CronExpr: "*/5 * * * *",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("CreateTask: %v", err)
|
|
}
|
|
entry, err := tasks.RunTaskNow(ctx, task.ID)
|
|
if err != nil {
|
|
t.Fatalf("RunTaskNow: %v", err)
|
|
}
|
|
if !entry.Success || entry.Message != "checked 2: 2 alive, 0 dead" {
|
|
t.Errorf("log = %+v, want success checked 2", entry)
|
|
}
|
|
|
|
scoped, err := tasks.CreateTask(ctx, CreateTaskInput{
|
|
Name: "范围测活", Type: model.TaskTypeHealthCheck, CronExpr: "*/5 * * * *",
|
|
Payload: json.RawMessage(`{"ociConfigIds":[1]}`),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("CreateTask scoped: %v", err)
|
|
}
|
|
entry, err = tasks.RunTaskNow(ctx, scoped.ID)
|
|
if err != nil {
|
|
t.Fatalf("RunTaskNow scoped: %v", err)
|
|
}
|
|
if entry.Message != "checked 1: 1 alive, 0 dead" {
|
|
t.Errorf("scoped message = %q, want checked 1", entry.Message)
|
|
}
|
|
}
|
|
|
|
func TestRunSnatchTaskUntilSuccess(t *testing.T) {
|
|
client := &snatchFake{failFirst: 2}
|
|
client.tenancy = oci.TenancyInfo{Name: "acme", HomeRegionKey: "FRA"}
|
|
tasks, configs, _ := newTaskEnv(t, client)
|
|
ctx := context.Background()
|
|
if _, err := configs.Import(ctx, trialImportInput()); err != nil {
|
|
t.Fatalf("import: %v", err)
|
|
}
|
|
task, err := tasks.CreateTask(ctx, CreateTaskInput{
|
|
Name: "抢机", Type: model.TaskTypeSnatch, CronExpr: "* * * * *",
|
|
Payload: json.RawMessage(`{"ociConfigId":1,"instance":{"displayName":"snatch-vm","shape":"VM.Standard.A1.Flex","ocpus":4,"memoryInGBs":24,"imageId":"ocid1.image.test"}}`),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("CreateTask: %v", err)
|
|
}
|
|
|
|
// 前两次容量不足：任务保持 active 并记录失败
|
|
for i := 1; i <= 2; i++ {
|
|
entry, err := tasks.RunTaskNow(ctx, task.ID)
|
|
if err != nil {
|
|
t.Fatalf("run %d: %v", i, err)
|
|
}
|
|
if entry.Success {
|
|
t.Fatalf("run %d succeeded, want capacity failure", i)
|
|
}
|
|
}
|
|
got, _ := tasks.GetTask(ctx, task.ID)
|
|
if got.Status != model.TaskStatusActive || got.RunCount != 2 || got.LastError == "" {
|
|
t.Errorf("after failures: status=%s runs=%d lastError=%q", got.Status, got.RunCount, got.LastError)
|
|
}
|
|
|
|
// 第三次成功：任务自动标记 succeeded
|
|
entry, err := tasks.RunTaskNow(ctx, task.ID)
|
|
if err != nil {
|
|
t.Fatalf("run 3: %v", err)
|
|
}
|
|
if !entry.Success {
|
|
t.Fatalf("run 3 = %+v, want success", entry)
|
|
}
|
|
got, _ = tasks.GetTask(ctx, task.ID)
|
|
if got.Status != model.TaskStatusSucceeded || got.LastError != "" {
|
|
t.Errorf("after success: status=%s lastError=%q, want succeeded", got.Status, got.LastError)
|
|
}
|
|
|
|
logs, err := tasks.TaskLogs(ctx, task.ID, 10)
|
|
if err != nil {
|
|
t.Fatalf("TaskLogs: %v", err)
|
|
}
|
|
if len(logs) != 3 || !logs[0].Success {
|
|
t.Errorf("logs = %d entries, newest success=%v; want 3 with newest success", len(logs), logs[0].Success)
|
|
}
|
|
}
|
|
|
|
// adRotationFake 记录每次 LaunchInstance 收到的可用域;创建一律按容量不足
|
|
// 失败,任务保持 active 以便连续执行验证轮询顺序。
|
|
type adRotationFake struct {
|
|
fakeClient
|
|
ads []string
|
|
got []string
|
|
}
|
|
|
|
func (f *adRotationFake) ListAvailabilityDomains(ctx context.Context, cred oci.Credentials, region string) ([]string, error) {
|
|
return f.ads, nil
|
|
}
|
|
|
|
func (f *adRotationFake) LaunchInstance(ctx context.Context, cred oci.Credentials, in oci.CreateInstanceInput) (oci.Instance, error) {
|
|
f.got = append(f.got, in.AvailabilityDomain)
|
|
return oci.Instance{}, fmt.Errorf("Out of host capacity")
|
|
}
|
|
|
|
// TestSnatchAdRotation 可用域留空(自动)时按执行序号轮询区域全部可用域,
|
|
// 显式指定时不轮询。
|
|
func TestSnatchAdRotation(t *testing.T) {
|
|
client := &adRotationFake{ads: []string{"ad-1", "ad-2", "ad-3"}}
|
|
client.tenancy = oci.TenancyInfo{Name: "acme", HomeRegionKey: "FRA"}
|
|
tasks, configs, _ := newTaskEnv(t, client)
|
|
ctx := context.Background()
|
|
if _, err := configs.Import(ctx, trialImportInput()); err != nil {
|
|
t.Fatalf("import: %v", err)
|
|
}
|
|
task, err := tasks.CreateTask(ctx, CreateTaskInput{
|
|
Name: "抢机", Type: model.TaskTypeSnatch, CronExpr: "* * * * *",
|
|
Payload: json.RawMessage(`{"ociConfigId":1,"instance":{"displayName":"vm","shape":"s","imageId":"i"}}`),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("CreateTask: %v", err)
|
|
}
|
|
for i := 0; i < 4; i++ {
|
|
if _, err := tasks.RunTaskNow(ctx, task.ID); err != nil {
|
|
t.Fatalf("run %d: %v", i, err)
|
|
}
|
|
}
|
|
if got := strings.Join(client.got, ","); got != "ad-1,ad-2,ad-3,ad-1" {
|
|
t.Errorf("自动轮询序列 = %s, want ad-1,ad-2,ad-3,ad-1", got)
|
|
}
|
|
client.got = nil
|
|
fixed, err := tasks.CreateTask(ctx, CreateTaskInput{
|
|
Name: "定向", Type: model.TaskTypeSnatch, CronExpr: "* * * * *",
|
|
Payload: json.RawMessage(`{"ociConfigId":1,"instance":{"displayName":"vm","shape":"s","imageId":"i","availabilityDomain":"ad-2"}}`),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("CreateTask fixed: %v", err)
|
|
}
|
|
for i := 0; i < 2; i++ {
|
|
if _, err := tasks.RunTaskNow(ctx, fixed.ID); err != nil {
|
|
t.Fatalf("run fixed %d: %v", i, err)
|
|
}
|
|
}
|
|
if got := strings.Join(client.got, ","); got != "ad-2,ad-2" {
|
|
t.Errorf("显式指定序列 = %s, want ad-2,ad-2", got)
|
|
}
|
|
}
|
|
|
|
func TestSnatchPartialSuccessKeepsRemaining(t *testing.T) {
|
|
client := &snatchFake{failFirst: 1} // 第 1 台失败、第 2 台成功
|
|
client.tenancy = oci.TenancyInfo{Name: "acme", HomeRegionKey: "FRA"}
|
|
tasks, configs, _ := newTaskEnv(t, client)
|
|
ctx := context.Background()
|
|
if _, err := configs.Import(ctx, trialImportInput()); err != nil {
|
|
t.Fatalf("import: %v", err)
|
|
}
|
|
task, err := tasks.CreateTask(ctx, CreateTaskInput{
|
|
Name: "抢两台", Type: model.TaskTypeSnatch, CronExpr: "* * * * *",
|
|
Payload: json.RawMessage(`{"ociConfigId":1,"count":2,"instance":{"displayName":"vm","shape":"s","imageId":"i"}}`),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("CreateTask: %v", err)
|
|
}
|
|
entry, err := tasks.RunTaskNow(ctx, task.ID)
|
|
if err != nil {
|
|
t.Fatalf("RunTaskNow: %v", err)
|
|
}
|
|
if !entry.Success {
|
|
t.Fatalf("log = %+v, want partial success", entry)
|
|
}
|
|
got, _ := tasks.GetTask(ctx, task.ID)
|
|
if got.Status != model.TaskStatusActive {
|
|
t.Errorf("status = %s, want active（还差 1 台）", got.Status)
|
|
}
|
|
var p snatchPayload
|
|
if err := json.Unmarshal([]byte(got.Payload), &p); err != nil || p.Count != 1 {
|
|
t.Errorf("payload count = %d (%v), want 1", p.Count, err)
|
|
}
|
|
}
|
|
|
|
func TestUpdateTaskPauseResume(t *testing.T) {
|
|
tasks, _, _ := newTaskEnv(t, &fakeClient{})
|
|
ctx := context.Background()
|
|
task, err := tasks.CreateTask(ctx, CreateTaskInput{
|
|
Name: "测活", Type: model.TaskTypeHealthCheck, CronExpr: "*/10 * * * *",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("CreateTask: %v", err)
|
|
}
|
|
paused := model.TaskStatusPaused
|
|
updated, err := tasks.UpdateTask(ctx, task.ID, UpdateTaskInput{Status: &paused})
|
|
if err != nil {
|
|
t.Fatalf("pause: %v", err)
|
|
}
|
|
if updated.Status != model.TaskStatusPaused {
|
|
t.Errorf("status = %s, want paused", updated.Status)
|
|
}
|
|
if _, ok := tasks.entries[task.ID]; ok {
|
|
t.Error("paused task still scheduled")
|
|
}
|
|
active := model.TaskStatusActive
|
|
cronExpr := "*/30 * * * *"
|
|
updated, err = tasks.UpdateTask(ctx, task.ID, UpdateTaskInput{Status: &active, CronExpr: &cronExpr})
|
|
if err != nil {
|
|
t.Fatalf("resume: %v", err)
|
|
}
|
|
if updated.CronExpr != cronExpr || updated.Status != model.TaskStatusActive {
|
|
t.Errorf("updated = %+v, want active with new cron", updated)
|
|
}
|
|
if _, ok := tasks.entries[task.ID]; !ok {
|
|
t.Error("resumed task not scheduled")
|
|
}
|
|
}
|
|
|
|
// enabledTelegramInput 返回启用态的 Telegram 配置输入，供通知测试复用。
|
|
func enabledTelegramInput() *UpdateTelegramInput {
|
|
token := "123456:AAfake"
|
|
return &UpdateTelegramInput{Enabled: true, BotToken: &token, ChatID: "42"}
|
|
}
|
|
|
|
func TestSnatchRepeatedFailureNotifiesOnce(t *testing.T) {
|
|
client := &snatchFake{failFirst: 3}
|
|
client.tenancy = oci.TenancyInfo{Name: "acme", HomeRegionKey: "FRA"}
|
|
tasks, configs, _ := newTaskEnv(t, client)
|
|
srv, rec := newFakeTelegram(t, `{"ok":true}`)
|
|
tasks.notifier = newTestNotifier(t, srv.URL, enabledTelegramInput())
|
|
ctx := context.Background()
|
|
if _, err := configs.Import(ctx, trialImportInput()); err != nil {
|
|
t.Fatalf("import: %v", err)
|
|
}
|
|
task, err := tasks.CreateTask(ctx, CreateTaskInput{
|
|
Name: "抢机", Type: model.TaskTypeSnatch, CronExpr: "* * * * *",
|
|
Payload: json.RawMessage(`{"ociConfigId":1,"instance":{"displayName":"vm","shape":"s","imageId":"i"}}`),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("CreateTask: %v", err)
|
|
}
|
|
|
|
// 连续失败 3 次：只发 1 条失败通知
|
|
for i := 1; i <= 3; i++ {
|
|
if _, err := tasks.RunTaskNow(ctx, task.ID); err != nil {
|
|
t.Fatalf("run %d: %v", i, err)
|
|
}
|
|
}
|
|
tasks.notifier.Wait()
|
|
if texts := rec.snapshot(); len(texts) != 1 || !strings.Contains(texts[0], "任务失败") {
|
|
t.Fatalf("失败阶段通知 = %v, want 仅 1 条任务失败", texts)
|
|
}
|
|
|
|
// 第 4 次抢满：追加 1 条抢机成功，不叠加恢复通知
|
|
if _, err := tasks.RunTaskNow(ctx, task.ID); err != nil {
|
|
t.Fatalf("run 4: %v", err)
|
|
}
|
|
tasks.notifier.Wait()
|
|
texts := rec.snapshot()
|
|
if len(texts) != 2 || !strings.Contains(texts[1], "抢机成功") {
|
|
t.Fatalf("成功后通知 = %v, want 追加 1 条抢机成功", texts)
|
|
}
|
|
}
|
|
|
|
// TestNotifyFiltersDisabledKinds 验证按事件类型过滤：关闭「任务失败」后
|
|
// 失败不再推送，任务恢复与抢机成功不受影响仍推送。
|
|
func TestNotifyFiltersDisabledKinds(t *testing.T) {
|
|
client := &snatchFake{failFirst: 3}
|
|
client.tenancy = oci.TenancyInfo{Name: "acme", HomeRegionKey: "FRA"}
|
|
tasks, configs, _ := newTaskEnv(t, client)
|
|
srv, rec := newFakeTelegram(t, `{"ok":true}`)
|
|
tasks.notifier = newTestNotifier(t, srv.URL, enabledTelegramInput())
|
|
ctx := context.Background()
|
|
off := NotifyEventsView{TaskRecover: true, SnatchSuccess: true, TenantDead: true, TaskStop: true}
|
|
if err := tasks.settings.UpdateNotifyEvents(ctx, off); err != nil {
|
|
t.Fatalf("UpdateNotifyEvents: %v", err)
|
|
}
|
|
if _, err := configs.Import(ctx, trialImportInput()); err != nil {
|
|
t.Fatalf("import: %v", err)
|
|
}
|
|
task, err := tasks.CreateTask(ctx, CreateTaskInput{
|
|
Name: "抢两台", Type: model.TaskTypeSnatch, CronExpr: "* * * * *",
|
|
Payload: json.RawMessage(`{"ociConfigId":1,"count":2,"instance":{"displayName":"vm","shape":"s","imageId":"i"}}`),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("CreateTask: %v", err)
|
|
}
|
|
|
|
// 三次执行依次触发：task_fail（已关闭）→ task_recover → snatch_success
|
|
steps := []struct {
|
|
name string
|
|
wantLen int // 执行后累计推送条数
|
|
wantKw string // 最新一条需包含的关键词；空表示本步无新增
|
|
}{
|
|
{name: "两台全失败：任务失败已关闭不发", wantLen: 0},
|
|
{name: "抢到 1 台剩 1：任务恢复仍发", wantLen: 1, wantKw: "任务恢复"},
|
|
{name: "抢满收尾：抢机成功仍发", wantLen: 2, wantKw: "抢机成功"},
|
|
}
|
|
for _, step := range steps {
|
|
if _, err := tasks.RunTaskNow(ctx, task.ID); err != nil {
|
|
t.Fatalf("%s: %v", step.name, err)
|
|
}
|
|
tasks.notifier.Wait()
|
|
texts := rec.snapshot()
|
|
if len(texts) != step.wantLen {
|
|
t.Fatalf("%s: 通知 = %v (%d 条), want %d 条", step.name, texts, len(texts), step.wantLen)
|
|
}
|
|
if step.wantKw != "" && !strings.Contains(texts[len(texts)-1], step.wantKw) {
|
|
t.Fatalf("%s: 最新通知 = %q, want contains %q", step.name, texts[len(texts)-1], step.wantKw)
|
|
}
|
|
}
|
|
}
|
|
|
|
// deadPickFake 按 UserOCID 控制测活结果，模拟部分租户失联。
|
|
type deadPickFake struct {
|
|
fakeClient
|
|
mu sync.Mutex
|
|
dead map[string]bool
|
|
}
|
|
|
|
func (f *deadPickFake) ValidateKey(ctx context.Context, cred oci.Credentials) (oci.TenancyInfo, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if f.dead[cred.UserOCID] {
|
|
return oci.TenancyInfo{}, fmt.Errorf("NotAuthenticated")
|
|
}
|
|
return f.tenancy, nil
|
|
}
|
|
|
|
func (f *deadPickFake) setDead(user string, dead bool) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.dead[user] = dead
|
|
}
|
|
|
|
func TestHealthCheckNotifiesDeadAliasOnChange(t *testing.T) {
|
|
client := &deadPickFake{dead: map[string]bool{}}
|
|
client.tenancy = oci.TenancyInfo{Name: "acme", HomeRegionKey: "FRA"}
|
|
tasks, configs, _ := newTaskEnv(t, client)
|
|
srv, rec := newFakeTelegram(t, `{"ok":true}`)
|
|
tasks.notifier = newTestNotifier(t, srv.URL, enabledTelegramInput())
|
|
ctx := context.Background()
|
|
if _, err := configs.Import(ctx, trialImportInput()); err != nil {
|
|
t.Fatalf("import A: %v", err)
|
|
}
|
|
second := trialImportInput()
|
|
second.Alias = "second-tenant"
|
|
second.UserOCID = "ocid1.user.oc1..u2"
|
|
if _, err := configs.Import(ctx, second); err != nil {
|
|
t.Fatalf("import B: %v", err)
|
|
}
|
|
task, err := tasks.CreateTask(ctx, CreateTaskInput{
|
|
Name: "测活", Type: model.TaskTypeHealthCheck, CronExpr: "*/5 * * * *",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("CreateTask: %v", err)
|
|
}
|
|
run := func(step string) {
|
|
t.Helper()
|
|
if _, err := tasks.RunTaskNow(ctx, task.ID); err != nil {
|
|
t.Fatalf("%s: %v", step, err)
|
|
}
|
|
tasks.notifier.Wait()
|
|
}
|
|
|
|
run("全部存活")
|
|
client.setDead("ocid1.user.oc1..u2", true)
|
|
run("B 失联") // 集合变化：发 1 条
|
|
run("B 持续失联") // 集合不变：不发
|
|
client.setDead("ocid1.user.oc1..u2", false)
|
|
run("B 恢复") // 集合清空：不发失联通知
|
|
client.setDead("ocid1.user.oc1..u2", true)
|
|
run("B 再失联") // 集合再次变化：再发 1 条
|
|
|
|
texts := rec.snapshot()
|
|
if len(texts) != 2 {
|
|
t.Fatalf("失联通知 %d 条 %v, want 2 条", len(texts), texts)
|
|
}
|
|
for i, text := range texts {
|
|
if !strings.Contains(text, "租户失联") || !strings.Contains(text, "second-tenant") {
|
|
t.Errorf("text[%d] = %q, want 含「租户失联」与失联别名", i, text)
|
|
}
|
|
}
|
|
}
|
|
|
|
// snatchAuthCount 解析任务 payload 中的连续鉴权失败计数。
|
|
func snatchAuthCount(t *testing.T, payload string) int {
|
|
t.Helper()
|
|
var p snatchPayload
|
|
if err := json.Unmarshal([]byte(payload), &p); err != nil {
|
|
t.Fatalf("parse payload %q: %v", payload, err)
|
|
}
|
|
return p.AuthFailCount
|
|
}
|
|
|
|
func TestSnatchAuthFuse(t *testing.T) {
|
|
notAuth := fmt.Errorf("NotAuthenticated: 鉴权失败")
|
|
capacity := fmt.Errorf("Out of host capacity")
|
|
tests := []struct {
|
|
name string
|
|
limit int // 0 表示不设置，用默认 3
|
|
count int // 目标台数；0 按 1 处理
|
|
script []error
|
|
runs int
|
|
wantStatus string
|
|
wantAuthCnt int
|
|
}{
|
|
{
|
|
name: "连续 2 次不熔断", script: []error{notAuth, notAuth}, runs: 2,
|
|
wantStatus: model.TaskStatusActive, wantAuthCnt: 2,
|
|
},
|
|
{
|
|
name: "默认阈值第 3 次熔断", script: []error{notAuth, notAuth, notAuth}, runs: 3,
|
|
wantStatus: model.TaskStatusFailed, wantAuthCnt: 3,
|
|
},
|
|
{
|
|
name: "中途成功清零", count: 2, script: []error{notAuth, notAuth, nil, capacity}, runs: 2,
|
|
wantStatus: model.TaskStatusActive, wantAuthCnt: 0,
|
|
},
|
|
{
|
|
name: "中途其他错误清零", script: []error{notAuth, capacity}, runs: 2,
|
|
wantStatus: model.TaskStatusActive, wantAuthCnt: 0,
|
|
},
|
|
{
|
|
name: "抢满收尾清零并写回", script: []error{notAuth, nil}, runs: 2,
|
|
wantStatus: model.TaskStatusSucceeded, wantAuthCnt: 0,
|
|
},
|
|
{
|
|
name: "limit=1 立即熔断", limit: 1, script: []error{notAuth}, runs: 1,
|
|
wantStatus: model.TaskStatusFailed, wantAuthCnt: 1,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
task, tasks := runAuthFuseCase(t, tt.limit, tt.count, tt.script, tt.runs)
|
|
got, err := tasks.GetTask(context.Background(), task.ID)
|
|
if err != nil {
|
|
t.Fatalf("GetTask: %v", err)
|
|
}
|
|
if got.Status != tt.wantStatus {
|
|
t.Errorf("status = %s, want %s", got.Status, tt.wantStatus)
|
|
}
|
|
if cnt := snatchAuthCount(t, got.Payload); cnt != tt.wantAuthCnt {
|
|
t.Errorf("authFailCount = %d, want %d", cnt, tt.wantAuthCnt)
|
|
}
|
|
assertAuthFuseSideEffects(t, tasks, got, tt.wantStatus, tt.wantAuthCnt)
|
|
})
|
|
}
|
|
}
|
|
|
|
// runAuthFuseCase 搭好环境并按用例参数执行 runs 次抢机任务。
|
|
func runAuthFuseCase(t *testing.T, limit, count int, script []error, runs int) (*model.Task, *TaskService) {
|
|
t.Helper()
|
|
client := &scriptedSnatchFake{script: script}
|
|
client.tenancy = oci.TenancyInfo{Name: "acme", HomeRegionKey: "FRA"}
|
|
tasks, configs, _ := newTaskEnv(t, client)
|
|
ctx := context.Background()
|
|
if _, err := configs.Import(ctx, trialImportInput()); err != nil {
|
|
t.Fatalf("import: %v", err)
|
|
}
|
|
if limit > 0 {
|
|
if err := tasks.settings.UpdateTaskSettings(ctx, TaskSettingsView{SnatchAuthFailLimit: limit}); err != nil {
|
|
t.Fatalf("UpdateTaskSettings: %v", err)
|
|
}
|
|
}
|
|
if count == 0 {
|
|
count = 1
|
|
}
|
|
task, err := tasks.CreateTask(ctx, CreateTaskInput{
|
|
Name: "抢机", Type: model.TaskTypeSnatch, CronExpr: "* * * * *",
|
|
Payload: json.RawMessage(fmt.Sprintf(
|
|
`{"ociConfigId":1,"count":%d,"instance":{"displayName":"vm","shape":"s","imageId":"i"}}`, count)),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("CreateTask: %v", err)
|
|
}
|
|
for i := 1; i <= runs; i++ {
|
|
if _, err := tasks.RunTaskNow(ctx, task.ID); err != nil {
|
|
t.Fatalf("run %d: %v", i, err)
|
|
}
|
|
}
|
|
return task, tasks
|
|
}
|
|
|
|
// assertAuthFuseSideEffects 断言熔断副作用：仅 active 保持调度（failed 熔断、
|
|
// succeeded 抢满均移出）；failed 的 LastError 须带熔断说明，其余不得出现。
|
|
func assertAuthFuseSideEffects(t *testing.T, tasks *TaskService, got *model.Task, wantStatus string, wantCnt int) {
|
|
t.Helper()
|
|
if _, scheduled := tasks.entries[got.ID]; scheduled != (wantStatus == model.TaskStatusActive) {
|
|
t.Errorf("scheduled = %v, want %v", scheduled, wantStatus == model.TaskStatusActive)
|
|
}
|
|
fused := wantStatus == model.TaskStatusFailed
|
|
fuseMsg := fmt.Sprintf("连续 %d 次 NotAuthenticated，任务已熔断停止", wantCnt)
|
|
if fused && !strings.Contains(got.LastError, fuseMsg) {
|
|
t.Errorf("LastError = %q, want 含 %q", got.LastError, fuseMsg)
|
|
}
|
|
if !fused && strings.Contains(got.LastError, "任务已熔断停止") {
|
|
t.Errorf("LastError = %q, 不应出现熔断说明", got.LastError)
|
|
}
|
|
}
|
|
|
|
// TestNotifyEventsFailedTransitions 锁定熔断相关通知语义：置 failed 当次
|
|
// 只发任务停止不叠加任务失败；重新启用后恢复只发任务恢复。
|
|
func TestNotifyEventsFailedTransitions(t *testing.T) {
|
|
fuseErr := "连续 3 次 NotAuthenticated，任务已熔断停止: NotAuthenticated"
|
|
tests := []struct {
|
|
name string
|
|
prev, cur taskSnapshot
|
|
wantKinds []notifyKind
|
|
wantText string // 首条通知变量值需包含的关键词;空表示不校验
|
|
}{
|
|
{
|
|
name: "熔断翻转只发任务停止不叠加任务失败",
|
|
prev: taskSnapshot{Name: "抢机", Status: model.TaskStatusActive},
|
|
cur: taskSnapshot{Name: "抢机", Status: model.TaskStatusFailed, LastError: fuseErr},
|
|
wantKinds: []notifyKind{notifyTaskStop},
|
|
wantText: "任务已熔断停止",
|
|
},
|
|
{
|
|
name: "持续 failed 不重复发",
|
|
prev: taskSnapshot{Name: "抢机", Status: model.TaskStatusFailed, LastError: fuseErr},
|
|
cur: taskSnapshot{Name: "抢机", Status: model.TaskStatusFailed, LastError: fuseErr},
|
|
wantKinds: nil,
|
|
},
|
|
{
|
|
name: "failed 重新启用后成功只发任务恢复",
|
|
prev: taskSnapshot{Name: "抢机", Status: model.TaskStatusActive, LastError: fuseErr},
|
|
cur: taskSnapshot{Name: "抢机", Status: model.TaskStatusActive},
|
|
wantKinds: []notifyKind{notifyTaskRecover},
|
|
wantText: "抢机",
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
events := notifyEvents(tt.prev, tt.cur)
|
|
if len(events) != len(tt.wantKinds) {
|
|
t.Fatalf("events = %+v (%d 条), want %v", events, len(events), tt.wantKinds)
|
|
}
|
|
for i, ev := range events {
|
|
if ev.Kind != tt.wantKinds[i] {
|
|
t.Errorf("kind[%d] = %s, want %s", i, ev.Kind, tt.wantKinds[i])
|
|
}
|
|
}
|
|
if tt.wantText != "" {
|
|
joined := ""
|
|
for _, v := range events[0].Vars {
|
|
joined += v + " "
|
|
}
|
|
if !strings.Contains(joined, tt.wantText) {
|
|
t.Errorf("vars = %q, want contains %q", joined, tt.wantText)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestUpdateTaskFailedToActiveResetsAuthFail(t *testing.T) {
|
|
notAuth := fmt.Errorf("NotAuthenticated: 鉴权失败")
|
|
task, tasks := runAuthFuseCase(t, 0, 1, []error{notAuth, notAuth, notAuth}, 3)
|
|
ctx := context.Background()
|
|
got, err := tasks.GetTask(ctx, task.ID)
|
|
if err != nil {
|
|
t.Fatalf("GetTask: %v", err)
|
|
}
|
|
if got.Status != model.TaskStatusFailed {
|
|
t.Fatalf("前置熔断未生效: status=%s", got.Status)
|
|
}
|
|
|
|
active := model.TaskStatusActive
|
|
updated, err := tasks.UpdateTask(ctx, task.ID, UpdateTaskInput{Status: &active})
|
|
if err != nil {
|
|
t.Fatalf("重新启用: %v", err)
|
|
}
|
|
if updated.Status != model.TaskStatusActive {
|
|
t.Errorf("status = %s, want active", updated.Status)
|
|
}
|
|
if cnt := snatchAuthCount(t, updated.Payload); cnt != 0 {
|
|
t.Errorf("authFailCount = %d, want 0（重新启用应清零）", cnt)
|
|
}
|
|
if _, ok := tasks.entries[task.ID]; !ok {
|
|
t.Error("重新启用后任务未回到调度")
|
|
}
|
|
}
|