677 lines
25 KiB
Go
677 lines
25 KiB
Go
package service
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"testing"
|
||
"time"
|
||
|
||
"github.com/glebarez/sqlite"
|
||
"gorm.io/gorm"
|
||
"gorm.io/gorm/logger"
|
||
|
||
"oci-portal/internal/crypto"
|
||
"oci-portal/internal/model"
|
||
"oci-portal/internal/oci"
|
||
)
|
||
|
||
type invalidatingClient struct {
|
||
*fakeClient
|
||
invalidated []string
|
||
}
|
||
|
||
type blockingClient struct {
|
||
*fakeClient
|
||
started chan struct{}
|
||
release chan struct{}
|
||
}
|
||
|
||
func (c *blockingClient) ValidateKey(context.Context, oci.Credentials) (oci.TenancyInfo, error) {
|
||
c.started <- struct{}{}
|
||
<-c.release
|
||
return oci.TenancyInfo{Name: "target"}, nil
|
||
}
|
||
|
||
func (c *invalidatingClient) InvalidateTenancy(id string) {
|
||
c.invalidated = append(c.invalidated, id)
|
||
}
|
||
|
||
func newTenantDeleteEnv(t *testing.T, client oci.Client) (*OciConfigService, *TaskService, *gorm.DB) {
|
||
t.Helper()
|
||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
|
||
if err != nil {
|
||
t.Fatalf("open database: %v", err)
|
||
}
|
||
sqlDB, err := db.DB()
|
||
if err != nil {
|
||
t.Fatalf("database handle: %v", err)
|
||
}
|
||
sqlDB.SetMaxOpenConns(1)
|
||
migrateTenantDeleteModels(t, db)
|
||
cipher, err := crypto.NewCipher("test-data-key")
|
||
if err != nil {
|
||
t.Fatalf("new cipher: %v", err)
|
||
}
|
||
configs := NewOciConfigService(db, cipher, client)
|
||
tasks := NewTaskService(db, configs, nil, nil)
|
||
configs.SetTenantCleanupDeps(tasks, nil)
|
||
return configs, tasks, db
|
||
}
|
||
|
||
func migrateTenantDeleteModels(t *testing.T, db *gorm.DB) {
|
||
t.Helper()
|
||
err := db.AutoMigrate(
|
||
&model.OciConfig{}, &model.Task{}, &model.TaskLog{}, &model.Setting{},
|
||
&model.CheckSnapshot{}, &model.CostSnapshot{}, &model.RegionCache{}, &model.CompartmentCache{},
|
||
&model.LogEvent{}, &model.AlertRule{}, &model.AlertRuleHit{},
|
||
&model.AiChannel{}, &model.AiModelCache{}, &model.AiCallLog{}, &model.AiContentLog{},
|
||
&model.Proxy{}, &model.AiKey{}, &model.SystemLog{},
|
||
)
|
||
if err != nil {
|
||
t.Fatalf("auto migrate: %v", err)
|
||
}
|
||
}
|
||
|
||
func TestPlanTenantTask(t *testing.T) {
|
||
for _, tt := range tenantTaskCases() {
|
||
t.Run(tt.name, func(t *testing.T) {
|
||
action, ok, err := planTenantTask(tt.task, 1)
|
||
if (err != nil) != tt.wantErr {
|
||
t.Fatalf("error = %v, wantErr %v", err, tt.wantErr)
|
||
}
|
||
if ok != tt.wantOK || action.deleteTask != tt.wantDelete {
|
||
t.Errorf("result = (ok=%v, delete=%v), want (%v, %v)", ok, action.deleteTask, tt.wantOK, tt.wantDelete)
|
||
}
|
||
if action.payload != tt.wantPayload {
|
||
t.Errorf("payload = %q, want %q", action.payload, tt.wantPayload)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
type tenantTaskCase struct {
|
||
name string
|
||
task model.Task
|
||
wantOK bool
|
||
wantDelete bool
|
||
wantPayload string
|
||
wantErr bool
|
||
}
|
||
|
||
func tenantTaskCases() []tenantTaskCase {
|
||
return []tenantTaskCase{
|
||
{name: "抢机命中", task: taskOf(model.TaskTypeSnatch, `{"ociConfigId":1}`), wantOK: true, wantDelete: true},
|
||
{name: "抢机未命中", task: taskOf(model.TaskTypeSnatch, `{"ociConfigId":2}`)},
|
||
{name: "测活全局", task: taskOf(model.TaskTypeHealthCheck, `{"ociConfigIds":[]}`), wantOK: true, wantPayload: `{"ociConfigIds":[]}`},
|
||
{name: "测活单租户", task: taskOf(model.TaskTypeHealthCheck, `{"ociConfigIds":[1]}`), wantOK: true, wantDelete: true},
|
||
{name: "测活多租户", task: taskOf(model.TaskTypeHealthCheck, `{"ociConfigIds":[1,2]}`), wantOK: true, wantPayload: `{"ociConfigIds":[2]}`},
|
||
{name: "成本去重命中", task: taskOf(model.TaskTypeCost, `{"ociConfigIds":[1,1,2]}`), wantOK: true, wantPayload: `{"ociConfigIds":[2]}`},
|
||
{name: "成本未命中", task: taskOf(model.TaskTypeCost, `{"ociConfigIds":[2]}`)},
|
||
// 坏 payload 记警告跳过(保留原任务),不再 fail-closed 阻断整个租户删除
|
||
{name: "非法 JSON 跳过", task: taskOf(model.TaskTypeCost, `{`)},
|
||
{name: "抢机坏 payload 跳过", task: taskOf(model.TaskTypeSnatch, `not-json`)},
|
||
{name: "抢机空 payload 跳过", task: taskOf(model.TaskTypeSnatch, ``)},
|
||
}
|
||
}
|
||
|
||
func taskOf(taskType, payload string) model.Task {
|
||
return model.Task{ID: 10, Type: taskType, Payload: payload}
|
||
}
|
||
|
||
func TestDeleteTenantCleansRelatedRows(t *testing.T) {
|
||
client := &invalidatingClient{fakeClient: &fakeClient{}}
|
||
configs, _, db := newTenantDeleteEnv(t, client)
|
||
target, other := seedDeleteTenants(t, db)
|
||
seedTenantSnapshots(t, db, target.ID, other.ID)
|
||
seedTenantEvents(t, db, target.ID, other.ID)
|
||
seedTenantAI(t, db, target.ID, other.ID)
|
||
seedRetainedGlobals(t, db)
|
||
if err := configs.Delete(context.Background(), target.ID); err != nil {
|
||
t.Fatalf("delete tenant: %v", err)
|
||
}
|
||
assertTenantRowsGone(t, db, target.ID)
|
||
assertOtherTenantRowsRemain(t, db, other.ID)
|
||
assertRetainedGlobals(t, db)
|
||
if len(client.invalidated) != 1 || client.invalidated[0] != target.TenancyOCID {
|
||
t.Errorf("invalidated = %v, want [%s]", client.invalidated, target.TenancyOCID)
|
||
}
|
||
}
|
||
|
||
func seedDeleteTenants(t *testing.T, db *gorm.DB) (model.OciConfig, model.OciConfig) {
|
||
t.Helper()
|
||
target := model.OciConfig{Alias: "target", TenancyOCID: "ocid1.tenancy.target"}
|
||
other := model.OciConfig{Alias: "other", TenancyOCID: "ocid1.tenancy.other"}
|
||
mustCreate(t, db, &target)
|
||
mustCreate(t, db, &other)
|
||
return target, other
|
||
}
|
||
|
||
func seedTenantSnapshots(t *testing.T, db *gorm.DB, target, other uint) {
|
||
t.Helper()
|
||
for _, id := range []uint{target, other} {
|
||
mustCreate(t, db, &model.CheckSnapshot{OciConfigID: id})
|
||
mustCreate(t, db, &model.CostSnapshot{OciConfigID: id, Day: "2026-07-10"})
|
||
mustCreate(t, db, &model.RegionCache{OciConfigID: id, Key: "PHX"})
|
||
mustCreate(t, db, &model.CompartmentCache{OciConfigID: id, OCID: fmt.Sprintf("comp-%d", id)})
|
||
mustCreate(t, db, &model.Setting{Key: secretKey(id), Value: fmt.Sprintf("secret-%d", id)})
|
||
}
|
||
}
|
||
|
||
func seedTenantEvents(t *testing.T, db *gorm.DB, target, other uint) {
|
||
t.Helper()
|
||
targetEvent := model.LogEvent{OciConfigID: target, MessageID: "target-event"}
|
||
otherEvent := model.LogEvent{OciConfigID: other, MessageID: "other-event"}
|
||
targetRule := model.AlertRule{Name: "target-rule", OciConfigID: target}
|
||
globalRule := model.AlertRule{Name: "global-rule", OciConfigID: 0}
|
||
for _, value := range []any{&targetEvent, &otherEvent, &targetRule, &globalRule} {
|
||
mustCreate(t, db, value)
|
||
}
|
||
hits := []model.AlertRuleHit{
|
||
{RuleID: targetRule.ID, LogEventID: otherEvent.ID},
|
||
{RuleID: globalRule.ID, LogEventID: targetEvent.ID},
|
||
{RuleID: globalRule.ID, LogEventID: otherEvent.ID},
|
||
}
|
||
for i := range hits {
|
||
mustCreate(t, db, &hits[i])
|
||
}
|
||
}
|
||
|
||
func seedTenantAI(t *testing.T, db *gorm.DB, target, other uint) {
|
||
t.Helper()
|
||
for _, id := range []uint{target, other} {
|
||
channel := model.AiChannel{Name: fmt.Sprintf("channel-%d", id), OciConfigID: id, Region: "us-phoenix-1"}
|
||
mustCreate(t, db, &channel)
|
||
mustCreate(t, db, &model.AiModelCache{ChannelID: channel.ID, ModelOcid: fmt.Sprintf("model-%d", id)})
|
||
call := model.AiCallLog{ChannelID: channel.ID, ChannelName: channel.Name}
|
||
mustCreate(t, db, &call)
|
||
mustCreate(t, db, &model.AiContentLog{CallLogID: call.ID, RequestBody: "sensitive"})
|
||
}
|
||
}
|
||
|
||
func seedRetainedGlobals(t *testing.T, db *gorm.DB) {
|
||
t.Helper()
|
||
mustCreate(t, db, &model.Proxy{Name: "shared", Type: "http"})
|
||
mustCreate(t, db, &model.AiKey{Name: "global-key", KeyHash: "hash", Tail: "hash"})
|
||
mustCreate(t, db, &model.SystemLog{Method: "DELETE", Path: "/api/v1/oci-configs/1"})
|
||
mustCreate(t, db, &model.Setting{Key: "notify_channels", Value: "[]"})
|
||
}
|
||
|
||
func assertTenantRowsGone(t *testing.T, db *gorm.DB, id uint) {
|
||
t.Helper()
|
||
rows := []any{
|
||
&model.OciConfig{}, &model.CheckSnapshot{}, &model.CostSnapshot{},
|
||
&model.RegionCache{}, &model.CompartmentCache{}, &model.LogEvent{},
|
||
&model.AlertRule{}, &model.AiChannel{},
|
||
}
|
||
for _, value := range rows {
|
||
column := "oci_config_id"
|
||
if _, ok := value.(*model.OciConfig); ok {
|
||
column = "id"
|
||
}
|
||
assertCount(t, db, value, column+" = ?", []any{id}, 0)
|
||
}
|
||
assertCount(t, db, &model.Setting{}, "key = ?", []any{secretKey(id)}, 0)
|
||
assertCount(t, db, &model.AlertRuleHit{}, "", nil, 1)
|
||
assertCount(t, db, &model.AiModelCache{}, "", nil, 1)
|
||
assertCount(t, db, &model.AiCallLog{}, "", nil, 1)
|
||
assertCount(t, db, &model.AiContentLog{}, "", nil, 1)
|
||
}
|
||
|
||
func assertOtherTenantRowsRemain(t *testing.T, db *gorm.DB, id uint) {
|
||
t.Helper()
|
||
for _, value := range []any{
|
||
&model.OciConfig{}, &model.CheckSnapshot{}, &model.CostSnapshot{},
|
||
&model.RegionCache{}, &model.CompartmentCache{}, &model.LogEvent{},
|
||
&model.AiChannel{},
|
||
} {
|
||
column := "oci_config_id"
|
||
if _, ok := value.(*model.OciConfig); ok {
|
||
column = "id"
|
||
}
|
||
assertCount(t, db, value, column+" = ?", []any{id}, 1)
|
||
}
|
||
assertCount(t, db, &model.Setting{}, "key = ?", []any{secretKey(id)}, 1)
|
||
assertRemainingIndirectRows(t, db, id)
|
||
}
|
||
|
||
func assertRemainingIndirectRows(t *testing.T, db *gorm.DB, otherID uint) {
|
||
t.Helper()
|
||
var channel model.AiChannel
|
||
if err := db.Where("oci_config_id = ?", otherID).First(&channel).Error; err != nil {
|
||
t.Fatalf("load other AI channel: %v", err)
|
||
}
|
||
assertCount(t, db, &model.AiModelCache{}, "channel_id = ?", []any{channel.ID}, 1)
|
||
assertCount(t, db, &model.AiCallLog{}, "channel_id = ?", []any{channel.ID}, 1)
|
||
var call model.AiCallLog
|
||
if err := db.Where("channel_id = ?", channel.ID).First(&call).Error; err != nil {
|
||
t.Fatalf("load other AI call: %v", err)
|
||
}
|
||
assertCount(t, db, &model.AiContentLog{}, "call_log_id = ?", []any{call.ID}, 1)
|
||
assertRemainingAlertHit(t, db, otherID)
|
||
}
|
||
|
||
func assertRemainingAlertHit(t *testing.T, db *gorm.DB, otherID uint) {
|
||
t.Helper()
|
||
var hit model.AlertRuleHit
|
||
if err := db.First(&hit).Error; err != nil {
|
||
t.Fatalf("load remaining alert hit: %v", err)
|
||
}
|
||
var rule model.AlertRule
|
||
var event model.LogEvent
|
||
if err := db.First(&rule, hit.RuleID).Error; err != nil {
|
||
t.Fatalf("load remaining rule: %v", err)
|
||
}
|
||
if err := db.First(&event, hit.LogEventID).Error; err != nil {
|
||
t.Fatalf("load remaining event: %v", err)
|
||
}
|
||
if rule.OciConfigID != 0 || event.OciConfigID != otherID {
|
||
t.Errorf("remaining hit = rule cfg %d/event cfg %d, want global/other", rule.OciConfigID, event.OciConfigID)
|
||
}
|
||
}
|
||
|
||
func assertRetainedGlobals(t *testing.T, db *gorm.DB) {
|
||
t.Helper()
|
||
assertCount(t, db, &model.AlertRule{}, "oci_config_id = 0", nil, 1)
|
||
assertCount(t, db, &model.Proxy{}, "", nil, 1)
|
||
assertCount(t, db, &model.AiKey{}, "", nil, 1)
|
||
assertCount(t, db, &model.SystemLog{}, "", nil, 1)
|
||
assertCount(t, db, &model.Setting{}, "key = ?", []any{"notify_channels"}, 1)
|
||
}
|
||
|
||
func TestDeleteTenantRewritesTasksAndCron(t *testing.T) {
|
||
configs, tasks, db := newTenantDeleteEnv(t, &fakeClient{})
|
||
target, other := seedDeleteTenants(t, db)
|
||
created := seedTenantTasks(t, tasks, target.ID, other.ID)
|
||
if err := configs.Delete(context.Background(), target.ID); err != nil {
|
||
t.Fatalf("delete tenant: %v", err)
|
||
}
|
||
assertDeletedTasks(t, db, tasks, created[:2])
|
||
assertRewrittenTasks(t, db, created[2:])
|
||
}
|
||
|
||
func TestDeleteTenantReconcilesAiProbe(t *testing.T) {
|
||
for _, keepOther := range []bool{false, true} {
|
||
name := fmt.Sprintf("keepOther=%v", keepOther)
|
||
t.Run(name, func(t *testing.T) {
|
||
configs, tasks, db := newTenantDeleteEnv(t, &fakeClient{})
|
||
target, other := seedDeleteTenants(t, db)
|
||
seedProbeChannels(t, db, target.ID, other.ID, keepOther)
|
||
tasks.SyncAiProbeTask(context.Background())
|
||
probe := loadAiProbe(t, db)
|
||
seedTaskHistory(t, db, probe.ID)
|
||
if err := configs.Delete(context.Background(), target.ID); err != nil {
|
||
t.Fatalf("delete tenant: %v", err)
|
||
}
|
||
assertAiProbeResult(t, db, tasks, probe.ID, keepOther)
|
||
})
|
||
}
|
||
}
|
||
|
||
func seedProbeChannels(t *testing.T, db *gorm.DB, target, other uint, keepOther bool) {
|
||
t.Helper()
|
||
mustCreate(t, db, &model.AiChannel{Name: "target", OciConfigID: target, Region: "r1"})
|
||
if keepOther {
|
||
mustCreate(t, db, &model.AiChannel{Name: "other", OciConfigID: other, Region: "r1"})
|
||
}
|
||
}
|
||
|
||
func loadAiProbe(t *testing.T, db *gorm.DB) model.Task {
|
||
t.Helper()
|
||
var task model.Task
|
||
if err := db.Where("type = ?", model.TaskTypeAiProbe).First(&task).Error; err != nil {
|
||
t.Fatalf("load AI probe: %v", err)
|
||
}
|
||
return task
|
||
}
|
||
|
||
func seedTaskHistory(t *testing.T, db *gorm.DB, taskID uint) {
|
||
t.Helper()
|
||
updates := map[string]any{"last_error": "old", "run_count": 2, "last_run_at": time.Now()}
|
||
if err := db.Model(&model.Task{}).Where("id = ?", taskID).Updates(updates).Error; err != nil {
|
||
t.Fatalf("seed task history: %v", err)
|
||
}
|
||
mustCreate(t, db, &model.TaskLog{TaskID: taskID, Message: "old"})
|
||
mustCreate(t, db, &model.Setting{Key: deadAliasKey(taskID), Value: "old"})
|
||
}
|
||
|
||
func assertAiProbeResult(t *testing.T, db *gorm.DB, tasks *TaskService, id uint, keep bool) {
|
||
t.Helper()
|
||
want := int64(0)
|
||
if keep {
|
||
want = 1
|
||
}
|
||
assertCount(t, db, &model.Task{}, "id = ?", []any{id}, want)
|
||
assertCount(t, db, &model.TaskLog{}, "task_id = ?", []any{id}, 0)
|
||
assertCount(t, db, &model.Setting{}, "key = ?", []any{deadAliasKey(id)}, 0)
|
||
if !keep {
|
||
if _, ok := tasks.entries[id]; ok {
|
||
t.Errorf("AI probe %d still scheduled", id)
|
||
}
|
||
return
|
||
}
|
||
probe := loadAiProbe(t, db)
|
||
if probe.RunCount != 0 || probe.LastRunAt != nil || probe.LastError != "" {
|
||
t.Errorf("AI probe history not reset: %+v", probe)
|
||
}
|
||
}
|
||
|
||
func seedTenantTasks(t *testing.T, tasks *TaskService, target, other uint) []model.Task {
|
||
t.Helper()
|
||
inputs := []CreateTaskInput{
|
||
{Name: "snatch", Type: model.TaskTypeSnatch, CronExpr: "0 0 * * *", Payload: []byte(fmt.Sprintf(`{"ociConfigId":%d,"instance":{"displayName":"vm","region":"r","availabilityDomain":"a","subnetId":"s","shape":"x","imageId":"i"}}`, target))},
|
||
{Name: "single", Type: model.TaskTypeCost, CronExpr: "0 0 * * *", Payload: []byte(fmt.Sprintf(`{"ociConfigIds":[%d]}`, target))},
|
||
{Name: "mixed", Type: model.TaskTypeHealthCheck, CronExpr: "0 0 * * *", Payload: []byte(fmt.Sprintf(`{"ociConfigIds":[%d,%d]}`, target, other))},
|
||
{Name: "global", Type: model.TaskTypeCost, CronExpr: "0 0 * * *", Payload: []byte(`{"ociConfigIds":[]}`)},
|
||
{Name: "other", Type: model.TaskTypeHealthCheck, CronExpr: "0 0 * * *", Payload: []byte(fmt.Sprintf(`{"ociConfigIds":[%d]}`, other))},
|
||
}
|
||
return createTasksWithHistory(t, tasks, inputs)
|
||
}
|
||
|
||
func createTasksWithHistory(t *testing.T, tasks *TaskService, inputs []CreateTaskInput) []model.Task {
|
||
t.Helper()
|
||
out := make([]model.Task, 0, len(inputs))
|
||
for _, input := range inputs {
|
||
task, err := tasks.CreateTask(context.Background(), input)
|
||
if err != nil {
|
||
t.Fatalf("create task %s: %v", input.Name, err)
|
||
}
|
||
tasks.db.Model(task).Updates(map[string]any{"last_error": "old", "run_count": 3, "last_run_at": time.Now()})
|
||
mustCreate(t, tasks.db, &model.TaskLog{TaskID: task.ID, Message: "old"})
|
||
mustCreate(t, tasks.db, &model.Setting{Key: deadAliasKey(task.ID), Value: `["target"]`})
|
||
out = append(out, *task)
|
||
}
|
||
return out
|
||
}
|
||
|
||
func assertDeletedTasks(t *testing.T, db *gorm.DB, tasks *TaskService, deleted []model.Task) {
|
||
t.Helper()
|
||
for _, task := range deleted {
|
||
assertCount(t, db, &model.Task{}, "id = ?", []any{task.ID}, 0)
|
||
assertCount(t, db, &model.TaskLog{}, "task_id = ?", []any{task.ID}, 0)
|
||
assertCount(t, db, &model.Setting{}, "key = ?", []any{deadAliasKey(task.ID)}, 0)
|
||
if _, ok := tasks.entries[task.ID]; ok {
|
||
t.Errorf("task %d still scheduled", task.ID)
|
||
}
|
||
}
|
||
}
|
||
|
||
func assertRewrittenTasks(t *testing.T, db *gorm.DB, tasks []model.Task) {
|
||
t.Helper()
|
||
wantPayload := []string{`{"ociConfigIds":[2]}`, `{"ociConfigIds":[]}`, `{"ociConfigIds":[2]}`}
|
||
for i, original := range tasks {
|
||
var got model.Task
|
||
if err := db.First(&got, original.ID).Error; err != nil {
|
||
t.Fatalf("load task %d: %v", original.ID, err)
|
||
}
|
||
if got.Payload != wantPayload[i] {
|
||
t.Errorf("task %d payload = %s, want %s", got.ID, got.Payload, wantPayload[i])
|
||
}
|
||
wantHistory := int64(0)
|
||
if original.Name == "other" {
|
||
wantHistory = 1
|
||
} else if got.RunCount != 0 || got.LastRunAt != nil || got.LastError != "" {
|
||
t.Errorf("task %d history fields not reset: %+v", got.ID, got)
|
||
}
|
||
assertCount(t, db, &model.TaskLog{}, "task_id = ?", []any{got.ID}, wantHistory)
|
||
assertCount(t, db, &model.Setting{}, "key = ?", []any{deadAliasKey(got.ID)}, wantHistory)
|
||
}
|
||
}
|
||
|
||
func TestDeleteTenantRollback(t *testing.T) {
|
||
configs, tasks, db := newTenantDeleteEnv(t, &fakeClient{})
|
||
target, _ := seedDeleteTenants(t, db)
|
||
mustCreate(t, db, &model.CheckSnapshot{OciConfigID: target.ID})
|
||
task := createHealthTask(t, tasks, target.ID)
|
||
seedTaskHistory(t, db, task.ID)
|
||
registerDeleteFailure(t, db, "cost_snapshots")
|
||
err := configs.Delete(context.Background(), target.ID)
|
||
if err == nil || !errors.Is(err, errInjectedTenantDelete) {
|
||
t.Fatalf("delete error = %v, want injected failure", err)
|
||
}
|
||
assertCount(t, db, &model.OciConfig{}, "id = ?", []any{target.ID}, 1)
|
||
assertCount(t, db, &model.CheckSnapshot{}, "oci_config_id = ?", []any{target.ID}, 1)
|
||
assertCount(t, db, &model.Task{}, "id = ?", []any{task.ID}, 1)
|
||
assertCount(t, db, &model.TaskLog{}, "task_id = ?", []any{task.ID}, 1)
|
||
assertCount(t, db, &model.Setting{}, "key = ?", []any{deadAliasKey(task.ID)}, 1)
|
||
}
|
||
|
||
func TestDeleteTenantWaitsForRunningTask(t *testing.T) {
|
||
client := &blockingClient{fakeClient: &fakeClient{}, started: make(chan struct{}, 1), release: make(chan struct{})}
|
||
configs, tasks, db := newTenantDeleteEnv(t, client)
|
||
target, _ := seedDeleteTenants(t, db)
|
||
setTenantPrivateKey(t, configs, target.ID)
|
||
task := createHealthTask(t, tasks, target.ID)
|
||
executed := make(chan struct{})
|
||
go func() {
|
||
tasks.execute(task.ID)
|
||
close(executed)
|
||
}()
|
||
<-client.started
|
||
deleted := make(chan error, 1)
|
||
go func() { deleted <- configs.Delete(context.Background(), target.ID) }()
|
||
assertDeleteBlocked(t, deleted)
|
||
close(client.release)
|
||
<-executed
|
||
if err := <-deleted; err != nil {
|
||
t.Fatalf("delete tenant: %v", err)
|
||
}
|
||
assertCount(t, db, &model.Task{}, "id = ?", []any{task.ID}, 0)
|
||
assertCount(t, db, &model.TaskLog{}, "task_id = ?", []any{task.ID}, 0)
|
||
}
|
||
|
||
func TestVerifyDoesNotResurrectDeletedTenant(t *testing.T) {
|
||
client := &blockingClient{fakeClient: &fakeClient{}, started: make(chan struct{}, 1), release: make(chan struct{})}
|
||
configs, _, db := newTenantDeleteEnv(t, client)
|
||
target, _ := seedDeleteTenants(t, db)
|
||
setTenantPrivateKey(t, configs, target.ID)
|
||
verified := make(chan error, 1)
|
||
go func() {
|
||
_, _, err := configs.Verify(context.Background(), target.ID)
|
||
verified <- err
|
||
}()
|
||
<-client.started
|
||
if err := configs.Delete(context.Background(), target.ID); err != nil {
|
||
t.Fatalf("delete tenant: %v", err)
|
||
}
|
||
close(client.release)
|
||
if err := <-verified; !errors.Is(err, gorm.ErrRecordNotFound) {
|
||
t.Fatalf("verify error = %v, want record not found", err)
|
||
}
|
||
assertCount(t, db, &model.OciConfig{}, "id = ?", []any{target.ID}, 0)
|
||
}
|
||
|
||
func TestScopeCacheRejectsDeletedTenant(t *testing.T) {
|
||
configs, _, db := newTenantDeleteEnv(t, &fakeClient{})
|
||
target, _ := seedDeleteTenants(t, db)
|
||
if err := configs.Delete(context.Background(), target.ID); err != nil {
|
||
t.Fatalf("delete tenant: %v", err)
|
||
}
|
||
err := configs.saveRegionCache(context.Background(), target.ID,
|
||
[]oci.RegionSubscription{{Key: "PHX", Name: "us-phoenix-1"}})
|
||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||
t.Fatalf("save region cache error = %v, want record not found", err)
|
||
}
|
||
err = configs.saveCompartmentCache(context.Background(), target.ID,
|
||
[]oci.Compartment{{ID: "compartment", Name: "deleted"}})
|
||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||
t.Fatalf("save compartment cache error = %v, want record not found", err)
|
||
}
|
||
assertCount(t, db, &model.RegionCache{}, "oci_config_id = ?", []any{target.ID}, 0)
|
||
assertCount(t, db, &model.CompartmentCache{}, "oci_config_id = ?", []any{target.ID}, 0)
|
||
}
|
||
|
||
func setTenantPrivateKey(t *testing.T, configs *OciConfigService, id uint) {
|
||
t.Helper()
|
||
encrypted, err := configs.cipher.EncryptString("private-key")
|
||
if err != nil {
|
||
t.Fatalf("encrypt private key: %v", err)
|
||
}
|
||
if err := configs.db.Model(&model.OciConfig{}).Where("id = ?", id).
|
||
Update("private_key_enc", encrypted).Error; err != nil {
|
||
t.Fatalf("set private key: %v", err)
|
||
}
|
||
}
|
||
|
||
func createHealthTask(t *testing.T, tasks *TaskService, cfgID uint) *model.Task {
|
||
t.Helper()
|
||
payload := []byte(fmt.Sprintf(`{"ociConfigIds":[%d]}`, cfgID))
|
||
task, err := tasks.CreateTask(context.Background(), CreateTaskInput{
|
||
Name: "running", Type: model.TaskTypeHealthCheck,
|
||
CronExpr: "0 0 * * *", Payload: payload,
|
||
})
|
||
if err != nil {
|
||
t.Fatalf("create task: %v", err)
|
||
}
|
||
return task
|
||
}
|
||
|
||
func assertDeleteBlocked(t *testing.T, deleted <-chan error) {
|
||
t.Helper()
|
||
select {
|
||
case err := <-deleted:
|
||
t.Fatalf("delete returned before running task finished: %v", err)
|
||
case <-time.After(50 * time.Millisecond):
|
||
}
|
||
}
|
||
|
||
func TestPersistTaskRunDoesNotResurrectDeletedTask(t *testing.T) {
|
||
_, tasks, db := newTenantDeleteEnv(t, &fakeClient{})
|
||
task := &model.Task{Name: "stale", Type: model.TaskTypeCost, Status: model.TaskStatusActive}
|
||
mustCreate(t, db, task)
|
||
stale := *task
|
||
if err := db.Delete(task).Error; err != nil {
|
||
t.Fatalf("delete task: %v", err)
|
||
}
|
||
stored, err := tasks.persistTaskRun(context.Background(), &stale)
|
||
if err != nil || stored {
|
||
t.Fatalf("persist stale task = (%v, %v), want (false, nil)", stored, err)
|
||
}
|
||
assertCount(t, db, &model.Task{}, "id = ?", []any{task.ID}, 0)
|
||
}
|
||
|
||
func TestAiLogsRejectDeletedTenantParents(t *testing.T) {
|
||
configs, _, db := newTenantDeleteEnv(t, &fakeClient{})
|
||
target, _ := seedDeleteTenants(t, db)
|
||
channel := model.AiChannel{Name: "target", OciConfigID: target.ID, Region: "r1"}
|
||
mustCreate(t, db, &channel)
|
||
gw := NewAiGatewayService(db, configs, &fakeClient{})
|
||
callID := gw.LogCall(model.AiCallLog{ChannelID: channel.ID, ChannelName: channel.Name})
|
||
if callID == 0 {
|
||
t.Fatal("initial call log was not created")
|
||
}
|
||
if err := configs.Delete(context.Background(), target.ID); err != nil {
|
||
t.Fatalf("delete tenant: %v", err)
|
||
}
|
||
lateID := gw.LogCall(model.AiCallLog{ChannelID: channel.ID, ChannelName: channel.Name})
|
||
if lateID != 0 {
|
||
t.Errorf("late call ID = %d, want 0", lateID)
|
||
}
|
||
gw.LogContent(model.AiContentLog{CallLogID: callID, RequestBody: "late"})
|
||
gw.LogContent(model.AiContentLog{CallLogID: 0, RequestBody: "orphan"})
|
||
assertCount(t, db, &model.AiCallLog{}, "channel_id = ?", []any{channel.ID}, 0)
|
||
assertCount(t, db, &model.AiContentLog{}, "", nil, 0)
|
||
}
|
||
|
||
var errInjectedTenantDelete = errors.New("injected tenant delete failure")
|
||
|
||
func registerDeleteFailure(t *testing.T, db *gorm.DB, table string) {
|
||
t.Helper()
|
||
name := "test:tenant_delete_failure"
|
||
err := db.Callback().Delete().Before("gorm:delete").Register(name, func(tx *gorm.DB) {
|
||
if tx.Statement.Table == table {
|
||
tx.AddError(errInjectedTenantDelete)
|
||
}
|
||
})
|
||
if err != nil {
|
||
t.Fatalf("register callback: %v", err)
|
||
}
|
||
t.Cleanup(func() { _ = db.Callback().Delete().Remove(name) })
|
||
}
|
||
|
||
func mustCreate(t *testing.T, db *gorm.DB, value any) {
|
||
t.Helper()
|
||
if err := db.Create(value).Error; err != nil {
|
||
t.Fatalf("create %T: %v", value, err)
|
||
}
|
||
}
|
||
|
||
func assertCount(t *testing.T, db *gorm.DB, value any, query string, args []any, want int64) {
|
||
t.Helper()
|
||
q := db.Model(value)
|
||
if query != "" {
|
||
q = q.Where(query, args...)
|
||
}
|
||
var got int64
|
||
if err := q.Count(&got).Error; err != nil {
|
||
t.Fatalf("count %T: %v", value, err)
|
||
}
|
||
if got != want {
|
||
t.Errorf("count %T = %d, want %d", value, got, want)
|
||
}
|
||
}
|
||
|
||
func TestDeleteTenantSkipsCorruptTaskPayload(t *testing.T) {
|
||
configs, _, db := newTenantDeleteEnv(t, &fakeClient{})
|
||
target, _ := seedDeleteTenants(t, db)
|
||
corrupt := model.Task{Name: "corrupt-snatch", Type: model.TaskTypeSnatch, Payload: `{broken`}
|
||
mustCreate(t, db, &corrupt)
|
||
mustCreate(t, db, &model.TaskLog{TaskID: corrupt.ID, Message: "keep"})
|
||
|
||
if err := configs.Delete(context.Background(), target.ID); err != nil {
|
||
t.Fatalf("坏 payload 不应阻断租户删除: %v", err)
|
||
}
|
||
var kept model.Task
|
||
if err := db.First(&kept, corrupt.ID).Error; err != nil || kept.Payload != `{broken` {
|
||
t.Errorf("坏任务应原样保留: %+v, %v", kept, err)
|
||
}
|
||
assertCount(t, db, &model.TaskLog{}, "task_id = ?", []any{corrupt.ID}, 1)
|
||
assertCount(t, db, &model.OciConfig{}, "id = ?", []any{target.ID}, 0)
|
||
}
|
||
|
||
func TestDeleteTenantEventsAffectedRules(t *testing.T) {
|
||
_, _, db := newTenantDeleteEnv(t, &fakeClient{})
|
||
target, other := seedDeleteTenants(t, db)
|
||
seedTenantEvents(t, db, target.ID, other.ID)
|
||
|
||
// 规则 ID 须在删除前取:target-rule 会随租户一并删除
|
||
var targetRule, globalRule model.AlertRule
|
||
db.Where("name = ?", "target-rule").First(&targetRule)
|
||
db.Where("name = ?", "global-rule").First(&globalRule)
|
||
|
||
result := &tenantDeleteResult{}
|
||
err := db.Transaction(func(tx *gorm.DB) error {
|
||
return deleteTenantEvents(tx, target.ID, result)
|
||
})
|
||
if err != nil {
|
||
t.Fatalf("deleteTenantEvents: %v", err)
|
||
}
|
||
// target-rule 属本租户,global-rule 的命中引用了本租户事件:都应回收冷却
|
||
got := map[uint]bool{}
|
||
for _, id := range result.alertRuleIDs {
|
||
got[id] = true
|
||
}
|
||
if len(got) != 2 || !got[targetRule.ID] || !got[globalRule.ID] {
|
||
t.Errorf("alertRuleIDs = %v, want 含 target-rule(%d) 与 global-rule(%d)", result.alertRuleIDs, targetRule.ID, globalRule.ID)
|
||
}
|
||
// 仅保留 global-rule × other-event 一条命中
|
||
assertCount(t, db, &model.AlertRuleHit{}, "", nil, 1)
|
||
}
|
||
|
||
func TestDeleteTenantManyEventsNoVarLimit(t *testing.T) {
|
||
// 回归:事件数超 SQLite 绑定变量上限(32766)时删除仍成功(旧实现 IN 展开必失败)
|
||
configs, _, db := newTenantDeleteEnv(t, &fakeClient{})
|
||
target, _ := seedDeleteTenants(t, db)
|
||
events := make([]model.LogEvent, 0, 33000)
|
||
for i := 0; i < 33000; i++ {
|
||
events = append(events, model.LogEvent{OciConfigID: target.ID, MessageID: fmt.Sprintf("bulk-%d", i)})
|
||
}
|
||
if err := db.CreateInBatches(&events, 500).Error; err != nil {
|
||
t.Fatalf("seed events: %v", err)
|
||
}
|
||
if err := configs.Delete(context.Background(), target.ID); err != nil {
|
||
t.Fatalf("数万事件时删除不应受绑定变量上限影响: %v", err)
|
||
}
|
||
assertCount(t, db, &model.LogEvent{}, "oci_config_id = ?", []any{target.ID}, 0)
|
||
}
|