发布 0.1.0:通知渠道、告警规则、令牌版本与安全加固
This commit is contained in:
Wang Defa
@@ -0,0 +1,609 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
"oci-portal/internal/crypto"
|
||||
"oci-portal/internal/model"
|
||||
"oci-portal/internal/oci"
|
||||
)
|
||||
|
||||
type invalidatingClient struct {
|
||||
*fakeClient
|
||||
invalidated []string
|
||||
}
|
||||
|
||||
type blockingClient struct {
|
||||
*fakeClient
|
||||
started chan struct{}
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (c *blockingClient) ValidateKey(context.Context, oci.Credentials) (oci.TenancyInfo, error) {
|
||||
c.started <- struct{}{}
|
||||
<-c.release
|
||||
return oci.TenancyInfo{Name: "target"}, nil
|
||||
}
|
||||
|
||||
func (c *invalidatingClient) InvalidateTenancy(id string) {
|
||||
c.invalidated = append(c.invalidated, id)
|
||||
}
|
||||
|
||||
func newTenantDeleteEnv(t *testing.T, client oci.Client) (*OciConfigService, *TaskService, *gorm.DB) {
|
||||
t.Helper()
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
|
||||
if err != nil {
|
||||
t.Fatalf("open database: %v", err)
|
||||
}
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
t.Fatalf("database handle: %v", err)
|
||||
}
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
migrateTenantDeleteModels(t, db)
|
||||
cipher, err := crypto.NewCipher("test-data-key")
|
||||
if err != nil {
|
||||
t.Fatalf("new cipher: %v", err)
|
||||
}
|
||||
configs := NewOciConfigService(db, cipher, client)
|
||||
tasks := NewTaskService(db, configs, nil, nil)
|
||||
configs.SetTenantCleanupDeps(tasks, nil)
|
||||
return configs, tasks, db
|
||||
}
|
||||
|
||||
func migrateTenantDeleteModels(t *testing.T, db *gorm.DB) {
|
||||
t.Helper()
|
||||
err := db.AutoMigrate(
|
||||
&model.OciConfig{}, &model.Task{}, &model.TaskLog{}, &model.Setting{},
|
||||
&model.CheckSnapshot{}, &model.CostSnapshot{}, &model.RegionCache{}, &model.CompartmentCache{},
|
||||
&model.LogEvent{}, &model.AlertRule{}, &model.AlertRuleHit{},
|
||||
&model.AiChannel{}, &model.AiModelCache{}, &model.AiCallLog{}, &model.AiContentLog{},
|
||||
&model.Proxy{}, &model.AiKey{}, &model.SystemLog{},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("auto migrate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanTenantTask(t *testing.T) {
|
||||
for _, tt := range tenantTaskCases() {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
action, ok, err := planTenantTask(tt.task, 1)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
if ok != tt.wantOK || action.deleteTask != tt.wantDelete {
|
||||
t.Errorf("result = (ok=%v, delete=%v), want (%v, %v)", ok, action.deleteTask, tt.wantOK, tt.wantDelete)
|
||||
}
|
||||
if action.payload != tt.wantPayload {
|
||||
t.Errorf("payload = %q, want %q", action.payload, tt.wantPayload)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type tenantTaskCase struct {
|
||||
name string
|
||||
task model.Task
|
||||
wantOK bool
|
||||
wantDelete bool
|
||||
wantPayload string
|
||||
wantErr bool
|
||||
}
|
||||
|
||||
func tenantTaskCases() []tenantTaskCase {
|
||||
return []tenantTaskCase{
|
||||
{name: "抢机命中", task: taskOf(model.TaskTypeSnatch, `{"ociConfigId":1}`), wantOK: true, wantDelete: true},
|
||||
{name: "抢机未命中", task: taskOf(model.TaskTypeSnatch, `{"ociConfigId":2}`)},
|
||||
{name: "测活全局", task: taskOf(model.TaskTypeHealthCheck, `{"ociConfigIds":[]}`), wantOK: true, wantPayload: `{"ociConfigIds":[]}`},
|
||||
{name: "测活单租户", task: taskOf(model.TaskTypeHealthCheck, `{"ociConfigIds":[1]}`), wantOK: true, wantDelete: true},
|
||||
{name: "测活多租户", task: taskOf(model.TaskTypeHealthCheck, `{"ociConfigIds":[1,2]}`), wantOK: true, wantPayload: `{"ociConfigIds":[2]}`},
|
||||
{name: "成本去重命中", task: taskOf(model.TaskTypeCost, `{"ociConfigIds":[1,1,2]}`), wantOK: true, wantPayload: `{"ociConfigIds":[2]}`},
|
||||
{name: "成本未命中", task: taskOf(model.TaskTypeCost, `{"ociConfigIds":[2]}`)},
|
||||
{name: "非法 JSON", task: taskOf(model.TaskTypeCost, `{`), wantErr: true},
|
||||
}
|
||||
}
|
||||
|
||||
func taskOf(taskType, payload string) model.Task {
|
||||
return model.Task{ID: 10, Type: taskType, Payload: payload}
|
||||
}
|
||||
|
||||
func TestDeleteTenantCleansRelatedRows(t *testing.T) {
|
||||
client := &invalidatingClient{fakeClient: &fakeClient{}}
|
||||
configs, _, db := newTenantDeleteEnv(t, client)
|
||||
target, other := seedDeleteTenants(t, db)
|
||||
seedTenantSnapshots(t, db, target.ID, other.ID)
|
||||
seedTenantEvents(t, db, target.ID, other.ID)
|
||||
seedTenantAI(t, db, target.ID, other.ID)
|
||||
seedRetainedGlobals(t, db)
|
||||
if err := configs.Delete(context.Background(), target.ID); err != nil {
|
||||
t.Fatalf("delete tenant: %v", err)
|
||||
}
|
||||
assertTenantRowsGone(t, db, target.ID)
|
||||
assertOtherTenantRowsRemain(t, db, other.ID)
|
||||
assertRetainedGlobals(t, db)
|
||||
if len(client.invalidated) != 1 || client.invalidated[0] != target.TenancyOCID {
|
||||
t.Errorf("invalidated = %v, want [%s]", client.invalidated, target.TenancyOCID)
|
||||
}
|
||||
}
|
||||
|
||||
func seedDeleteTenants(t *testing.T, db *gorm.DB) (model.OciConfig, model.OciConfig) {
|
||||
t.Helper()
|
||||
target := model.OciConfig{Alias: "target", TenancyOCID: "ocid1.tenancy.target"}
|
||||
other := model.OciConfig{Alias: "other", TenancyOCID: "ocid1.tenancy.other"}
|
||||
mustCreate(t, db, &target)
|
||||
mustCreate(t, db, &other)
|
||||
return target, other
|
||||
}
|
||||
|
||||
func seedTenantSnapshots(t *testing.T, db *gorm.DB, target, other uint) {
|
||||
t.Helper()
|
||||
for _, id := range []uint{target, other} {
|
||||
mustCreate(t, db, &model.CheckSnapshot{OciConfigID: id})
|
||||
mustCreate(t, db, &model.CostSnapshot{OciConfigID: id, Day: "2026-07-10"})
|
||||
mustCreate(t, db, &model.RegionCache{OciConfigID: id, Key: "PHX"})
|
||||
mustCreate(t, db, &model.CompartmentCache{OciConfigID: id, OCID: fmt.Sprintf("comp-%d", id)})
|
||||
mustCreate(t, db, &model.Setting{Key: secretKey(id), Value: fmt.Sprintf("secret-%d", id)})
|
||||
}
|
||||
}
|
||||
|
||||
func seedTenantEvents(t *testing.T, db *gorm.DB, target, other uint) {
|
||||
t.Helper()
|
||||
targetEvent := model.LogEvent{OciConfigID: target, MessageID: "target-event"}
|
||||
otherEvent := model.LogEvent{OciConfigID: other, MessageID: "other-event"}
|
||||
targetRule := model.AlertRule{Name: "target-rule", OciConfigID: target}
|
||||
globalRule := model.AlertRule{Name: "global-rule", OciConfigID: 0}
|
||||
for _, value := range []any{&targetEvent, &otherEvent, &targetRule, &globalRule} {
|
||||
mustCreate(t, db, value)
|
||||
}
|
||||
hits := []model.AlertRuleHit{
|
||||
{RuleID: targetRule.ID, LogEventID: otherEvent.ID},
|
||||
{RuleID: globalRule.ID, LogEventID: targetEvent.ID},
|
||||
{RuleID: globalRule.ID, LogEventID: otherEvent.ID},
|
||||
}
|
||||
for i := range hits {
|
||||
mustCreate(t, db, &hits[i])
|
||||
}
|
||||
}
|
||||
|
||||
func seedTenantAI(t *testing.T, db *gorm.DB, target, other uint) {
|
||||
t.Helper()
|
||||
for _, id := range []uint{target, other} {
|
||||
channel := model.AiChannel{Name: fmt.Sprintf("channel-%d", id), OciConfigID: id, Region: "us-phoenix-1"}
|
||||
mustCreate(t, db, &channel)
|
||||
mustCreate(t, db, &model.AiModelCache{ChannelID: channel.ID, ModelOcid: fmt.Sprintf("model-%d", id)})
|
||||
call := model.AiCallLog{ChannelID: channel.ID, ChannelName: channel.Name}
|
||||
mustCreate(t, db, &call)
|
||||
mustCreate(t, db, &model.AiContentLog{CallLogID: call.ID, RequestBody: "sensitive"})
|
||||
}
|
||||
}
|
||||
|
||||
func seedRetainedGlobals(t *testing.T, db *gorm.DB) {
|
||||
t.Helper()
|
||||
mustCreate(t, db, &model.Proxy{Name: "shared", Type: "http"})
|
||||
mustCreate(t, db, &model.AiKey{Name: "global-key", KeyHash: "hash", Tail: "hash"})
|
||||
mustCreate(t, db, &model.SystemLog{Method: "DELETE", Path: "/api/v1/oci-configs/1"})
|
||||
mustCreate(t, db, &model.Setting{Key: "notify_channels", Value: "[]"})
|
||||
}
|
||||
|
||||
func assertTenantRowsGone(t *testing.T, db *gorm.DB, id uint) {
|
||||
t.Helper()
|
||||
rows := []any{
|
||||
&model.OciConfig{}, &model.CheckSnapshot{}, &model.CostSnapshot{},
|
||||
&model.RegionCache{}, &model.CompartmentCache{}, &model.LogEvent{},
|
||||
&model.AlertRule{}, &model.AiChannel{},
|
||||
}
|
||||
for _, value := range rows {
|
||||
column := "oci_config_id"
|
||||
if _, ok := value.(*model.OciConfig); ok {
|
||||
column = "id"
|
||||
}
|
||||
assertCount(t, db, value, column+" = ?", []any{id}, 0)
|
||||
}
|
||||
assertCount(t, db, &model.Setting{}, "key = ?", []any{secretKey(id)}, 0)
|
||||
assertCount(t, db, &model.AlertRuleHit{}, "", nil, 1)
|
||||
assertCount(t, db, &model.AiModelCache{}, "", nil, 1)
|
||||
assertCount(t, db, &model.AiCallLog{}, "", nil, 1)
|
||||
assertCount(t, db, &model.AiContentLog{}, "", nil, 1)
|
||||
}
|
||||
|
||||
func assertOtherTenantRowsRemain(t *testing.T, db *gorm.DB, id uint) {
|
||||
t.Helper()
|
||||
for _, value := range []any{
|
||||
&model.OciConfig{}, &model.CheckSnapshot{}, &model.CostSnapshot{},
|
||||
&model.RegionCache{}, &model.CompartmentCache{}, &model.LogEvent{},
|
||||
&model.AiChannel{},
|
||||
} {
|
||||
column := "oci_config_id"
|
||||
if _, ok := value.(*model.OciConfig); ok {
|
||||
column = "id"
|
||||
}
|
||||
assertCount(t, db, value, column+" = ?", []any{id}, 1)
|
||||
}
|
||||
assertCount(t, db, &model.Setting{}, "key = ?", []any{secretKey(id)}, 1)
|
||||
assertRemainingIndirectRows(t, db, id)
|
||||
}
|
||||
|
||||
func assertRemainingIndirectRows(t *testing.T, db *gorm.DB, otherID uint) {
|
||||
t.Helper()
|
||||
var channel model.AiChannel
|
||||
if err := db.Where("oci_config_id = ?", otherID).First(&channel).Error; err != nil {
|
||||
t.Fatalf("load other AI channel: %v", err)
|
||||
}
|
||||
assertCount(t, db, &model.AiModelCache{}, "channel_id = ?", []any{channel.ID}, 1)
|
||||
assertCount(t, db, &model.AiCallLog{}, "channel_id = ?", []any{channel.ID}, 1)
|
||||
var call model.AiCallLog
|
||||
if err := db.Where("channel_id = ?", channel.ID).First(&call).Error; err != nil {
|
||||
t.Fatalf("load other AI call: %v", err)
|
||||
}
|
||||
assertCount(t, db, &model.AiContentLog{}, "call_log_id = ?", []any{call.ID}, 1)
|
||||
assertRemainingAlertHit(t, db, otherID)
|
||||
}
|
||||
|
||||
func assertRemainingAlertHit(t *testing.T, db *gorm.DB, otherID uint) {
|
||||
t.Helper()
|
||||
var hit model.AlertRuleHit
|
||||
if err := db.First(&hit).Error; err != nil {
|
||||
t.Fatalf("load remaining alert hit: %v", err)
|
||||
}
|
||||
var rule model.AlertRule
|
||||
var event model.LogEvent
|
||||
if err := db.First(&rule, hit.RuleID).Error; err != nil {
|
||||
t.Fatalf("load remaining rule: %v", err)
|
||||
}
|
||||
if err := db.First(&event, hit.LogEventID).Error; err != nil {
|
||||
t.Fatalf("load remaining event: %v", err)
|
||||
}
|
||||
if rule.OciConfigID != 0 || event.OciConfigID != otherID {
|
||||
t.Errorf("remaining hit = rule cfg %d/event cfg %d, want global/other", rule.OciConfigID, event.OciConfigID)
|
||||
}
|
||||
}
|
||||
|
||||
func assertRetainedGlobals(t *testing.T, db *gorm.DB) {
|
||||
t.Helper()
|
||||
assertCount(t, db, &model.AlertRule{}, "oci_config_id = 0", nil, 1)
|
||||
assertCount(t, db, &model.Proxy{}, "", nil, 1)
|
||||
assertCount(t, db, &model.AiKey{}, "", nil, 1)
|
||||
assertCount(t, db, &model.SystemLog{}, "", nil, 1)
|
||||
assertCount(t, db, &model.Setting{}, "key = ?", []any{"notify_channels"}, 1)
|
||||
}
|
||||
|
||||
func TestDeleteTenantRewritesTasksAndCron(t *testing.T) {
|
||||
configs, tasks, db := newTenantDeleteEnv(t, &fakeClient{})
|
||||
target, other := seedDeleteTenants(t, db)
|
||||
created := seedTenantTasks(t, tasks, target.ID, other.ID)
|
||||
if err := configs.Delete(context.Background(), target.ID); err != nil {
|
||||
t.Fatalf("delete tenant: %v", err)
|
||||
}
|
||||
assertDeletedTasks(t, db, tasks, created[:2])
|
||||
assertRewrittenTasks(t, db, created[2:])
|
||||
}
|
||||
|
||||
func TestDeleteTenantReconcilesAiProbe(t *testing.T) {
|
||||
for _, keepOther := range []bool{false, true} {
|
||||
name := fmt.Sprintf("keepOther=%v", keepOther)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
configs, tasks, db := newTenantDeleteEnv(t, &fakeClient{})
|
||||
target, other := seedDeleteTenants(t, db)
|
||||
seedProbeChannels(t, db, target.ID, other.ID, keepOther)
|
||||
tasks.SyncAiProbeTask(context.Background())
|
||||
probe := loadAiProbe(t, db)
|
||||
seedTaskHistory(t, db, probe.ID)
|
||||
if err := configs.Delete(context.Background(), target.ID); err != nil {
|
||||
t.Fatalf("delete tenant: %v", err)
|
||||
}
|
||||
assertAiProbeResult(t, db, tasks, probe.ID, keepOther)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func seedProbeChannels(t *testing.T, db *gorm.DB, target, other uint, keepOther bool) {
|
||||
t.Helper()
|
||||
mustCreate(t, db, &model.AiChannel{Name: "target", OciConfigID: target, Region: "r1"})
|
||||
if keepOther {
|
||||
mustCreate(t, db, &model.AiChannel{Name: "other", OciConfigID: other, Region: "r1"})
|
||||
}
|
||||
}
|
||||
|
||||
func loadAiProbe(t *testing.T, db *gorm.DB) model.Task {
|
||||
t.Helper()
|
||||
var task model.Task
|
||||
if err := db.Where("type = ?", model.TaskTypeAiProbe).First(&task).Error; err != nil {
|
||||
t.Fatalf("load AI probe: %v", err)
|
||||
}
|
||||
return task
|
||||
}
|
||||
|
||||
func seedTaskHistory(t *testing.T, db *gorm.DB, taskID uint) {
|
||||
t.Helper()
|
||||
updates := map[string]any{"last_error": "old", "run_count": 2, "last_run_at": time.Now()}
|
||||
if err := db.Model(&model.Task{}).Where("id = ?", taskID).Updates(updates).Error; err != nil {
|
||||
t.Fatalf("seed task history: %v", err)
|
||||
}
|
||||
mustCreate(t, db, &model.TaskLog{TaskID: taskID, Message: "old"})
|
||||
mustCreate(t, db, &model.Setting{Key: deadAliasKey(taskID), Value: "old"})
|
||||
}
|
||||
|
||||
func assertAiProbeResult(t *testing.T, db *gorm.DB, tasks *TaskService, id uint, keep bool) {
|
||||
t.Helper()
|
||||
want := int64(0)
|
||||
if keep {
|
||||
want = 1
|
||||
}
|
||||
assertCount(t, db, &model.Task{}, "id = ?", []any{id}, want)
|
||||
assertCount(t, db, &model.TaskLog{}, "task_id = ?", []any{id}, 0)
|
||||
assertCount(t, db, &model.Setting{}, "key = ?", []any{deadAliasKey(id)}, 0)
|
||||
if !keep {
|
||||
if _, ok := tasks.entries[id]; ok {
|
||||
t.Errorf("AI probe %d still scheduled", id)
|
||||
}
|
||||
return
|
||||
}
|
||||
probe := loadAiProbe(t, db)
|
||||
if probe.RunCount != 0 || probe.LastRunAt != nil || probe.LastError != "" {
|
||||
t.Errorf("AI probe history not reset: %+v", probe)
|
||||
}
|
||||
}
|
||||
|
||||
func seedTenantTasks(t *testing.T, tasks *TaskService, target, other uint) []model.Task {
|
||||
t.Helper()
|
||||
inputs := []CreateTaskInput{
|
||||
{Name: "snatch", Type: model.TaskTypeSnatch, CronExpr: "0 0 * * *", Payload: []byte(fmt.Sprintf(`{"ociConfigId":%d,"instance":{"displayName":"vm","region":"r","availabilityDomain":"a","subnetId":"s","shape":"x","imageId":"i"}}`, target))},
|
||||
{Name: "single", Type: model.TaskTypeCost, CronExpr: "0 0 * * *", Payload: []byte(fmt.Sprintf(`{"ociConfigIds":[%d]}`, target))},
|
||||
{Name: "mixed", Type: model.TaskTypeHealthCheck, CronExpr: "0 0 * * *", Payload: []byte(fmt.Sprintf(`{"ociConfigIds":[%d,%d]}`, target, other))},
|
||||
{Name: "global", Type: model.TaskTypeCost, CronExpr: "0 0 * * *", Payload: []byte(`{"ociConfigIds":[]}`)},
|
||||
{Name: "other", Type: model.TaskTypeHealthCheck, CronExpr: "0 0 * * *", Payload: []byte(fmt.Sprintf(`{"ociConfigIds":[%d]}`, other))},
|
||||
}
|
||||
return createTasksWithHistory(t, tasks, inputs)
|
||||
}
|
||||
|
||||
func createTasksWithHistory(t *testing.T, tasks *TaskService, inputs []CreateTaskInput) []model.Task {
|
||||
t.Helper()
|
||||
out := make([]model.Task, 0, len(inputs))
|
||||
for _, input := range inputs {
|
||||
task, err := tasks.CreateTask(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatalf("create task %s: %v", input.Name, err)
|
||||
}
|
||||
tasks.db.Model(task).Updates(map[string]any{"last_error": "old", "run_count": 3, "last_run_at": time.Now()})
|
||||
mustCreate(t, tasks.db, &model.TaskLog{TaskID: task.ID, Message: "old"})
|
||||
mustCreate(t, tasks.db, &model.Setting{Key: deadAliasKey(task.ID), Value: `["target"]`})
|
||||
out = append(out, *task)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func assertDeletedTasks(t *testing.T, db *gorm.DB, tasks *TaskService, deleted []model.Task) {
|
||||
t.Helper()
|
||||
for _, task := range deleted {
|
||||
assertCount(t, db, &model.Task{}, "id = ?", []any{task.ID}, 0)
|
||||
assertCount(t, db, &model.TaskLog{}, "task_id = ?", []any{task.ID}, 0)
|
||||
assertCount(t, db, &model.Setting{}, "key = ?", []any{deadAliasKey(task.ID)}, 0)
|
||||
if _, ok := tasks.entries[task.ID]; ok {
|
||||
t.Errorf("task %d still scheduled", task.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertRewrittenTasks(t *testing.T, db *gorm.DB, tasks []model.Task) {
|
||||
t.Helper()
|
||||
wantPayload := []string{`{"ociConfigIds":[2]}`, `{"ociConfigIds":[]}`, `{"ociConfigIds":[2]}`}
|
||||
for i, original := range tasks {
|
||||
var got model.Task
|
||||
if err := db.First(&got, original.ID).Error; err != nil {
|
||||
t.Fatalf("load task %d: %v", original.ID, err)
|
||||
}
|
||||
if got.Payload != wantPayload[i] {
|
||||
t.Errorf("task %d payload = %s, want %s", got.ID, got.Payload, wantPayload[i])
|
||||
}
|
||||
wantHistory := int64(0)
|
||||
if original.Name == "other" {
|
||||
wantHistory = 1
|
||||
} else if got.RunCount != 0 || got.LastRunAt != nil || got.LastError != "" {
|
||||
t.Errorf("task %d history fields not reset: %+v", got.ID, got)
|
||||
}
|
||||
assertCount(t, db, &model.TaskLog{}, "task_id = ?", []any{got.ID}, wantHistory)
|
||||
assertCount(t, db, &model.Setting{}, "key = ?", []any{deadAliasKey(got.ID)}, wantHistory)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteTenantRollback(t *testing.T) {
|
||||
configs, tasks, db := newTenantDeleteEnv(t, &fakeClient{})
|
||||
target, _ := seedDeleteTenants(t, db)
|
||||
mustCreate(t, db, &model.CheckSnapshot{OciConfigID: target.ID})
|
||||
task := createHealthTask(t, tasks, target.ID)
|
||||
seedTaskHistory(t, db, task.ID)
|
||||
registerDeleteFailure(t, db, "cost_snapshots")
|
||||
err := configs.Delete(context.Background(), target.ID)
|
||||
if err == nil || !errors.Is(err, errInjectedTenantDelete) {
|
||||
t.Fatalf("delete error = %v, want injected failure", err)
|
||||
}
|
||||
assertCount(t, db, &model.OciConfig{}, "id = ?", []any{target.ID}, 1)
|
||||
assertCount(t, db, &model.CheckSnapshot{}, "oci_config_id = ?", []any{target.ID}, 1)
|
||||
assertCount(t, db, &model.Task{}, "id = ?", []any{task.ID}, 1)
|
||||
assertCount(t, db, &model.TaskLog{}, "task_id = ?", []any{task.ID}, 1)
|
||||
assertCount(t, db, &model.Setting{}, "key = ?", []any{deadAliasKey(task.ID)}, 1)
|
||||
}
|
||||
|
||||
func TestDeleteTenantWaitsForRunningTask(t *testing.T) {
|
||||
client := &blockingClient{fakeClient: &fakeClient{}, started: make(chan struct{}, 1), release: make(chan struct{})}
|
||||
configs, tasks, db := newTenantDeleteEnv(t, client)
|
||||
target, _ := seedDeleteTenants(t, db)
|
||||
setTenantPrivateKey(t, configs, target.ID)
|
||||
task := createHealthTask(t, tasks, target.ID)
|
||||
executed := make(chan struct{})
|
||||
go func() {
|
||||
tasks.execute(task.ID)
|
||||
close(executed)
|
||||
}()
|
||||
<-client.started
|
||||
deleted := make(chan error, 1)
|
||||
go func() { deleted <- configs.Delete(context.Background(), target.ID) }()
|
||||
assertDeleteBlocked(t, deleted)
|
||||
close(client.release)
|
||||
<-executed
|
||||
if err := <-deleted; err != nil {
|
||||
t.Fatalf("delete tenant: %v", err)
|
||||
}
|
||||
assertCount(t, db, &model.Task{}, "id = ?", []any{task.ID}, 0)
|
||||
assertCount(t, db, &model.TaskLog{}, "task_id = ?", []any{task.ID}, 0)
|
||||
}
|
||||
|
||||
func TestVerifyDoesNotResurrectDeletedTenant(t *testing.T) {
|
||||
client := &blockingClient{fakeClient: &fakeClient{}, started: make(chan struct{}, 1), release: make(chan struct{})}
|
||||
configs, _, db := newTenantDeleteEnv(t, client)
|
||||
target, _ := seedDeleteTenants(t, db)
|
||||
setTenantPrivateKey(t, configs, target.ID)
|
||||
verified := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, err := configs.Verify(context.Background(), target.ID)
|
||||
verified <- err
|
||||
}()
|
||||
<-client.started
|
||||
if err := configs.Delete(context.Background(), target.ID); err != nil {
|
||||
t.Fatalf("delete tenant: %v", err)
|
||||
}
|
||||
close(client.release)
|
||||
if err := <-verified; !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
t.Fatalf("verify error = %v, want record not found", err)
|
||||
}
|
||||
assertCount(t, db, &model.OciConfig{}, "id = ?", []any{target.ID}, 0)
|
||||
}
|
||||
|
||||
func TestScopeCacheRejectsDeletedTenant(t *testing.T) {
|
||||
configs, _, db := newTenantDeleteEnv(t, &fakeClient{})
|
||||
target, _ := seedDeleteTenants(t, db)
|
||||
if err := configs.Delete(context.Background(), target.ID); err != nil {
|
||||
t.Fatalf("delete tenant: %v", err)
|
||||
}
|
||||
err := configs.saveRegionCache(context.Background(), target.ID,
|
||||
[]oci.RegionSubscription{{Key: "PHX", Name: "us-phoenix-1"}})
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
t.Fatalf("save region cache error = %v, want record not found", err)
|
||||
}
|
||||
err = configs.saveCompartmentCache(context.Background(), target.ID,
|
||||
[]oci.Compartment{{ID: "compartment", Name: "deleted"}})
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
t.Fatalf("save compartment cache error = %v, want record not found", err)
|
||||
}
|
||||
assertCount(t, db, &model.RegionCache{}, "oci_config_id = ?", []any{target.ID}, 0)
|
||||
assertCount(t, db, &model.CompartmentCache{}, "oci_config_id = ?", []any{target.ID}, 0)
|
||||
}
|
||||
|
||||
func setTenantPrivateKey(t *testing.T, configs *OciConfigService, id uint) {
|
||||
t.Helper()
|
||||
encrypted, err := configs.cipher.EncryptString("private-key")
|
||||
if err != nil {
|
||||
t.Fatalf("encrypt private key: %v", err)
|
||||
}
|
||||
if err := configs.db.Model(&model.OciConfig{}).Where("id = ?", id).
|
||||
Update("private_key_enc", encrypted).Error; err != nil {
|
||||
t.Fatalf("set private key: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func createHealthTask(t *testing.T, tasks *TaskService, cfgID uint) *model.Task {
|
||||
t.Helper()
|
||||
payload := []byte(fmt.Sprintf(`{"ociConfigIds":[%d]}`, cfgID))
|
||||
task, err := tasks.CreateTask(context.Background(), CreateTaskInput{
|
||||
Name: "running", Type: model.TaskTypeHealthCheck,
|
||||
CronExpr: "0 0 * * *", Payload: payload,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create task: %v", err)
|
||||
}
|
||||
return task
|
||||
}
|
||||
|
||||
func assertDeleteBlocked(t *testing.T, deleted <-chan error) {
|
||||
t.Helper()
|
||||
select {
|
||||
case err := <-deleted:
|
||||
t.Fatalf("delete returned before running task finished: %v", err)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersistTaskRunDoesNotResurrectDeletedTask(t *testing.T) {
|
||||
_, tasks, db := newTenantDeleteEnv(t, &fakeClient{})
|
||||
task := &model.Task{Name: "stale", Type: model.TaskTypeCost, Status: model.TaskStatusActive}
|
||||
mustCreate(t, db, task)
|
||||
stale := *task
|
||||
if err := db.Delete(task).Error; err != nil {
|
||||
t.Fatalf("delete task: %v", err)
|
||||
}
|
||||
stored, err := tasks.persistTaskRun(context.Background(), &stale)
|
||||
if err != nil || stored {
|
||||
t.Fatalf("persist stale task = (%v, %v), want (false, nil)", stored, err)
|
||||
}
|
||||
assertCount(t, db, &model.Task{}, "id = ?", []any{task.ID}, 0)
|
||||
}
|
||||
|
||||
func TestAiLogsRejectDeletedTenantParents(t *testing.T) {
|
||||
configs, _, db := newTenantDeleteEnv(t, &fakeClient{})
|
||||
target, _ := seedDeleteTenants(t, db)
|
||||
channel := model.AiChannel{Name: "target", OciConfigID: target.ID, Region: "r1"}
|
||||
mustCreate(t, db, &channel)
|
||||
gw := NewAiGatewayService(db, configs, &fakeClient{})
|
||||
callID := gw.LogCall(model.AiCallLog{ChannelID: channel.ID, ChannelName: channel.Name})
|
||||
if callID == 0 {
|
||||
t.Fatal("initial call log was not created")
|
||||
}
|
||||
if err := configs.Delete(context.Background(), target.ID); err != nil {
|
||||
t.Fatalf("delete tenant: %v", err)
|
||||
}
|
||||
lateID := gw.LogCall(model.AiCallLog{ChannelID: channel.ID, ChannelName: channel.Name})
|
||||
if lateID != 0 {
|
||||
t.Errorf("late call ID = %d, want 0", lateID)
|
||||
}
|
||||
gw.LogContent(model.AiContentLog{CallLogID: callID, RequestBody: "late"})
|
||||
gw.LogContent(model.AiContentLog{CallLogID: 0, RequestBody: "orphan"})
|
||||
assertCount(t, db, &model.AiCallLog{}, "channel_id = ?", []any{channel.ID}, 0)
|
||||
assertCount(t, db, &model.AiContentLog{}, "", nil, 0)
|
||||
}
|
||||
|
||||
var errInjectedTenantDelete = errors.New("injected tenant delete failure")
|
||||
|
||||
func registerDeleteFailure(t *testing.T, db *gorm.DB, table string) {
|
||||
t.Helper()
|
||||
name := "test:tenant_delete_failure"
|
||||
err := db.Callback().Delete().Before("gorm:delete").Register(name, func(tx *gorm.DB) {
|
||||
if tx.Statement.Table == table {
|
||||
tx.AddError(errInjectedTenantDelete)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("register callback: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = db.Callback().Delete().Remove(name) })
|
||||
}
|
||||
|
||||
func mustCreate(t *testing.T, db *gorm.DB, value any) {
|
||||
t.Helper()
|
||||
if err := db.Create(value).Error; err != nil {
|
||||
t.Fatalf("create %T: %v", value, err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertCount(t *testing.T, db *gorm.DB, value any, query string, args []any, want int64) {
|
||||
t.Helper()
|
||||
q := db.Model(value)
|
||||
if query != "" {
|
||||
q = q.Where(query, args...)
|
||||
}
|
||||
var got int64
|
||||
if err := q.Count(&got).Error; err != nil {
|
||||
t.Fatalf("count %T: %v", value, err)
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("count %T = %d, want %d", value, got, want)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user