package service import ( "context" "errors" "fmt" "strings" "testing" "time" "oci-portal/internal/aiwire" "oci-portal/internal/model" "oci-portal/internal/oci" ) // stubServiceError 实现 common.ServiceError,用于模拟带状态码的 OCI 服务端错误。 type stubServiceError struct { status int msg string } func (e stubServiceError) Error() string { return "stub service error" } func (e stubServiceError) GetHTTPStatusCode() int { return e.status } func (e stubServiceError) GetMessage() string { if e.msg != "" { return e.msg } return "stub" } func (e stubServiceError) GetCode() string { return "Stub" } func (e stubServiceError) GetOpcRequestID() string { return "req-1" } // finetuneBaseErr 模拟「微调基座模型不可按需调用」的 OCI 400。 func finetuneBaseErr() stubServiceError { return stubServiceError{status: 400, msg: "Not allowed to call finetune base model ocid1.generativeaimodel.oc1.eu-frankfurt-1.tpel5q, use Endpoint: false"} } // gatewayStubClient 覆写 GenAI 四方法;chatErrs 逐次弹出以模拟先失败后成功。 type gatewayStubClient struct { *fakeClient models []oci.GenAiModel modelsErr error probeCode int probeErr error // probeSeq 非空时逐次弹出,模拟按候选依次试调;弹尽后回落 probeCode/probeErr probeSeq []probeResult chatResp *aiwire.ChatResponse chatErrs []error chatCalls int regions []string embedVecs [][]float32 embedUsage *aiwire.Usage embedErr error } func (f *gatewayStubClient) GenAiEmbed(ctx context.Context, cred oci.Credentials, region, modelOcid string, inputs []string, dimensions *int) ([][]float32, *aiwire.Usage, error) { return f.embedVecs, f.embedUsage, f.embedErr } func (f *gatewayStubClient) ListGenAiModels(ctx context.Context, cred oci.Credentials, region string) ([]oci.GenAiModel, error) { return f.models, f.modelsErr } // probeResult 是 gatewayStubClient.probeSeq 的单次探测结果。 type probeResult struct { code int err error } func (f *gatewayStubClient) GenAiProbeChat(ctx context.Context, cred oci.Credentials, region, modelOcid, modelName string) (int, error) { if len(f.probeSeq) > 0 { r := f.probeSeq[0] f.probeSeq = f.probeSeq[1:] return r.code, r.err } return f.probeCode, f.probeErr } func (f *gatewayStubClient) GenAiChat(ctx context.Context, cred oci.Credentials, region, modelOcid string, ir aiwire.ChatRequest) (*aiwire.ChatResponse, error) { f.chatCalls++ f.regions = append(f.regions, region) if len(f.chatErrs) > 0 { err := f.chatErrs[0] f.chatErrs = f.chatErrs[1:] if err != nil { return nil, err } } return f.chatResp, nil } func newTestGateway(t *testing.T, client oci.Client) (*AiGatewayService, *OciConfigService) { t.Helper() svc := newTestService(t, client) if err := svc.db.AutoMigrate(&model.AiKey{}, &model.AiChannel{}, &model.AiModelCache{}, &model.AiCallLog{}); err != nil { t.Fatalf("auto migrate ai tables: %v", err) } return NewAiGatewayService(svc.db, svc, client), svc } func TestAiKeyLifecycle(t *testing.T) { gw, _ := newTestGateway(t, &gatewayStubClient{fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}}) ctx := context.Background() raw, key, err := gw.CreateKey(ctx, "免费-ai-api-key", "test-api-key", "") if err != nil || raw != "test-api-key" || key.Tail != "-key" { t.Fatalf("CreateKey 自定义值 = %q %+v, %v", raw, key, err) } if _, _, err := gw.CreateKey(ctx, "short", "abc", ""); err == nil { t.Error("过短自定义密钥应被拒绝") } auto, _, err := gw.CreateKey(ctx, "auto", "", "") if err != nil || len(auto) < 40 || auto[:3] != "sk-" { t.Fatalf("CreateKey 随机值 = %q, %v", auto, err) } got, err := gw.VerifyKey(ctx, "test-api-key") if err != nil || got.Name != "免费-ai-api-key" { t.Fatalf("VerifyKey = %+v, %v", got, err) } if _, err := gw.VerifyKey(ctx, "wrong"); !errors.Is(err, ErrAiKeyInvalid) { t.Errorf("错误密钥 err = %v", err) } off := false _ = gw.UpdateKey(ctx, key.ID, "", &off, nil) if _, err := gw.VerifyKey(ctx, "test-api-key"); !errors.Is(err, ErrAiKeyInvalid) { t.Errorf("禁用后 VerifyKey err = %v", err) } } func TestAiChannelProbe(t *testing.T) { client := &gatewayStubClient{ fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}, models: []oci.GenAiModel{{Ocid: "ocid1.generativeaimodel.oc1..m1", Name: "meta.llama-3.3-70b-instruct", Vendor: "meta"}}, probeCode: 200, } gw, svc := newTestGateway(t, client) cfg := importAliveConfig(t, svc) ctx := context.Background() ch, err := gw.CreateChannel(ctx, ChannelInput{OciConfigID: cfg.ID, Region: "eu-frankfurt-1"}) if err != nil || ch.Name == "" || !ch.Enabled { t.Fatalf("CreateChannel = %+v, %v", ch, err) } if _, err := gw.CreateChannel(ctx, ChannelInput{OciConfigID: cfg.ID, Region: "eu-frankfurt-1"}); err == nil { t.Error("重复渠道应被拒绝") } probed, err := gw.ProbeChannel(ctx, ch.ID) if err != nil || probed.ProbeStatus != "ok" { t.Fatalf("ProbeChannel = %+v, %v", probed, err) } models, err := gw.channelModels(ctx, ch.ID) if err != nil || len(models) != 1 || models[0].Name != "meta.llama-3.3-70b-instruct" { t.Fatalf("模型缓存未同步: %+v, %v", models, err) } // 配额拒绝 → no_quota client.probeCode, client.probeErr = 403, stubServiceError{status: 403} probed, _ = gw.ProbeChannel(ctx, ch.ID) if probed.ProbeStatus != "no_quota" { t.Errorf("403 探测 status = %q, want no_quota", probed.ProbeStatus) } // 区域无模型 → no_service client.models = nil probed, _ = gw.ProbeChannel(ctx, ch.ID) if probed.ProbeStatus != "no_service" { t.Errorf("空模型探测 status = %q, want no_service", probed.ProbeStatus) } list, _ := gw.GatewayModels(ctx, "") if len(list.Data) != 0 { t.Errorf("no_service 后模型缓存应清空, got %+v", list.Data) } } // seedChannel 直插渠道与模型缓存,绕过探测。 func seedChannel(t *testing.T, gw *AiGatewayService, cfgID uint, region string, priority, weight int) *model.AiChannel { t.Helper() ch := &model.AiChannel{Name: region, OciConfigID: cfgID, Region: region, Enabled: true, Priority: priority, Weight: weight} if err := gw.db.Create(ch).Error; err != nil { t.Fatalf("seed channel: %v", err) } cache := &model.AiModelCache{ChannelID: ch.ID, ModelOcid: "ocid1..m-" + region, Name: "meta.llama-3.3-70b-instruct", Vendor: "meta", SyncedAt: time.Now()} if err := gw.db.Create(cache).Error; err != nil { t.Fatalf("seed cache: %v", err) } return ch } func TestAiChatRetrySwitchesChannel(t *testing.T) { client := &gatewayStubClient{ fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}, chatResp: &aiwire.ChatResponse{Model: "meta.llama-3.3-70b-instruct", Choices: []aiwire.Choice{{Message: aiwire.ChatMessage{Role: "assistant", Content: aiwire.NewTextContent("hi")}, FinishReason: "stop"}}}, chatErrs: []error{stubServiceError{status: 429}}, } gw, svc := newTestGateway(t, client) cfg := importAliveConfig(t, svc) // 两个同优先级渠道 seedChannel(t, gw, cfg.ID, "eu-frankfurt-1", 1, 1) seedChannel(t, gw, cfg.ID, "us-chicago-1", 1, 1) ctx := context.Background() resp, meta, err := gw.Chat(ctx, aiwire.ChatRequest{Model: "meta.llama-3.3-70b-instruct", Messages: []aiwire.ChatMessage{{Role: "user", Content: aiwire.NewTextContent("你好")}}}, "") if err != nil || resp == nil { t.Fatalf("Chat = %v, %v", resp, err) } if meta.Retries != 1 || client.chatCalls != 2 { t.Errorf("应换渠道重试一次: retries=%d calls=%d", meta.Retries, client.chatCalls) } if len(client.regions) != 2 && client.regions[0] == client.regions[1] { t.Errorf("重试未换渠道: %v", client.regions) } // 未知模型 if _, _, err := gw.Chat(ctx, aiwire.ChatRequest{Model: "no-such-model"}, ""); !errors.Is(err, ErrAiUnknownModel) { t.Errorf("未知模型 err = %v", err) } } func TestAiChatNonRetryablePassThrough(t *testing.T) { client := &gatewayStubClient{ fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}, chatErrs: []error{stubServiceError{status: 400}}, } gw, svc := newTestGateway(t, client) cfg := importAliveConfig(t, svc) seedChannel(t, gw, cfg.ID, "eu-frankfurt-1", 1, 1) seedChannel(t, gw, cfg.ID, "us-chicago-1", 1, 1) _, meta, err := gw.Chat(context.Background(), aiwire.ChatRequest{Model: "meta.llama-3.3-70b-instruct"}, "") if err == nil || meta.Retries != 0 || client.chatCalls != 1 { t.Errorf("400 应直接透传不重试: err=%v retries=%d calls=%d", err, meta.Retries, client.chatCalls) } } func TestPickPriorityAndBreaker(t *testing.T) { gw, svc := newTestGateway(t, &gatewayStubClient{fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}}) cfg := importAliveConfig(t, svc) high := seedChannel(t, gw, cfg.ID, "eu-frankfurt-1", 1, 1) seedChannel(t, gw, cfg.ID, "us-chicago-1", 2, 1) ctx := context.Background() cand, err := gw.pick(ctx, "meta.llama-3.3-70b-instruct", "", "CHAT", map[uint]bool{}) if err != nil || cand.ch.ID != high.ID { t.Fatalf("应选高优先级渠道: %+v, %v", cand, err) } // 高优先级熔断 → 降级到低优先级 until := time.Now().Add(10 * time.Minute) gw.db.Model(&model.AiChannel{}).Where("id = ?", high.ID).Update("disabled_until", until) cand, err = gw.pick(ctx, "meta.llama-3.3-70b-instruct", "", "CHAT", map[uint]bool{}) if err != nil || cand.ch.ID == high.ID { t.Fatalf("熔断渠道应被跳过: %+v, %v", cand, err) } // 全部排除 → ErrAiNoChannel _, err = gw.pick(ctx, "meta.llama-3.3-70b-instruct", "", "CHAT", map[uint]bool{cand.ch.ID: true}) if !errors.Is(err, ErrAiNoChannel) { t.Errorf("全排除 err = %v", err) } } func TestMarkFailureBackoff(t *testing.T) { gw, svc := newTestGateway(t, &gatewayStubClient{fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}}) cfg := importAliveConfig(t, svc) ch := seedChannel(t, gw, cfg.ID, "eu-frankfurt-1", 1, 1) ctx := context.Background() for i := 0; i < aiFailThreshold; i++ { gw.markFailure(ctx, ch.ID) } var got model.AiChannel gw.db.First(&got, ch.ID) if got.FailCount != aiFailThreshold || got.DisabledUntil == nil { t.Fatalf("达到阈值应熔断: fail=%d until=%v", got.FailCount, got.DisabledUntil) } gw.markSuccess(ctx, ch.ID) // 新变量重读:gorm 对 NULL 列不覆盖已有值的结构体字段 var reset model.AiChannel gw.db.First(&reset, ch.ID) if reset.FailCount != 0 || reset.DisabledUntil != nil { t.Errorf("成功后应复位: %+v", reset) } } func TestAiGroupRouting(t *testing.T) { client := &gatewayStubClient{ fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}, chatResp: &aiwire.ChatResponse{Model: "meta.llama-3.3-70b-instruct", Choices: []aiwire.Choice{{Message: aiwire.ChatMessage{Role: "assistant", Content: aiwire.NewTextContent("hi")}, FinishReason: "stop"}}}, } gw, svc := newTestGateway(t, client) cfg := importAliveConfig(t, svc) vip := seedChannel(t, gw, cfg.ID, "eu-frankfurt-1", 1, 1) other := seedChannel(t, gw, cfg.ID, "us-chicago-1", 1, 1) gw.db.Model(vip).Update("channel_group", "vip") gw.db.Create(&model.AiModelCache{ChannelID: other.ID, ModelOcid: "ocid1..cr", Name: "cohere.command-r", Vendor: "cohere", SyncedAt: time.Now()}) ctx := context.Background() req := aiwire.ChatRequest{Model: "meta.llama-3.3-70b-instruct", Messages: []aiwire.ChatMessage{{Role: "user", Content: aiwire.NewTextContent("你好")}}} // 分组密钥只落同分组渠道 for i := 0; i < 5; i++ { _, meta, err := gw.Chat(ctx, req, "vip") if err != nil || meta.ChannelID != vip.ID { t.Fatalf("vip 组第 %d 次落点 = %d, err %v, want %d", i, meta.ChannelID, err, vip.ID) } } // 分组内无渠道 → 无可用渠道 if _, _, err := gw.Chat(ctx, req, "nope"); !errors.Is(err, ErrAiNoChannel) { t.Errorf("空分组 err = %v, want ErrAiNoChannel", err) } // 模型列表按分组过滤 list, _ := gw.GatewayModels(ctx, "vip") if len(list.Data) != 1 || list.Data[0].ID != "meta.llama-3.3-70b-instruct" { t.Errorf("vip 组模型 = %+v", list.Data) } all, _ := gw.GatewayModels(ctx, "") if len(all.Data) != 2 { t.Errorf("不限组模型数 = %d, want 2", len(all.Data)) } // 密钥分组落库 _, key, err := gw.CreateKey(ctx, "vip-key", "vip-secret-1234", "vip") if err != nil || key.Group != "vip" { t.Fatalf("CreateKey group = %+v, %v", key, err) } empty := "" _ = gw.UpdateKey(ctx, key.ID, "", nil, &empty) var fresh model.AiKey gw.db.First(&fresh, key.ID) if fresh.Group != "" { t.Errorf("清空分组后 = %q", fresh.Group) } } func TestSyncAiProbeTask(t *testing.T) { client := &gatewayStubClient{ fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}, models: []oci.GenAiModel{{Ocid: "ocid1.generativeaimodel.oc1..m1", Name: "meta.llama-3.3-70b-instruct", Vendor: "meta"}}, probeCode: 200, } gw, svc := newTestGateway(t, client) if err := gw.db.AutoMigrate(&model.Task{}, &model.TaskLog{}, &model.Setting{}); err != nil { t.Fatalf("migrate task tables: %v", err) } tasks := NewTaskService(gw.db, svc, nil, nil) tasks.AttachAiGateway(gw) gw.SetOnChannelsChanged(tasks.SyncAiProbeTask) cfg := importAliveConfig(t, svc) ctx := context.Background() // 创建渠道 → 自动建任务并激活 ch, err := gw.CreateChannel(ctx, ChannelInput{OciConfigID: cfg.ID, Region: "eu-frankfurt-1"}) if err != nil { t.Fatalf("CreateChannel: %v", err) } var task model.Task if err := gw.db.Where("type = ?", model.TaskTypeAiProbe).First(&task).Error; err != nil { t.Fatalf("探测任务未自动创建: %v", err) } if task.Status != model.TaskStatusActive { t.Errorf("任务状态 = %q, want active", task.Status) } if task.CronExpr != "0 0 * * *" { t.Errorf("cron = %q, want 每天 00:00", task.CronExpr) } // 存量旧 cron 自动对齐 gw.db.Model(&task).Update("cron_expr", "*/10 * * * *") tasks.SyncAiProbeTask(ctx) var aligned model.Task gw.db.First(&aligned, task.ID) if aligned.CronExpr != "0 0 * * *" { t.Errorf("存量 cron 未对齐: %q", aligned.CronExpr) } // 手动重复创建被拒 if _, err := tasks.CreateTask(ctx, CreateTaskInput{Name: "dup", Type: model.TaskTypeAiProbe, CronExpr: "*/10 * * * *"}); err == nil { t.Error("重复 AI 探测任务应被拒绝") } // 立即执行一次:探测 ok entry, err := tasks.RunTaskNow(ctx, task.ID) if err != nil || entry == nil || !entry.Success { t.Fatalf("RunTaskNow = %+v, %v", entry, err) } if !strings.Contains(entry.Message, "1 ok") { t.Errorf("探测汇总 = %q", entry.Message) } // 手动删除被拒:任务由系统维护 if err := tasks.DeleteTask(ctx, task.ID); err == nil { t.Error("手动删除 AI 探测任务应被拒绝") } // 渠道归零 → 任务连同日志自动删除 if err := gw.DeleteChannel(ctx, ch.ID); err != nil { t.Fatalf("DeleteChannel: %v", err) } var gone int64 gw.db.Model(&model.Task{}).Where("type = ?", model.TaskTypeAiProbe).Count(&gone) if gone != 0 { t.Errorf("渠道归零后任务应被删除,剩 %d", gone) } var logsLeft int64 gw.db.Model(&model.TaskLog{}).Where("task_id = ?", task.ID).Count(&logsLeft) if logsLeft != 0 { t.Errorf("任务日志应随任务删除,剩 %d", logsLeft) } // 再建渠道 → 自动新建任务 if _, err := gw.CreateChannel(ctx, ChannelInput{OciConfigID: cfg.ID, Region: "us-chicago-1"}); err != nil { t.Fatalf("CreateChannel again: %v", err) } var again model.Task if err := gw.db.Where("type = ?", model.TaskTypeAiProbe).First(&again).Error; err != nil { t.Fatalf("渠道恢复后任务未重建: %v", err) } if again.Status != model.TaskStatusActive { t.Errorf("渠道恢复后任务状态 = %q, want active", again.Status) } } func TestProbeCandidates(t *testing.T) { models := []oci.GenAiModel{ {Ocid: "o1", Name: "xai.grok-voice-agent"}, {Ocid: "o2", Name: "cohere.command-r-plus"}, {Ocid: "o3", Name: "meta.llama-3.3-70b-instruct"}, {Ocid: "o4", Name: "google.gemini-2.5-flash"}, } got := probeCandidates(models) if len(got) != 3 || got[0].Name != "meta.llama-3.3-70b-instruct" || got[1].Name != "google.gemini-2.5-flash" { t.Errorf("候选排序 = %+v", got) } for _, m := range got { if m.Name == "xai.grok-voice-agent" { t.Error("voice 模型不应进候选前 3") } } } func TestProbeCandidatesVendorDiversity(t *testing.T) { // 部分区域单一厂商全为微调基座:候选须跨厂商分散,不能被 3 个 llama 占满前排 models := []oci.GenAiModel{ {Ocid: "l1", Name: "meta.llama-3-70b-instruct"}, {Ocid: "l2", Name: "meta.llama-3.1-405b-instruct"}, {Ocid: "l3", Name: "meta.llama-3.3-70b-instruct"}, {Ocid: "c1", Name: "cohere.command-a-03-2025"}, {Ocid: "g1", Name: "xai.grok-4"}, } got := probeCandidates(models) if len(got) != 5 || got[0].Name != "meta.llama-3-70b-instruct" { t.Fatalf("上限内全量返回且最高分居首: %+v", got) } vendors := map[string]bool{} for _, m := range got[:3] { vendors[modelVendor(m)] = true } if len(vendors) != 3 { t.Errorf("前 3 个候选应覆盖 3 个厂商: %+v", got) } // 超过上限时截断到 probeCandidateCap var many []oci.GenAiModel for i := 0; i < 12; i++ { many = append(many, oci.GenAiModel{Ocid: fmt.Sprintf("m%d", i), Name: fmt.Sprintf("meta.llama-%d", i)}) } if capped := probeCandidates(many); len(capped) != probeCandidateCap { t.Errorf("候选应截断到 %d: got %d", probeCandidateCap, len(capped)) } } // entityNotFoundErr 模拟「实体不存在」404(模型在区域内无按需供给)。 func entityNotFoundErr() stubServiceError { return stubServiceError{status: 404, msg: "Entity with key ocid1.generativeaimodel.oc1.eu-frankfurt-1.2flsfq not found"} } func TestProbeSkipsUnavailableModels(t *testing.T) { // 首选 llama 不可按需调用(基座 400 / 实体 404)→ 标记剔除 → 换候选成功 → 渠道判可用 tests := []struct { name string bad probeResult }{ {"微调基座 400", probeResult{400, finetuneBaseErr()}}, {"实体不存在 404", probeResult{404, entityNotFoundErr()}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { client := &gatewayStubClient{ fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}, models: []oci.GenAiModel{ {Ocid: "m1", Name: "meta.llama-3-70b-instruct", Vendor: "meta"}, {Ocid: "m2", Name: "meta.llama-3.1-70b-instruct", Vendor: "meta"}, {Ocid: "m3", Name: "cohere.command-a-03-2025", Vendor: "cohere"}, }, probeSeq: []probeResult{tt.bad, {200, nil}}, } gw, svc := newTestGateway(t, client) cfg := importAliveConfig(t, svc) ctx := context.Background() ch, err := gw.CreateChannel(ctx, ChannelInput{OciConfigID: cfg.ID, Region: "eu-frankfurt-1"}) if err != nil { t.Fatalf("CreateChannel: %v", err) } probed, err := gw.ProbeChannel(ctx, ch.ID) if err != nil || probed.ProbeStatus != "ok" { t.Fatalf("坏候选后应换候选并判可用: %+v, %v", probed, err) } var row model.AiModelCache if err := gw.db.Where("channel_id = ? AND model_ocid = ?", ch.ID, "m1").First(&row).Error; err != nil || !row.Unusable { t.Fatalf("首个坏候选应被标记不可用: %+v, %v", row, err) } // 再次探测:同步保留标记,m1 不再进候选(probeSeq 只需一次 200) client.probeSeq = []probeResult{{200, nil}} probed, err = gw.ProbeChannel(ctx, ch.ID) if err != nil || probed.ProbeStatus != "ok" { t.Fatalf("复测应跳过已标记模型: %+v, %v", probed, err) } var again model.AiModelCache gw.db.Where("channel_id = ? AND model_ocid = ?", ch.ID, "m1").First(&again) if !again.Unusable { t.Error("探测触发的同步应保留不可用标记") } // 手动同步同样保留标记(坏模型不随重新同步复活);近期已检的标记不被后台验证翻转 client.probeCode = 200 if _, err := gw.SyncModels(ctx, ch.ID); err != nil { t.Fatalf("SyncModels: %v", err) } gw.Wait() var kept model.AiModelCache gw.db.Where("channel_id = ? AND model_ocid = ?", ch.ID, "m1").First(&kept) if !kept.Unusable { t.Error("手动同步不应清除不可用标记") } }) } } func TestValidateModelsAfterSync(t *testing.T) { // 同步后后台验证:坏模型标记剔除、好模型记录已检、其他 4xx 不改状态、非对话模型不试调 client := &gatewayStubClient{ fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}, models: []oci.GenAiModel{ {Ocid: "v1", Name: "meta.llama-4-maverick", Vendor: "meta", Capability: "CHAT"}, {Ocid: "v2", Name: "xai.grok-4", Vendor: "xai", Capability: "CHAT"}, {Ocid: "v3", Name: "xai.grok-voice-agent", Vendor: "xai", Capability: "CHAT"}, {Ocid: "v4", Name: "cohere.embed-v4.0", Vendor: "cohere", Capability: "EMBEDDING"}, }, probeSeq: []probeResult{{404, entityNotFoundErr()}, {200, nil}, {400, stubServiceError{status: 400}}}, } gw, svc := newTestGateway(t, client) cfg := importAliveConfig(t, svc) ctx := context.Background() ch, err := gw.CreateChannel(ctx, ChannelInput{OciConfigID: cfg.ID, Region: "us-ashburn-1"}) if err != nil { t.Fatalf("CreateChannel: %v", err) } if _, err := gw.SyncModels(ctx, ch.ID); err != nil { t.Fatalf("SyncModels: %v", err) } gw.Wait() want := map[string]struct { unusable bool checked bool }{ "v1": {true, true}, // 实体 404 → 标记剔除 "v2": {false, true}, // 200 → 可用已检 "v3": {false, true}, // 普通 400 → 已检不标记 "v4": {false, false}, // EMBEDDING 不试调 } rows, _ := gw.channelModels(ctx, ch.ID) for _, r := range rows { w := want[r.ModelOcid] if r.Unusable != w.unusable || (r.CheckedAt != nil) != w.checked { t.Errorf("%s: unusable=%v checked=%v, want %+v", r.ModelOcid, r.Unusable, r.CheckedAt != nil, w) } } list, err := gw.GatewayModels(ctx, "") if err != nil || len(list.Data) != 3 { t.Errorf("网关列表应只剔除被标记的坏模型(余 v2/v3/v4), got %+v, %v", list.Data, err) } } func TestValidateRecheckUnmarksRecovered(t *testing.T) { // 已标记模型超过复检间隔后重验:恢复供给(200)自动解除标记 client := &gatewayStubClient{fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}, probeCode: 200} gw, svc := newTestGateway(t, client) cfg := importAliveConfig(t, svc) ch := seedChannel(t, gw, cfg.ID, "eu-frankfurt-1", 1, 1) ctx := context.Background() old := time.Now().Add(-25 * time.Hour) gw.db.Model(&model.AiModelCache{}).Where("channel_id = ?", ch.ID). Updates(map[string]any{"unusable": true, "unusable_reason": "x", "checked_at": old}) gw.validateChannelModels(ctx, ch.ID) var row model.AiModelCache gw.db.Where("channel_id = ?", ch.ID).First(&row) if row.Unusable || row.UnusableReason != "" || row.CheckedAt == nil || !row.CheckedAt.After(old) { t.Errorf("超期复检应解除标记并刷新已检时间: %+v", row) } // 未超期的标记不复检(probeSeq 为空、fallback 200 也不会被消费) fresh := time.Now() gw.db.Model(&model.AiModelCache{}).Where("channel_id = ?", ch.ID). Updates(map[string]any{"unusable": true, "checked_at": fresh}) gw.validateChannelModels(ctx, ch.ID) gw.db.Where("channel_id = ?", ch.ID).First(&row) if !row.Unusable { t.Error("未超期的标记不应被复检翻转") } } func TestProbeAuth404StillNoQuota(t *testing.T) { // 鉴权类 404(NotAuthorizedOrNotFound)仍属租户级,直接定论 no_quota 且不标记模型 client := &gatewayStubClient{ fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}, models: []oci.GenAiModel{{Ocid: "m1", Name: "meta.llama-3.3-70b-instruct", Vendor: "meta"}}, probeCode: 404, probeErr: stubServiceError{status: 404, msg: "Authorization failed or requested resource not found."}, } gw, svc := newTestGateway(t, client) cfg := importAliveConfig(t, svc) ctx := context.Background() ch, _ := gw.CreateChannel(ctx, ChannelInput{OciConfigID: cfg.ID, Region: "eu-frankfurt-1"}) probed, _ := gw.ProbeChannel(ctx, ch.ID) if probed.ProbeStatus != "no_quota" { t.Errorf("鉴权 404 status = %q, want no_quota", probed.ProbeStatus) } var marked int64 gw.db.Model(&model.AiModelCache{}).Where("unusable = ?", true).Count(&marked) if marked != 0 { t.Errorf("鉴权 404 不应标记模型, marked=%d", marked) } } func TestAiChatFinetuneSwitchesChannelWithoutPenalty(t *testing.T) { // 微调基座 400 换渠道重试成功,且不计入熔断失败 client := &gatewayStubClient{ fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}, chatResp: &aiwire.ChatResponse{Model: "meta.llama-3.3-70b-instruct", Choices: []aiwire.Choice{{Message: aiwire.ChatMessage{Role: "assistant", Content: aiwire.NewTextContent("hi")}, FinishReason: "stop"}}}, chatErrs: []error{finetuneBaseErr()}, } gw, svc := newTestGateway(t, client) cfg := importAliveConfig(t, svc) seedChannel(t, gw, cfg.ID, "eu-frankfurt-1", 1, 1) seedChannel(t, gw, cfg.ID, "us-chicago-1", 1, 1) resp, meta, err := gw.Chat(context.Background(), aiwire.ChatRequest{Model: "meta.llama-3.3-70b-instruct", Messages: []aiwire.ChatMessage{{Role: "user", Content: aiwire.NewTextContent("你好")}}}, "") if err != nil || resp == nil { t.Fatalf("Chat = %v, %v", resp, err) } if meta.Retries != 1 || client.chatCalls != 2 { t.Errorf("应换渠道重试一次: retries=%d calls=%d", meta.Retries, client.chatCalls) } var chs []model.AiChannel gw.db.Find(&chs) for _, ch := range chs { if ch.FailCount != 0 { t.Errorf("微调基座 400 不应计入熔断: 渠道 %s failCount=%d", ch.Name, ch.FailCount) } } // 失败渠道的该模型被标记,不再参与路由;成功渠道不受影响 region := client.regions[0] var row model.AiModelCache gw.db.Where("model_ocid = ?", "ocid1..m-"+region).First(&row) if !row.Unusable { t.Errorf("失败渠道的模型应被标记不可用: %+v", row) } var usable int64 gw.db.Model(&model.AiModelCache{}).Where("unusable = ?", false).Count(&usable) if usable != 1 { t.Errorf("成功渠道模型应保持可用, usable=%d", usable) } } func TestUnusableModelExcludedFromPoolAndRouting(t *testing.T) { // 唯一渠道的模型被标记后:网关列表不再展示,路由按未知模型拒绝,渠道详情仍可见标记 gw, svc := newTestGateway(t, &gatewayStubClient{fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}}) cfg := importAliveConfig(t, svc) ch := seedChannel(t, gw, cfg.ID, "eu-frankfurt-1", 1, 1) ctx := context.Background() gw.markModelUnusable(ctx, ch.ID, "ocid1..m-eu-frankfurt-1", "Entity with key … not found") list, err := gw.GatewayModels(ctx, "") if err != nil || len(list.Data) != 0 { t.Errorf("已标记模型不应出现在网关列表: %+v, %v", list.Data, err) } if _, _, err := gw.Chat(ctx, aiwire.ChatRequest{Model: "meta.llama-3.3-70b-instruct"}, ""); !errors.Is(err, ErrAiUnknownModel) { t.Errorf("已标记模型路由应拒绝: %v", err) } rows, err := gw.channelModels(ctx, ch.ID) if err != nil || len(rows) != 1 || !rows[0].Unusable || rows[0].UnusableReason == "" { t.Errorf("渠道详情应保留标记行便于排查: %+v, %v", rows, err) } } func TestDeprecatingModels(t *testing.T) { gw, _ := newTestGateway(t, &gatewayStubClient{fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}}) ctx := context.Background() now := time.Now() soon := now.Add(10 * 24 * time.Hour) far := now.Add(90 * 24 * time.Hour) past := now.Add(-30 * 24 * time.Hour) retireSoon := now.Add(15 * 24 * time.Hour) rows := []model.AiModelCache{ {ChannelID: 1, Name: "meta.llama-old", SyncedAt: now, DeprecatedAt: &soon}, {ChannelID: 2, Name: "meta.llama-old", SyncedAt: now, DeprecatedAt: &soon}, // 跨渠道去重 {ChannelID: 1, Name: "xai.grok-4.3", SyncedAt: now, DeprecatedAt: &far}, // 弃用窗口外 {ChannelID: 1, Name: "cohere.command-latest", SyncedAt: now}, // 未宣布 {ChannelID: 1, Name: "xai.grok-3", SyncedAt: now, DeprecatedAt: &past}, // 已过弃用日仍可用:不再告警 {ChannelID: 1, Name: "meta.llama-3.2-11b", SyncedAt: now, DeprecatedAt: &past, RetiredAt: &retireSoon}, // 即将退役:重点告警 {ChannelID: 1, Name: "cohere.embed-img", SyncedAt: now, DeprecatedAt: &past, RetiredAt: &far}, // 退役窗口外 } if err := gw.db.Create(&rows).Error; err != nil { t.Fatalf("seed: %v", err) } names, err := gw.DeprecatingModels(ctx, 30*24*time.Hour) if err != nil || len(names) != 2 { t.Fatalf("DeprecatingModels = %v, %v, want 2 条", names, err) } joined := strings.Join(names, "\n") if !strings.Contains(joined, "meta.llama-3.2-11b(") || !strings.Contains(joined, "退役,届时无法调用") { t.Errorf("缺少退役告警: %v", names) } if !strings.Contains(joined, "meta.llama-old(") || !strings.Contains(joined, "退役前仍可调用") { t.Errorf("缺少弃用预告: %v", names) } if strings.Contains(joined, "xai.grok-3(") { t.Errorf("已过弃用日且未近退役的模型不应告警: %v", names) } } func TestAiEmbeddings(t *testing.T) { client := &gatewayStubClient{ fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}, embedVecs: [][]float32{{0.1, 0.2}, {0.3, 0.4}}, embedUsage: &aiwire.Usage{PromptTokens: 6, TotalTokens: 6}, } gw, svc := newTestGateway(t, client) cfg := importAliveConfig(t, svc) ch := seedChannel(t, gw, cfg.ID, "eu-frankfurt-1", 1, 1) gw.db.Create(&model.AiModelCache{ChannelID: ch.ID, ModelOcid: "ocid1..emb", Name: "cohere.embed-v4.0", Vendor: "cohere", Capability: "EMBEDDING", SyncedAt: time.Now()}) ctx := context.Background() resp, meta, err := gw.Embeddings(ctx, aiwire.EmbeddingsRequest{Model: "cohere.embed-v4.0", Input: aiwire.StringList{"a", "b"}}, "") if err != nil || len(resp.Data) != 2 || resp.Data[1].Index != 1 || resp.Usage.TotalTokens != 6 { t.Fatalf("Embeddings = %+v, meta=%+v, %v", resp, meta, err) } if resp.Object != "list" || resp.Data[0].Object != "embedding" { t.Errorf("响应形态 = %+v", resp) } // 对话模型名打 embeddings:能力不匹配 → 未知模型 if _, _, err := gw.Embeddings(ctx, aiwire.EmbeddingsRequest{Model: "meta.llama-3.3-70b-instruct", Input: aiwire.StringList{"a"}}, ""); !errors.Is(err, ErrAiUnknownModel) { t.Errorf("chat 模型走 embeddings err = %v, want ErrAiUnknownModel", err) } // embedding 模型名打 chat:同样未知模型 if _, _, err := gw.Chat(ctx, aiwire.ChatRequest{Model: "cohere.embed-v4.0", Messages: []aiwire.ChatMessage{{Role: "user", Content: aiwire.NewTextContent("x")}}}, ""); !errors.Is(err, ErrAiUnknownModel) { t.Errorf("embedding 模型走 chat err = %v, want ErrAiUnknownModel", err) } } func TestAiContentLogSwitch(t *testing.T) { gw, _ := newTestGateway(t, &gatewayStubClient{fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}}) if err := gw.db.AutoMigrate(&model.AiContentLog{}); err != nil { t.Fatalf("migrate content log: %v", err) } ctx := context.Background() _, key, err := gw.CreateKey(ctx, "k1", "content-key-1234", "") if err != nil { t.Fatalf("CreateKey: %v", err) } if key.ContentLogUntil != nil { t.Error("新密钥内容日志应默认永关") } // 开启 24 小时 fresh, err := gw.UpdateKeyContentLog(ctx, key.ID, 24) if err != nil || fresh.ContentLogUntil == nil || time.Until(*fresh.ContentLogUntil) < 23*time.Hour { t.Fatalf("开启失败: %+v, %v", fresh.ContentLogUntil, err) } // 超上限拒绝 if _, err := gw.UpdateKeyContentLog(ctx, key.ID, 169); err == nil { t.Error("超过 7 天上限应被拒绝") } // 写入与截断(带调用日志关联) mustCreate(t, gw.db, &model.AiCallLog{ID: 42, KeyID: key.ID, ChannelID: 0}) gw.LogContent(model.AiContentLog{CallLogID: 42, KeyID: key.ID, KeyName: "k1", Endpoint: "openai", Model: "m", RequestBody: strings.Repeat("x", 70*1024)}) rows, total, err := gw.ContentLogs(ctx, key.ID, 0, 1, 20) if err != nil || total != 1 || len(rows[0].RequestBody) != 64*1024 { t.Fatalf("ContentLogs = total %d len %d, %v", total, len(rows[0].RequestBody), err) } // 按调用日志 ID 反查:命中与不命中 if rows, total, err = gw.ContentLogs(ctx, 0, 42, 1, 20); err != nil || total != 1 || rows[0].CallLogID != 42 { t.Fatalf("按 callLogId 反查 = total %d, %v", total, err) } if _, total, err = gw.ContentLogs(ctx, 0, 999, 1, 20); err != nil || total != 0 { t.Fatalf("callLogId 不命中应为空 = total %d, %v", total, err) } // 关闭 fresh, err = gw.UpdateKeyContentLog(ctx, key.ID, 0) if err != nil || fresh.ContentLogUntil != nil { t.Fatalf("关闭失败: %+v, %v", fresh.ContentLogUntil, err) } } func TestLegacyNullUnusableRowsStayVisible(t *testing.T) { // 升级路径回归:AutoMigrate 加列后、首次重同步前,存量行 unusable 为 NULL, // 网关列表与路由必须照常包含这些行,不能因 unusable = false 过滤而整池消失 gw, svc := newTestGateway(t, &gatewayStubClient{fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}}) cfg := importAliveConfig(t, svc) ch := seedChannel(t, gw, cfg.ID, "eu-frankfurt-1", 1, 1) ctx := context.Background() if err := gw.db.Exec("UPDATE ai_model_caches SET unusable = NULL WHERE channel_id = ?", ch.ID).Error; err != nil { t.Fatalf("set legacy null: %v", err) } list, err := gw.GatewayModels(ctx, "") if err != nil || len(list.Data) != 1 { t.Errorf("存量 NULL 行应仍在网关列表: %+v, %v", list.Data, err) } _, ids, err := gw.modelChannels(ctx, "meta.llama-3.3-70b-instruct", "CHAT") if err != nil || len(ids) != 1 { t.Errorf("存量 NULL 行应仍参与路由: %v, %v", ids, err) } }