发布 0.2.0:模型池自愈、探测修正、任务异步触发与删除加固

2026-07-10 20:25:37 +08:00
parent dbba1f4905
commit 7706f59549
19 changed files with 978 additions and 107 deletions
@@ -3,6 +3,7 @@ package service
 import (
 	"context"
 	"errors"
+	"fmt"
 	"strings"
 	"testing"
 	"time"
@@ -13,14 +14,28 @@ import (
 )

 // stubServiceError 实现 common.ServiceError,用于模拟带状态码的 OCI 服务端错误。
-type stubServiceError struct{ status int }
+type stubServiceError struct {
+	status int
+	msg    string
+}

-func (e stubServiceError) Error() string           { return "stub service error" }
-func (e stubServiceError) GetHTTPStatusCode() int  { return e.status }
-func (e stubServiceError) GetMessage() string      { return "stub" }
+func (e stubServiceError) Error() string          { return "stub service error" }
+func (e stubServiceError) GetHTTPStatusCode() int { return e.status }
+func (e stubServiceError) GetMessage() string {
+	if e.msg != "" {
+		return e.msg
+	}
+	return "stub"
+}
 func (e stubServiceError) GetCode() string         { return "Stub" }
 func (e stubServiceError) GetOpcRequestID() string { return "req-1" }

+// finetuneBaseErr 模拟「微调基座模型不可按需调用」的 OCI 400。
+func finetuneBaseErr() stubServiceError {
+	return stubServiceError{status: 400,
+		msg: "Not allowed to call finetune base model ocid1.generativeaimodel.oc1.eu-frankfurt-1.tpel5q, use Endpoint: false"}
+}
+
 // gatewayStubClient 覆写 GenAI 四方法;chatErrs 逐次弹出以模拟先失败后成功。
 type gatewayStubClient struct {
 	*fakeClient
@@ -29,6 +44,8 @@ type gatewayStubClient struct {
 	modelsErr error
 	probeCode int
 	probeErr  error
+	// probeSeq 非空时逐次弹出,模拟按候选依次试调;弹尽后回落 probeCode/probeErr
+	probeSeq  []probeResult
 	chatResp  *aiwire.ChatResponse
 	chatErrs  []error
 	chatCalls int
@@ -47,7 +64,18 @@ func (f *gatewayStubClient) ListGenAiModels(ctx context.Context, cred oci.Creden
 	return f.models, f.modelsErr
 }

+// probeResult 是 gatewayStubClient.probeSeq 的单次探测结果。
+type probeResult struct {
+	code int
+	err  error
+}
+
 func (f *gatewayStubClient) GenAiProbeChat(ctx context.Context, cred oci.Credentials, region, modelOcid, modelName string) (int, error) {
+	if len(f.probeSeq) > 0 {
+		r := f.probeSeq[0]
+		f.probeSeq = f.probeSeq[1:]
+		return r.code, r.err
+	}
 	return f.probeCode, f.probeErr
 }

@@ -400,6 +428,263 @@ func TestProbeCandidates(t *testing.T) {
 	}
 }

+func TestProbeCandidatesVendorDiversity(t *testing.T) {
+	// 部分区域单一厂商全为微调基座:候选须跨厂商分散,不能被 3 个 llama 占满前排
+	models := []oci.GenAiModel{
+		{Ocid: "l1", Name: "meta.llama-3-70b-instruct"},
+		{Ocid: "l2", Name: "meta.llama-3.1-405b-instruct"},
+		{Ocid: "l3", Name: "meta.llama-3.3-70b-instruct"},
+		{Ocid: "c1", Name: "cohere.command-a-03-2025"},
+		{Ocid: "g1", Name: "xai.grok-4"},
+	}
+	got := probeCandidates(models)
+	if len(got) != 5 || got[0].Name != "meta.llama-3-70b-instruct" {
+		t.Fatalf("上限内全量返回且最高分居首: %+v", got)
+	}
+	vendors := map[string]bool{}
+	for _, m := range got[:3] {
+		vendors[modelVendor(m)] = true
+	}
+	if len(vendors) != 3 {
+		t.Errorf("前 3 个候选应覆盖 3 个厂商: %+v", got)
+	}
+	// 超过上限时截断到 probeCandidateCap
+	var many []oci.GenAiModel
+	for i := 0; i < 12; i++ {
+		many = append(many, oci.GenAiModel{Ocid: fmt.Sprintf("m%d", i), Name: fmt.Sprintf("meta.llama-%d", i)})
+	}
+	if capped := probeCandidates(many); len(capped) != probeCandidateCap {
+		t.Errorf("候选应截断到 %d: got %d", probeCandidateCap, len(capped))
+	}
+}
+
+// entityNotFoundErr 模拟「实体不存在」404(模型在区域内无按需供给)。
+func entityNotFoundErr() stubServiceError {
+	return stubServiceError{status: 404,
+		msg: "Entity with key ocid1.generativeaimodel.oc1.eu-frankfurt-1.2flsfq not found"}
+}
+
+func TestProbeSkipsUnavailableModels(t *testing.T) {
+	// 首选 llama 不可按需调用(基座 400 / 实体 404)→ 标记剔除 → 换候选成功 → 渠道判可用
+	tests := []struct {
+		name string
+		bad  probeResult
+	}{
+		{"微调基座 400", probeResult{400, finetuneBaseErr()}},
+		{"实体不存在 404", probeResult{404, entityNotFoundErr()}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			client := &gatewayStubClient{
+				fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}},
+				models: []oci.GenAiModel{
+					{Ocid: "m1", Name: "meta.llama-3-70b-instruct", Vendor: "meta"},
+					{Ocid: "m2", Name: "meta.llama-3.1-70b-instruct", Vendor: "meta"},
+					{Ocid: "m3", Name: "cohere.command-a-03-2025", Vendor: "cohere"},
+				},
+				probeSeq: []probeResult{tt.bad, {200, nil}},
+			}
+			gw, svc := newTestGateway(t, client)
+			cfg := importAliveConfig(t, svc)
+			ctx := context.Background()
+
+			ch, err := gw.CreateChannel(ctx, ChannelInput{OciConfigID: cfg.ID, Region: "eu-frankfurt-1"})
+			if err != nil {
+				t.Fatalf("CreateChannel: %v", err)
+			}
+			probed, err := gw.ProbeChannel(ctx, ch.ID)
+			if err != nil || probed.ProbeStatus != "ok" {
+				t.Fatalf("坏候选后应换候选并判可用: %+v, %v", probed, err)
+			}
+			var row model.AiModelCache
+			if err := gw.db.Where("channel_id = ? AND model_ocid = ?", ch.ID, "m1").First(&row).Error; err != nil || !row.Unusable {
+				t.Fatalf("首个坏候选应被标记不可用: %+v, %v", row, err)
+			}
+			// 再次探测:同步保留标记,m1 不再进候选(probeSeq 只需一次 200)
+			client.probeSeq = []probeResult{{200, nil}}
+			probed, err = gw.ProbeChannel(ctx, ch.ID)
+			if err != nil || probed.ProbeStatus != "ok" {
+				t.Fatalf("复测应跳过已标记模型: %+v, %v", probed, err)
+			}
+			var again model.AiModelCache
+			gw.db.Where("channel_id = ? AND model_ocid = ?", ch.ID, "m1").First(&again)
+			if !again.Unusable {
+				t.Error("探测触发的同步应保留不可用标记")
+			}
+			// 手动同步同样保留标记(坏模型不随重新同步复活);近期已检的标记不被后台验证翻转
+			client.probeCode = 200
+			if _, err := gw.SyncModels(ctx, ch.ID); err != nil {
+				t.Fatalf("SyncModels: %v", err)
+			}
+			gw.Wait()
+			var kept model.AiModelCache
+			gw.db.Where("channel_id = ? AND model_ocid = ?", ch.ID, "m1").First(&kept)
+			if !kept.Unusable {
+				t.Error("手动同步不应清除不可用标记")
+			}
+		})
+	}
+}
+
+func TestValidateModelsAfterSync(t *testing.T) {
+	// 同步后后台验证:坏模型标记剔除、好模型记录已检、其他 4xx 不改状态、非对话模型不试调
+	client := &gatewayStubClient{
+		fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}},
+		models: []oci.GenAiModel{
+			{Ocid: "v1", Name: "meta.llama-4-maverick", Vendor: "meta", Capability: "CHAT"},
+			{Ocid: "v2", Name: "xai.grok-4", Vendor: "xai", Capability: "CHAT"},
+			{Ocid: "v3", Name: "xai.grok-voice-agent", Vendor: "xai", Capability: "CHAT"},
+			{Ocid: "v4", Name: "cohere.embed-v4.0", Vendor: "cohere", Capability: "EMBEDDING"},
+		},
+		probeSeq: []probeResult{{404, entityNotFoundErr()}, {200, nil}, {400, stubServiceError{status: 400}}},
+	}
+	gw, svc := newTestGateway(t, client)
+	cfg := importAliveConfig(t, svc)
+	ctx := context.Background()
+	ch, err := gw.CreateChannel(ctx, ChannelInput{OciConfigID: cfg.ID, Region: "us-ashburn-1"})
+	if err != nil {
+		t.Fatalf("CreateChannel: %v", err)
+	}
+	if _, err := gw.SyncModels(ctx, ch.ID); err != nil {
+		t.Fatalf("SyncModels: %v", err)
+	}
+	gw.Wait()
+
+	want := map[string]struct {
+		unusable bool
+		checked  bool
+	}{
+		"v1": {true, true},   // 实体 404 → 标记剔除
+		"v2": {false, true},  // 200 → 可用已检
+		"v3": {false, true},  // 普通 400 → 已检不标记
+		"v4": {false, false}, // EMBEDDING 不试调
+	}
+	rows, _ := gw.channelModels(ctx, ch.ID)
+	for _, r := range rows {
+		w := want[r.ModelOcid]
+		if r.Unusable != w.unusable || (r.CheckedAt != nil) != w.checked {
+			t.Errorf("%s: unusable=%v checked=%v, want %+v", r.ModelOcid, r.Unusable, r.CheckedAt != nil, w)
+		}
+	}
+	list, err := gw.GatewayModels(ctx, "")
+	if err != nil || len(list.Data) != 3 {
+		t.Errorf("网关列表应只剔除被标记的坏模型(余 v2/v3/v4), got %+v, %v", list.Data, err)
+	}
+}
+
+func TestValidateRecheckUnmarksRecovered(t *testing.T) {
+	// 已标记模型超过复检间隔后重验:恢复供给(200)自动解除标记
+	client := &gatewayStubClient{fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}, probeCode: 200}
+	gw, svc := newTestGateway(t, client)
+	cfg := importAliveConfig(t, svc)
+	ch := seedChannel(t, gw, cfg.ID, "eu-frankfurt-1", 1, 1)
+	ctx := context.Background()
+	old := time.Now().Add(-25 * time.Hour)
+	gw.db.Model(&model.AiModelCache{}).Where("channel_id = ?", ch.ID).
+		Updates(map[string]any{"unusable": true, "unusable_reason": "x", "checked_at": old})
+
+	gw.validateChannelModels(ctx, ch.ID)
+	var row model.AiModelCache
+	gw.db.Where("channel_id = ?", ch.ID).First(&row)
+	if row.Unusable || row.UnusableReason != "" || row.CheckedAt == nil || !row.CheckedAt.After(old) {
+		t.Errorf("超期复检应解除标记并刷新已检时间: %+v", row)
+	}
+	// 未超期的标记不复检(probeSeq 为空、fallback 200 也不会被消费)
+	fresh := time.Now()
+	gw.db.Model(&model.AiModelCache{}).Where("channel_id = ?", ch.ID).
+		Updates(map[string]any{"unusable": true, "checked_at": fresh})
+	gw.validateChannelModels(ctx, ch.ID)
+	gw.db.Where("channel_id = ?", ch.ID).First(&row)
+	if !row.Unusable {
+		t.Error("未超期的标记不应被复检翻转")
+	}
+}
+
+func TestProbeAuth404StillNoQuota(t *testing.T) {
+	// 鉴权类 404(NotAuthorizedOrNotFound)仍属租户级,直接定论 no_quota 且不标记模型
+	client := &gatewayStubClient{
+		fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}},
+		models:     []oci.GenAiModel{{Ocid: "m1", Name: "meta.llama-3.3-70b-instruct", Vendor: "meta"}},
+		probeCode:  404,
+		probeErr:   stubServiceError{status: 404, msg: "Authorization failed or requested resource not found."},
+	}
+	gw, svc := newTestGateway(t, client)
+	cfg := importAliveConfig(t, svc)
+	ctx := context.Background()
+
+	ch, _ := gw.CreateChannel(ctx, ChannelInput{OciConfigID: cfg.ID, Region: "eu-frankfurt-1"})
+	probed, _ := gw.ProbeChannel(ctx, ch.ID)
+	if probed.ProbeStatus != "no_quota" {
+		t.Errorf("鉴权 404 status = %q, want no_quota", probed.ProbeStatus)
+	}
+	var marked int64
+	gw.db.Model(&model.AiModelCache{}).Where("unusable = ?", true).Count(&marked)
+	if marked != 0 {
+		t.Errorf("鉴权 404 不应标记模型, marked=%d", marked)
+	}
+}
+
+func TestAiChatFinetuneSwitchesChannelWithoutPenalty(t *testing.T) {
+	// 微调基座 400 换渠道重试成功,且不计入熔断失败
+	client := &gatewayStubClient{
+		fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}},
+		chatResp:   &aiwire.ChatResponse{Model: "meta.llama-3.3-70b-instruct", Choices: []aiwire.Choice{{Message: aiwire.ChatMessage{Role: "assistant", Content: aiwire.NewTextContent("hi")}, FinishReason: "stop"}}},
+		chatErrs:   []error{finetuneBaseErr()},
+	}
+	gw, svc := newTestGateway(t, client)
+	cfg := importAliveConfig(t, svc)
+	seedChannel(t, gw, cfg.ID, "eu-frankfurt-1", 1, 1)
+	seedChannel(t, gw, cfg.ID, "us-chicago-1", 1, 1)
+
+	resp, meta, err := gw.Chat(context.Background(), aiwire.ChatRequest{Model: "meta.llama-3.3-70b-instruct", Messages: []aiwire.ChatMessage{{Role: "user", Content: aiwire.NewTextContent("你好")}}}, "")
+	if err != nil || resp == nil {
+		t.Fatalf("Chat = %v, %v", resp, err)
+	}
+	if meta.Retries != 1 || client.chatCalls != 2 {
+		t.Errorf("应换渠道重试一次: retries=%d calls=%d", meta.Retries, client.chatCalls)
+	}
+	var chs []model.AiChannel
+	gw.db.Find(&chs)
+	for _, ch := range chs {
+		if ch.FailCount != 0 {
+			t.Errorf("微调基座 400 不应计入熔断: 渠道 %s failCount=%d", ch.Name, ch.FailCount)
+		}
+	}
+	// 失败渠道的该模型被标记,不再参与路由;成功渠道不受影响
+	region := client.regions[0]
+	var row model.AiModelCache
+	gw.db.Where("model_ocid = ?", "ocid1..m-"+region).First(&row)
+	if !row.Unusable {
+		t.Errorf("失败渠道的模型应被标记不可用: %+v", row)
+	}
+	var usable int64
+	gw.db.Model(&model.AiModelCache{}).Where("unusable = ?", false).Count(&usable)
+	if usable != 1 {
+		t.Errorf("成功渠道模型应保持可用, usable=%d", usable)
+	}
+}
+
+func TestUnusableModelExcludedFromPoolAndRouting(t *testing.T) {
+	// 唯一渠道的模型被标记后:网关列表不再展示,路由按未知模型拒绝,渠道详情仍可见标记
+	gw, svc := newTestGateway(t, &gatewayStubClient{fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}})
+	cfg := importAliveConfig(t, svc)
+	ch := seedChannel(t, gw, cfg.ID, "eu-frankfurt-1", 1, 1)
+	ctx := context.Background()
+
+	gw.markModelUnusable(ctx, ch.ID, "ocid1..m-eu-frankfurt-1", "Entity with key … not found")
+	list, err := gw.GatewayModels(ctx, "")
+	if err != nil || len(list.Data) != 0 {
+		t.Errorf("已标记模型不应出现在网关列表: %+v, %v", list.Data, err)
+	}
+	if _, _, err := gw.Chat(ctx, aiwire.ChatRequest{Model: "meta.llama-3.3-70b-instruct"}, ""); !errors.Is(err, ErrAiUnknownModel) {
+		t.Errorf("已标记模型路由应拒绝: %v", err)
+	}
+	rows, err := gw.channelModels(ctx, ch.ID)
+	if err != nil || len(rows) != 1 || !rows[0].Unusable || rows[0].UnusableReason == "" {
+		t.Errorf("渠道详情应保留标记行便于排查: %+v, %v", rows, err)
+	}
+}
+
 func TestDeprecatingModels(t *testing.T) {
 	gw, _ := newTestGateway(t, &gatewayStubClient{fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}})
 	ctx := context.Background()
@@ -507,3 +792,24 @@ func TestAiContentLogSwitch(t *testing.T) {
 		t.Fatalf("关闭失败: %+v, %v", fresh.ContentLogUntil, err)
 	}
 }
+
+func TestLegacyNullUnusableRowsStayVisible(t *testing.T) {
+	// 升级路径回归:AutoMigrate 加列后、首次重同步前,存量行 unusable 为 NULL,
+	// 网关列表与路由必须照常包含这些行,不能因 unusable = false 过滤而整池消失
+	gw, svc := newTestGateway(t, &gatewayStubClient{fakeClient: &fakeClient{tenancy: oci.TenancyInfo{Name: "t"}}})
+	cfg := importAliveConfig(t, svc)
+	ch := seedChannel(t, gw, cfg.ID, "eu-frankfurt-1", 1, 1)
+	ctx := context.Background()
+	if err := gw.db.Exec("UPDATE ai_model_caches SET unusable = NULL WHERE channel_id = ?", ch.ID).Error; err != nil {
+		t.Fatalf("set legacy null: %v", err)
+	}
+
+	list, err := gw.GatewayModels(ctx, "")
+	if err != nil || len(list.Data) != 1 {
+		t.Errorf("存量 NULL 行应仍在网关列表: %+v, %v", list.Data, err)
+	}
+	_, ids, err := gw.modelChannels(ctx, "meta.llama-3.3-70b-instruct", "CHAT")
+	if err != nil || len(ids) != 1 {
+		t.Errorf("存量 NULL 行应仍参与路由: %v, %v", ids, err)
+	}
+}