发布 0.2.0:模型池自愈、探测修正、任务异步触发与删除加固

2026-07-10 20:25:37 +08:00
parent dbba1f4905
commit 7706f59549
19 changed files with 978 additions and 107 deletions
@@ -30,6 +30,10 @@ const (
 	aiBackoffCap    = 30 * time.Minute
 	// aiKeyTouchGap 是 LastUsedAt 的最小写库间隔,避免高频调用刷库
 	aiKeyTouchGap = time.Minute
+	// modelRecheckGap 是已标记不可用模型的复检间隔(恢复供给自动解除标记);
+	// validateBatchCap 限制单渠道单轮验证的试调次数
+	modelRecheckGap  = 20 * time.Hour
+	validateBatchCap = 32
 	// 内容日志(红线例外)约束:开启必须限时(上限 7 天),正文截断,短保留
 	aiContentLogMaxHours  = 168
 	aiContentLogRetention = 7 * 24 * time.Hour
@@ -55,13 +59,17 @@ type AiGatewayService struct {
 	// touchMu 保护各密钥的最近触达时间(内存节流,不追求跨实例精确)
 	touchMu   sync.Mutex
 	lastTouch map[uint]time.Time
+	// validateMu 保护 validating:同一渠道的模型验证不并发
+	validateMu sync.Mutex
+	validating map[uint]bool
 	// onChannelsChanged 在渠道增删后触发,由 main 装配为探测任务同步钩子
 	onChannelsChanged func(context.Context)
 }

 // NewAiGatewayService 组装依赖;调用 StartCleanup 后开始调用日志周期清理。
 func NewAiGatewayService(db *gorm.DB, configs *OciConfigService, client oci.Client) *AiGatewayService {
-	return &AiGatewayService{db: db, configs: configs, client: client, lastTouch: map[uint]time.Time{}}
+	return &AiGatewayService{db: db, configs: configs, client: client,
+		lastTouch: map[uint]time.Time{}, validating: map[uint]bool{}}
 }

 // SetOnChannelsChanged 注册渠道数量变化钩子(渠道创建/删除成功后调用)。
@@ -290,7 +298,7 @@ func (s *AiGatewayService) ProbeChannel(ctx context.Context, id uint) (*model.Ai
 	return &fresh, nil
 }

-// probe 执行探测并返回 (状态, 错误摘要);同时完成模型缓存同步。
+// probe 执行探测并返回 (状态, 错误摘要);同时完成模型缓存同步(保留不可用标记)。
 func (s *AiGatewayService) probe(ctx context.Context, cred oci.Credentials, ch *model.AiChannel) (string, string) {
 	models, err := s.client.ListGenAiModels(ctx, cred, ch.Region)
 	if err != nil {
@@ -303,43 +311,209 @@ func (s *AiGatewayService) probe(ctx context.Context, cred oci.Credentials, ch *
 	if err := s.replaceModels(ctx, ch.ID, models); err != nil {
 		return "error", truncateErr(err.Error())
 	}
-	return s.probeChat(ctx, cred, ch, models)
+	usable, err := s.usableModels(ctx, ch.ID, models)
+	if err != nil {
+		return "error", truncateErr(err.Error())
+	}
+	return s.probeChat(ctx, cred, ch, usable)
 }

-// probeChat 按偏好挑选至多 3 个模型依次试调:部分模型元数据标 CHAT 但实际
-// 不可对话(如 voice agent),遇 400/5xx 换下一个;401/403/404 属租户级直接定论。
+// usableModels 过滤掉缓存中已标记不可按需调用的模型,探测候选不再反复踩坑。
+func (s *AiGatewayService) usableModels(ctx context.Context, channelID uint, models []oci.GenAiModel) ([]oci.GenAiModel, error) {
+	marks, err := s.loadModelMarks(ctx, channelID)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]oci.GenAiModel, 0, len(models))
+	for _, m := range models {
+		if !marks[m.Ocid].Unusable {
+			out = append(out, m)
+		}
+	}
+	return out, nil
+}
+
+// probeChat 按候选顺序试调(上限 8):遇「模型不可按需调用」标记剔除并换下一个;
+// 401/403 与鉴权类 404 属租户级直接定论 no_quota;其余错误(部分模型元数据标 CHAT
+// 但实际不可对话,如 voice agent)累计 3 次止损。
 func (s *AiGatewayService) probeChat(ctx context.Context, cred oci.Credentials, ch *model.AiChannel, models []oci.GenAiModel) (string, string) {
 	status, detail := "error", "无可试调对话模型"
+	errBudget := 3
 	for _, m := range probeCandidates(models) {
 		code, err := s.client.GenAiProbeChat(ctx, cred, ch.Region, m.Ocid, m.Name)
 		switch {
 		case code == 200 || code == 429:
 			return "ok", ""
+		case oci.IsModelUnavailable(err):
+			s.markModelUnusable(ctx, ch.ID, m.Ocid, oci.CompactError(err))
+			status, detail = "error", truncateErr(fmt.Sprintf("%s: 不可按需调用,已从模型池剔除", m.Name))
 		case code == 401 || code == 403 || code == 404:
 			return "no_quota", truncateErr(oci.CompactError(err))
 		default:
 			status, detail = "error", truncateErr(fmt.Sprintf("%s: %s", m.Name, oci.CompactError(err)))
+			if errBudget--; errBudget == 0 {
+				return status, detail
+			}
 		}
 	}
 	return status, detail
 }

-// probeCandidates 只取对话模型并按可靠度排序取前 3:主流文本模型优先,
-// voice 等非常规形态殿后(embedding / rerank 已被能力筛选排除)。
+// markModelUnusable 把 (渠道, 模型OCID) 标记为不可按需调用;失败仅记日志不阻断主流程。
+func (s *AiGatewayService) markModelUnusable(ctx context.Context, channelID uint, ocid, reason string) {
+	err := s.db.WithContext(ctx).Model(&model.AiModelCache{}).
+		Where("channel_id = ? AND model_ocid = ?", channelID, ocid).
+		Updates(map[string]any{"unusable": true, "unusable_reason": shortReason(reason), "checked_at": time.Now()}).Error
+	if err != nil {
+		log.Printf("mark model unusable: %v", err)
+	}
+}
+
+func shortReason(reason string) string {
+	if len(reason) > 200 {
+		return reason[:200]
+	}
+	return reason
+}
+
+// beginValidate 抢占渠道的验证执行权,同渠道同时只跑一轮。
+func (s *AiGatewayService) beginValidate(id uint) bool {
+	s.validateMu.Lock()
+	defer s.validateMu.Unlock()
+	if s.validating[id] {
+		return false
+	}
+	s.validating[id] = true
+	return true
+}
+
+func (s *AiGatewayService) endValidate(id uint) {
+	s.validateMu.Lock()
+	delete(s.validating, id)
+	s.validateMu.Unlock()
+}
+
+// validateTargets 取待验证行:从未验证的新模型,以及标记超过复检间隔的模型(仅对话能力,
+// 兼容存量空串);单轮上限 validateBatchCap 控制调用量。
+func (s *AiGatewayService) validateTargets(ctx context.Context, channelID uint) ([]model.AiModelCache, error) {
+	stale := time.Now().Add(-modelRecheckGap)
+	var rows []model.AiModelCache
+	err := s.db.WithContext(ctx).
+		Where("channel_id = ? AND capability IN ?", channelID, []string{"CHAT", ""}).
+		Where("checked_at IS NULL OR (unusable = ? AND checked_at < ?)", true, stale).
+		Order("id ASC").Limit(validateBatchCap).Find(&rows).Error
+	return rows, err
+}
+
+// validateChannelModels 逐个试调渠道内待验证模型并落结论,把不可按需调用的模型
+// 从池中剔除、把恢复供给的解除标记;ListModels 无字段可事先判别,只能试调习得。
+func (s *AiGatewayService) validateChannelModels(ctx context.Context, channelID uint) {
+	if !s.beginValidate(channelID) {
+		return
+	}
+	defer s.endValidate(channelID)
+	var ch model.AiChannel
+	if err := s.db.WithContext(ctx).First(&ch, channelID).Error; err != nil {
+		return
+	}
+	cred, err := s.configs.credentialsByID(ctx, ch.OciConfigID)
+	if err != nil {
+		return
+	}
+	rows, err := s.validateTargets(ctx, channelID)
+	if err != nil {
+		log.Printf("validate channel %d models: %v", channelID, err)
+		return
+	}
+	for _, r := range rows {
+		s.validateOne(ctx, cred, &ch, r)
+	}
+}
+
+// validateOne 试调单个模型并落库:可用(200/429)清除标记;模型级不可用标记剔除;
+// 其余 4xx 记录已检不改状态;5xx/网络类瞬态不落 checked_at,下轮再验。
+func (s *AiGatewayService) validateOne(ctx context.Context, cred oci.Credentials, ch *model.AiChannel, r model.AiModelCache) {
+	code, err := s.client.GenAiProbeChat(ctx, cred, ch.Region, r.ModelOcid, r.Name)
+	updates := map[string]any{"checked_at": time.Now()}
+	switch {
+	case code == 200 || code == 429:
+		updates["unusable"], updates["unusable_reason"] = false, ""
+	case oci.IsModelUnavailable(err):
+		updates["unusable"], updates["unusable_reason"] = true, shortReason(oci.CompactError(err))
+	case code == 0 || code >= 500:
+		return
+	}
+	s.db.WithContext(ctx).Model(&model.AiModelCache{}).Where("id = ?", r.ID).Updates(updates)
+}
+
+// validateModelsAsync 后台验证渠道模型池(手动同步后触发,不阻塞接口响应)。
+func (s *AiGatewayService) validateModelsAsync(ctx context.Context, channelID uint) {
+	s.wg.Add(1)
+	go func() {
+		defer s.wg.Done()
+		vctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Minute)
+		defer cancel()
+		s.validateChannelModels(vctx, channelID)
+	}()
+}
+
+// probeCandidateCap 是单次探测的候选上限:模型级不可用会被标记剔除不再重试,
+// 放宽到 8 让一次探测有机会越过整批坏模型找到可用者;其他错误另有 3 次止损预算。
+const probeCandidateCap = 8
+
+// probeCandidates 只取对话模型,按可靠度排序后跨厂商取候选:主流文本模型优先;
+// voice 等负分形态(元数据标 CHAT 但实际不可对话)直接排除,不浪费试调预算;
+// 每厂商先取最高分再按分数补位——部分区域某厂商全为微调基座(调用必失败),
+// 不能让单一厂商占满候选名额拖垮整个渠道的探测结论。
 func probeCandidates(models []oci.GenAiModel) []oci.GenAiModel {
 	var sorted []oci.GenAiModel
 	for _, m := range models {
-		if m.Capability == "" || m.Capability == "CHAT" {
+		if (m.Capability == "" || m.Capability == "CHAT") && probeScore(m.Name) >= 0 {
 			sorted = append(sorted, m)
 		}
 	}
 	sort.SliceStable(sorted, func(i, j int) bool {
 		return probeScore(sorted[i].Name) > probeScore(sorted[j].Name)
 	})
-	if len(sorted) > 3 {
-		sorted = sorted[:3]
+	return diversifyByVendor(sorted, probeCandidateCap)
+}
+
+// diversifyByVendor 从已排序列表先每厂商各取一个,不足 limit 再按原序补位。
+func diversifyByVendor(sorted []oci.GenAiModel, limit int) []oci.GenAiModel {
+	picked := make([]oci.GenAiModel, 0, limit)
+	used := make(map[int]bool)
+	seenVendor := make(map[string]bool)
+	for i, m := range sorted {
+		if len(picked) >= limit {
+			break
+		}
+		if v := modelVendor(m); !seenVendor[v] {
+			seenVendor[v] = true
+			used[i] = true
+			picked = append(picked, m)
+		}
 	}
-	return sorted
+	for i, m := range sorted {
+		if len(picked) >= limit {
+			break
+		}
+		if !used[i] {
+			picked = append(picked, m)
+		}
+	}
+	return picked
+}
+
+// modelVendor 取厂商标识;OCI 未回填 vendor 时退化为模型名「.」前缀。
+func modelVendor(m oci.GenAiModel) string {
+	if m.Vendor != "" {
+		return strings.ToLower(m.Vendor)
+	}
+	name := strings.ToLower(m.Name)
+	if i := strings.Index(name, "."); i > 0 {
+		return name[:i]
+	}
+	return name
 }

 func probeScore(name string) int {
@@ -377,7 +551,8 @@ func truncateErr(msg string) string {
 	return msg
 }

-// SyncModels 重新拉取渠道区域的模型列表并覆盖缓存。
+// SyncModels 重新拉取渠道区域的模型列表并覆盖缓存(标记按 OCID 结转),
+// 随后触发后台验证:新模型逐个试调,不可按需调用的数十秒内从池中剔除。
 func (s *AiGatewayService) SyncModels(ctx context.Context, id uint) ([]model.AiModelCache, error) {
 	var ch model.AiChannel
 	if err := s.db.WithContext(ctx).First(&ch, id).Error; err != nil {
@@ -394,16 +569,26 @@ func (s *AiGatewayService) SyncModels(ctx context.Context, id uint) ([]model.AiM
 	if err := s.replaceModels(ctx, id, models); err != nil {
 		return nil, err
 	}
+	s.validateModelsAsync(ctx, id)
 	return s.channelModels(ctx, id)
 }

-// replaceModels 以事务整组覆盖渠道模型缓存。
+// replaceModels 以事务整组覆盖渠道模型缓存,按 OCID 结转不可用标记与验证时间
+// (OCID 变化视为新条目,自然回到待验证状态)。
 func (s *AiGatewayService) replaceModels(ctx context.Context, channelID uint, models []oci.GenAiModel) error {
+	marks, err := s.loadModelMarks(ctx, channelID)
+	if err != nil {
+		return err
+	}
 	rows := make([]model.AiModelCache, 0, len(models))
 	now := time.Now()
 	for _, m := range models {
-		rows = append(rows, model.AiModelCache{ChannelID: channelID, ModelOcid: m.Ocid, Name: m.Name, Vendor: m.Vendor,
-			Capability: m.Capability, SyncedAt: now, DeprecatedAt: m.Deprecated, RetiredAt: m.Retired})
+		row := model.AiModelCache{ChannelID: channelID, ModelOcid: m.Ocid, Name: m.Name, Vendor: m.Vendor,
+			Capability: m.Capability, SyncedAt: now, DeprecatedAt: m.Deprecated, RetiredAt: m.Retired}
+		if prev, ok := marks[m.Ocid]; ok {
+			row.Unusable, row.UnusableReason, row.CheckedAt = prev.Unusable, prev.UnusableReason, prev.CheckedAt
+		}
+		rows = append(rows, row)
 	}
 	return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
 		if err := tx.Where("channel_id = ?", channelID).Delete(&model.AiModelCache{}).Error; err != nil {
@@ -416,17 +601,33 @@ func (s *AiGatewayService) replaceModels(ctx context.Context, channelID uint, mo
 	})
 }

+// loadModelMarks 取渠道内带标记或已验证过的行(OCID → 行),供同步结转与候选过滤。
+func (s *AiGatewayService) loadModelMarks(ctx context.Context, channelID uint) (map[string]model.AiModelCache, error) {
+	var rows []model.AiModelCache
+	err := s.db.WithContext(ctx).Select("model_ocid", "unusable", "unusable_reason", "checked_at").
+		Where("channel_id = ? AND (unusable = ? OR checked_at IS NOT NULL)", channelID, true).Find(&rows).Error
+	if err != nil {
+		return nil, fmt.Errorf("load model marks: %w", err)
+	}
+	marks := make(map[string]model.AiModelCache, len(rows))
+	for _, r := range rows {
+		marks[r.ModelOcid] = r
+	}
+	return marks, nil
+}
+
 func (s *AiGatewayService) channelModels(ctx context.Context, channelID uint) ([]model.AiModelCache, error) {
 	var rows []model.AiModelCache
 	err := s.db.WithContext(ctx).Where("channel_id = ?", channelID).Order("name ASC").Find(&rows).Error
 	return rows, err
 }

-// GatewayModels 聚合启用渠道的模型(按名称去重),供 /ai/v1/models;
-// group 非空时仅聚合该分组渠道(与密钥分组路由口径一致)。
+// GatewayModels 聚合启用渠道的可用模型(按名称去重,剔除不可按需调用标记),
+// 供 /ai/v1/models;group 非空时仅聚合该分组渠道(与密钥分组路由口径一致)。
 func (s *AiGatewayService) GatewayModels(ctx context.Context, group string) (aiwire.ModelList, error) {
 	q := s.db.WithContext(ctx).
-		Joins("JOIN ai_channels ON ai_channels.id = ai_model_caches.channel_id AND ai_channels.enabled = ?", true)
+		Joins("JOIN ai_channels ON ai_channels.id = ai_model_caches.channel_id AND ai_channels.enabled = ?", true).
+		Where("(ai_model_caches.unusable = ? OR ai_model_caches.unusable IS NULL)", false)
 	if group != "" {
 		q = q.Where("ai_channels.channel_group = ?", group)
 	}
@@ -457,6 +658,7 @@ func (s *AiGatewayService) DeprecatingModels(ctx context.Context, within time.Du
 	err := s.db.WithContext(ctx).
 		Where("(retired_at IS NOT NULL AND retired_at > ? AND retired_at <= ?) OR (deprecated_at IS NOT NULL AND deprecated_at >= ? AND deprecated_at <= ?)",
 			now, deadline, now, deadline).
+		Where("(unusable = ? OR unusable IS NULL)", false).
 		Order("name ASC").Find(&rows).Error
 	if err != nil {
 		return nil, err
@@ -477,7 +679,7 @@ func (s *AiGatewayService) DeprecatingModels(ctx context.Context, within time.Du
 	return out, nil
 }

-// ProbeAll 逐个探测全部渠道,返回状态汇总;供 AI 探测后台任务调用。
+// ProbeAll 逐个探测全部渠道并顺带验证模型池,返回状态汇总;供 AI 探测后台任务调用。
 func (s *AiGatewayService) ProbeAll(ctx context.Context) (string, error) {
 	chs, err := s.Channels(ctx)
 	if err != nil {
@@ -493,6 +695,8 @@ func (s *AiGatewayService) ProbeAll(ctx context.Context) (string, error) {
 			continue
 		}
 		counts[fresh.ProbeStatus]++
+		// 后台任务里同步执行:新模型验证入池、坏模型定期复检,零人工收敛
+		s.validateChannelModels(ctx, ch.ID)
 	}
 	msg := fmt.Sprintf("probed %d: %d ok, %d no_service, %d no_quota, %d error",
 		len(chs), counts["ok"], counts["no_service"], counts["no_quota"], counts["error"])