f8fcc81438
- 新增图片可读性预检查机制，识别前先判断图片质量 - 设置置信度阈值为60%，低于阈值自动识别失败 - 支持多图片上传（正面、侧面、辅助图片）提高识别准确度 - 完善识别失败场景的错误分类和用户指导提示 - 新增药品有效期字段支持 - 优化AI提示词，强调安全优先原则 - 更新模型版本为 glm-4.5v 和 glm-4.5-air 数据库变更： - Medication表新增 sideImageUrl, auxiliaryImageUrl, expiryDate 字段 - DTO层同步支持新增字段的传递和更新 质量控制策略： - 图片模糊或不可读时直接返回失败 - 无法识别药品名称时主动失败 - 置信度<60%时拒绝识别，建议重新拍摄 - 宁可识别失败也不提供不准确的药品信息
564 lines
17 KiB
TypeScript
564 lines
17 KiB
TypeScript
import { Injectable, Logger, NotFoundException } from '@nestjs/common';
|
||
import { ConfigService } from '@nestjs/config';
|
||
import { InjectModel } from '@nestjs/sequelize';
|
||
import { OpenAI } from 'openai';
|
||
import { MedicationRecognitionTask } from '../models/medication-recognition-task.model';
|
||
import { CreateRecognitionTaskDto } from '../dto/create-recognition-task.dto';
|
||
import { RecognitionStatusDto } from '../dto/recognition-status.dto';
|
||
import { RecognitionResultDto } from '../dto/recognition-result.dto';
|
||
import {
|
||
RecognitionStatusEnum,
|
||
RECOGNITION_STATUS_DESCRIPTIONS,
|
||
} from '../enums/recognition-status.enum';
|
||
|
||
/**
|
||
* 药物AI识别服务
|
||
* 负责多图片药物识别、分析和结构化数据提取
|
||
*/
|
||
@Injectable()
|
||
export class MedicationRecognitionService {
|
||
private readonly logger = new Logger(MedicationRecognitionService.name);
|
||
private readonly client: OpenAI;
|
||
private readonly visionModel: string;
|
||
private readonly textModel: string;
|
||
|
||
constructor(
|
||
private readonly configService: ConfigService,
|
||
@InjectModel(MedicationRecognitionTask)
|
||
private readonly taskModel: typeof MedicationRecognitionTask,
|
||
) {
|
||
const glmApiKey = this.configService.get<string>('GLM_API_KEY');
|
||
const glmBaseURL =
|
||
this.configService.get<string>('GLM_BASE_URL') ||
|
||
'https://open.bigmodel.cn/api/paas/v4';
|
||
|
||
this.client = new OpenAI({
|
||
apiKey: glmApiKey,
|
||
baseURL: glmBaseURL,
|
||
});
|
||
|
||
this.visionModel =
|
||
this.configService.get<string>('GLM_VISION_MODEL') || 'glm-4.5v';
|
||
this.textModel =
|
||
this.configService.get<string>('GLM_MODEL') || 'glm-4.5-air';
|
||
}
|
||
|
||
/**
|
||
* 创建识别任务
|
||
*/
|
||
async createRecognitionTask(
|
||
userId: string,
|
||
dto: CreateRecognitionTaskDto,
|
||
): Promise<{ taskId: string; status: RecognitionStatusEnum }> {
|
||
const taskId = `task_${userId}_${Date.now()}`;
|
||
|
||
this.logger.log(`创建药物识别任务: ${taskId}, 用户: ${userId}`);
|
||
|
||
await this.taskModel.create({
|
||
id: taskId,
|
||
userId,
|
||
frontImageUrl: dto.frontImageUrl,
|
||
sideImageUrl: dto.sideImageUrl,
|
||
auxiliaryImageUrl: dto.auxiliaryImageUrl,
|
||
status: RecognitionStatusEnum.PENDING,
|
||
currentStep: RECOGNITION_STATUS_DESCRIPTIONS[RecognitionStatusEnum.PENDING],
|
||
progress: 0,
|
||
});
|
||
|
||
// 异步开始识别过程（不阻塞当前请求）
|
||
this.startRecognitionProcess(taskId).catch((error) => {
|
||
this.logger.error(
|
||
`识别任务 ${taskId} 处理失败: ${error instanceof Error ? error.message : String(error)}`,
|
||
);
|
||
});
|
||
|
||
return { taskId, status: RecognitionStatusEnum.PENDING };
|
||
}
|
||
|
||
/**
|
||
* 查询识别状态
|
||
*/
|
||
async getRecognitionStatus(
|
||
taskId: string,
|
||
userId: string,
|
||
): Promise<RecognitionStatusDto> {
|
||
const task = await this.taskModel.findOne({
|
||
where: { id: taskId, userId },
|
||
});
|
||
|
||
if (!task) {
|
||
throw new NotFoundException('识别任务不存在');
|
||
}
|
||
|
||
return {
|
||
taskId: task.id,
|
||
status: task.status as RecognitionStatusEnum,
|
||
currentStep: task.currentStep,
|
||
progress: task.progress,
|
||
result: task.recognitionResult
|
||
? JSON.parse(task.recognitionResult)
|
||
: undefined,
|
||
errorMessage: task.errorMessage,
|
||
createdAt: task.createdAt,
|
||
completedAt: task.completedAt,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 开始识别处理流程
|
||
*/
|
||
private async startRecognitionProcess(taskId: string): Promise<void> {
|
||
try {
|
||
const task = await this.taskModel.findByPk(taskId);
|
||
if (!task) return;
|
||
|
||
// 阶段1: 产品识别分析 (0-40%)
|
||
await this.updateTaskStatus(
|
||
taskId,
|
||
RecognitionStatusEnum.ANALYZING_PRODUCT,
|
||
'正在识别药品基本信息...',
|
||
10,
|
||
);
|
||
const productInfo = await this.recognizeProduct(task);
|
||
await this.updateTaskStatus(
|
||
taskId,
|
||
RecognitionStatusEnum.ANALYZING_PRODUCT,
|
||
'药品基本信息识别完成',
|
||
40,
|
||
);
|
||
|
||
// 阶段2: 适宜人群分析 (40-60%)
|
||
await this.updateTaskStatus(
|
||
taskId,
|
||
RecognitionStatusEnum.ANALYZING_SUITABILITY,
|
||
'正在分析适宜人群...',
|
||
50,
|
||
);
|
||
const suitabilityInfo = await this.analyzeSuitability(productInfo);
|
||
await this.updateTaskStatus(
|
||
taskId,
|
||
RecognitionStatusEnum.ANALYZING_SUITABILITY,
|
||
'适宜人群分析完成',
|
||
60,
|
||
);
|
||
|
||
// 阶段3: 成分分析 (60-80%)
|
||
await this.updateTaskStatus(
|
||
taskId,
|
||
RecognitionStatusEnum.ANALYZING_INGREDIENTS,
|
||
'正在分析主要成分...',
|
||
70,
|
||
);
|
||
const ingredientsInfo = await this.analyzeIngredients(productInfo);
|
||
await this.updateTaskStatus(
|
||
taskId,
|
||
RecognitionStatusEnum.ANALYZING_INGREDIENTS,
|
||
'成分分析完成',
|
||
80,
|
||
);
|
||
|
||
// 阶段4: 副作用分析 (80-100%)
|
||
await this.updateTaskStatus(
|
||
taskId,
|
||
RecognitionStatusEnum.ANALYZING_EFFECTS,
|
||
'正在分析副作用和健康建议...',
|
||
90,
|
||
);
|
||
const effectsInfo = await this.analyzeEffects(productInfo);
|
||
|
||
// 合并所有结果，透传所有原始图片URL（避免被AI模型修改）
|
||
const finalResult = {
|
||
...productInfo,
|
||
...suitabilityInfo,
|
||
...ingredientsInfo,
|
||
...effectsInfo,
|
||
// 强制使用任务记录中存储的原始图片URL，覆盖AI可能返回的不正确链接
|
||
photoUrl: task.frontImageUrl,
|
||
sideImageUrl: task.sideImageUrl,
|
||
auxiliaryImageUrl: task.auxiliaryImageUrl,
|
||
} as RecognitionResultDto;
|
||
|
||
// 完成识别
|
||
await this.completeTask(taskId, finalResult);
|
||
this.logger.log(`识别任务 ${taskId} 完成`);
|
||
} catch (error) {
|
||
const errorMessage =
|
||
error instanceof Error ? error.message : String(error);
|
||
this.logger.error(`识别任务 ${taskId} 失败: ${errorMessage}`);
|
||
await this.failTask(taskId, errorMessage);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 阶段1: 识别药品基本信息
|
||
*/
|
||
private async recognizeProduct(
|
||
task: MedicationRecognitionTask,
|
||
): Promise<Partial<RecognitionResultDto>> {
|
||
const prompt = this.buildProductRecognitionPrompt();
|
||
const images = [task.frontImageUrl, task.sideImageUrl];
|
||
if (task.auxiliaryImageUrl) images.push(task.auxiliaryImageUrl);
|
||
|
||
this.logger.log(
|
||
`调用视觉模型识别药品，图片数量: ${images.length}, 任务ID: ${task.id}`,
|
||
);
|
||
|
||
const response = await this.client.chat.completions.create({
|
||
model: this.visionModel,
|
||
temperature: 0.3,
|
||
messages: [
|
||
{
|
||
role: 'user',
|
||
content: [
|
||
{ type: 'text', text: prompt },
|
||
...images.map((url) => ({
|
||
type: 'image_url',
|
||
image_url: { url },
|
||
})),
|
||
] as any,
|
||
},
|
||
],
|
||
response_format: { type: 'json_object' },
|
||
} as any);
|
||
|
||
const content = response.choices[0]?.message?.content;
|
||
if (!content) {
|
||
throw new Error('AI模型返回内容为空');
|
||
}
|
||
|
||
const parsed = this.parseJsonResponse(content);
|
||
|
||
// 严格的置信度和可读性验证
|
||
const confidence = parsed.confidence || 0;
|
||
const isReadable = parsed.isReadable !== false; // 默认为 true 保持向后兼容
|
||
|
||
this.logger.log(
|
||
`药品识别结果: ${parsed.name}, 置信度: ${confidence}, 可读性: ${isReadable}`,
|
||
);
|
||
|
||
// 如果模型明确表示看不清或置信度过低，则失败
|
||
if (!isReadable) {
|
||
throw new Error('图片模糊或光线不足，无法清晰识别药品信息。请重新拍摄更清晰的照片。');
|
||
}
|
||
|
||
if (parsed.name === '无法识别' || parsed.name === '未知' || !parsed.name) {
|
||
throw new Error('无法从图片中识别出药品名称。请确保药品名称清晰可见，或选择手动输入。');
|
||
}
|
||
|
||
// 置信度阈值检查：低于 0.6 视为不可靠
|
||
if (confidence < 0.6) {
|
||
throw new Error(
|
||
`识别置信度过低 (${(confidence * 100).toFixed(0)}%)。建议重新拍摄更清晰的照片，确保药品名称、规格等信息清晰可见。`,
|
||
);
|
||
}
|
||
|
||
return parsed;
|
||
}
|
||
|
||
/**
|
||
* 阶段2: 分析适宜人群
|
||
*/
|
||
private async analyzeSuitability(
|
||
productInfo: Partial<RecognitionResultDto>,
|
||
): Promise<Partial<RecognitionResultDto>> {
|
||
const prompt = this.buildSuitabilityAnalysisPrompt(productInfo);
|
||
|
||
this.logger.log(`分析适宜人群: ${productInfo.name}`);
|
||
|
||
const response = await this.client.chat.completions.create({
|
||
model: this.textModel,
|
||
temperature: 0.7,
|
||
messages: [
|
||
{
|
||
role: 'user',
|
||
content: prompt,
|
||
},
|
||
],
|
||
response_format: { type: 'json_object' },
|
||
});
|
||
|
||
const content = response.choices[0]?.message?.content;
|
||
if (!content) {
|
||
throw new Error('AI模型返回内容为空');
|
||
}
|
||
|
||
return this.parseJsonResponse(content);
|
||
}
|
||
|
||
/**
|
||
* 阶段3: 分析主要成分
|
||
*/
|
||
private async analyzeIngredients(
|
||
productInfo: Partial<RecognitionResultDto>,
|
||
): Promise<Partial<RecognitionResultDto>> {
|
||
const prompt = this.buildIngredientsAnalysisPrompt(productInfo);
|
||
|
||
this.logger.log(`分析主要成分: ${productInfo.name}`);
|
||
|
||
const response = await this.client.chat.completions.create({
|
||
model: this.textModel,
|
||
temperature: 0.7,
|
||
messages: [
|
||
{
|
||
role: 'user',
|
||
content: prompt,
|
||
},
|
||
],
|
||
response_format: { type: 'json_object' },
|
||
});
|
||
|
||
const content = response.choices[0]?.message?.content;
|
||
if (!content) {
|
||
throw new Error('AI模型返回内容为空');
|
||
}
|
||
|
||
return this.parseJsonResponse(content);
|
||
}
|
||
|
||
/**
|
||
* 阶段4: 分析副作用和健康建议
|
||
*/
|
||
private async analyzeEffects(
|
||
productInfo: Partial<RecognitionResultDto>,
|
||
): Promise<Partial<RecognitionResultDto>> {
|
||
const prompt = this.buildEffectsAnalysisPrompt(productInfo);
|
||
|
||
this.logger.log(`分析副作用和健康建议: ${productInfo.name}`);
|
||
|
||
const response = await this.client.chat.completions.create({
|
||
model: this.textModel,
|
||
temperature: 0.7,
|
||
messages: [
|
||
{
|
||
role: 'user',
|
||
content: prompt,
|
||
},
|
||
],
|
||
response_format: { type: 'json_object' },
|
||
});
|
||
|
||
const content = response.choices[0]?.message?.content;
|
||
if (!content) {
|
||
throw new Error('AI模型返回内容为空');
|
||
}
|
||
|
||
return this.parseJsonResponse(content);
|
||
}
|
||
|
||
/**
|
||
* 构建产品识别提示词
|
||
*/
|
||
private buildProductRecognitionPrompt(): string {
|
||
return `你是一位拥有20年从业经验的资深药剂师，请根据提供的药品图片（包括正面、侧面和可能的辅助面）进行详细分析。
|
||
|
||
**重要前提条件 - 图片可读性判断**：
|
||
⚠️ 在进行任何识别之前，你必须首先判断图片是否足够清晰可读：
|
||
1. 检查图片是否模糊、过曝、欠曝或有严重反光
|
||
2. 检查药品名称、规格等关键信息是否清晰可见
|
||
3. 检查文字是否完整、无遮挡
|
||
4. 如果图片质量不佳，无法清晰辨认关键信息，必须设置 isReadable 为 false
|
||
|
||
**只有在图片清晰可读的情况下才能继续分析**：
|
||
1. 仔细观察药品包装、说明书上的所有信息
|
||
2. 识别药品的完整名称（通用名和商品名）
|
||
3. 确定药物剂型（片剂/胶囊/注射剂等）
|
||
4. 提取规格剂量信息
|
||
5. 推荐合理的服用次数和时间
|
||
|
||
**置信度评估标准（仅在图片可读时评估）**：
|
||
- 如果图片清晰且信息完整，置信度应 >= 0.8
|
||
- 如果部分信息不清晰但大部分可推断，置信度 0.6-0.8
|
||
- 如果关键信息缺失或模糊不清，置信度 < 0.6，name返回"无法识别"
|
||
- 置信度评估必须严格基于实际可见信息，不能猜测或臆断
|
||
|
||
**返回严格的JSON格式**（不要包含任何markdown标记）：
|
||
{
|
||
"isReadable": true或false（图片是否足够清晰可读）,
|
||
"name": "药品完整名称",
|
||
"photoUrl": "使用正面图片URL",
|
||
"form": "剂型(tablet/capsule/injection/drops/syrup/ointment/powder/granules)",
|
||
"dosageValue": 剂量数值(数字),
|
||
"dosageUnit": "剂量单位",
|
||
"timesPerDay": 建议每日服用次数(数字),
|
||
"medicationTimes": ["建议的服药时间，格式HH:mm"],
|
||
"confidence": 识别置信度(0-1的小数)
|
||
}
|
||
|
||
**关键规则（必须遵守）**：
|
||
1. isReadable 是最重要的字段，如果为 false，其他识别结果将被忽略
|
||
2. 当图片模糊、反光、文字不清晰时，必须设置 isReadable 为 false
|
||
3. 只有在确实能看清并理解图片内容时，才能设置 isReadable 为 true
|
||
4. confidence 必须反映真实的识别把握程度，不能虚高
|
||
5. 如果 isReadable 为 false，name 必须返回"无法识别"，confidence 设为 0
|
||
6. dosageValue 和 timesPerDay 必须是数字类型，不要加引号
|
||
7. medicationTimes 必须是 HH:mm 格式的时间数组
|
||
8. form 必须是枚举值之一
|
||
|
||
**宁可识别失败，也不要提供不准确的药品信息。用药安全高于一切！**`;
|
||
}
|
||
|
||
/**
|
||
* 构建适宜人群分析提示词
|
||
*/
|
||
private buildSuitabilityAnalysisPrompt(
|
||
productInfo: Partial<RecognitionResultDto>,
|
||
): string {
|
||
return `作为资深药剂师，请分析以下药品的适宜人群和禁忌人群：
|
||
|
||
**药品信息**：
|
||
- 名称：${productInfo.name}
|
||
- 剂型：${productInfo.form}
|
||
- 剂量：${productInfo.dosageValue}${productInfo.dosageUnit}
|
||
|
||
请以严格的JSON格式返回（不要包含任何markdown标记）：
|
||
{
|
||
"suitableFor": ["适合人群1", "适合人群2", "适合人群3"],
|
||
"unsuitableFor": ["不适合人群1", "不适合人群2", "不适合人群3"],
|
||
"mainUsage": "药品的主要用途和适应症描述"
|
||
}
|
||
|
||
**要求**：
|
||
- suitableFor 和 unsuitableFor 必须是字符串数组，至少包含3项
|
||
- mainUsage 是字符串，描述药品的主要治疗用途
|
||
- 如果无法识别药品，所有数组返回空数组，mainUsage返回"无法识别药品"`;
|
||
}
|
||
|
||
/**
|
||
* 构建成分分析提示词
|
||
*/
|
||
private buildIngredientsAnalysisPrompt(
|
||
productInfo: Partial<RecognitionResultDto>,
|
||
): string {
|
||
return `作为资深药剂师，请分析以下药品的主要成分：
|
||
|
||
**药品信息**：
|
||
- 名称：${productInfo.name}
|
||
- 用途：${productInfo.mainUsage}
|
||
|
||
请以严格的JSON格式返回（不要包含任何markdown标记）：
|
||
{
|
||
"mainIngredients": ["主要成分1", "主要成分2", "主要成分3"]
|
||
}
|
||
|
||
**要求**：
|
||
- mainIngredients 必须是字符串数组，列出药品的主要活性成分
|
||
- 至少包含1-3个主要成分
|
||
- 如果无法确定，返回空数组`;
|
||
}
|
||
|
||
/**
|
||
* 构建副作用分析提示词
|
||
*/
|
||
private buildEffectsAnalysisPrompt(
|
||
productInfo: Partial<RecognitionResultDto>,
|
||
): string {
|
||
return `作为资深药剂师，请分析以下药品的副作用、储存建议和健康建议：
|
||
|
||
**药品信息**：
|
||
- 名称：${productInfo.name}
|
||
- 用途：${productInfo.mainUsage}
|
||
- 成分：${productInfo.mainIngredients?.join('、')}
|
||
|
||
请以严格的JSON格式返回（不要包含任何markdown标记）：
|
||
{
|
||
"sideEffects": ["副作用1", "副作用2", "副作用3"],
|
||
"storageAdvice": ["储存建议1", "储存建议2", "储存建议3"],
|
||
"healthAdvice": ["健康建议1", "健康建议2", "健康建议3"]
|
||
}
|
||
|
||
**要求**：
|
||
- 所有字段都是字符串数组
|
||
- sideEffects: 列出常见和严重的副作用，至少3项
|
||
- storageAdvice: 提供正确的储存方法，至少2项
|
||
- healthAdvice: 给出配合用药的生活建议，至少3项
|
||
- 如果无法确定，返回空数组`;
|
||
}
|
||
|
||
/**
|
||
* 解析JSON响应
|
||
*/
|
||
private parseJsonResponse(content: string): any {
|
||
try {
|
||
// 移除可能的 markdown 代码块标记
|
||
let jsonString = content.trim();
|
||
const jsonMatch = content.match(/```json\s*([\s\S]*?)\s*```/);
|
||
if (jsonMatch) {
|
||
jsonString = jsonMatch[1];
|
||
} else {
|
||
// 尝试提取第一个 { 到最后一个 }
|
||
const firstBrace = content.indexOf('{');
|
||
const lastBrace = content.lastIndexOf('}');
|
||
if (firstBrace !== -1 && lastBrace !== -1) {
|
||
jsonString = content.substring(firstBrace, lastBrace + 1);
|
||
}
|
||
}
|
||
|
||
return JSON.parse(jsonString);
|
||
} catch (error) {
|
||
this.logger.error(
|
||
`解析JSON响应失败: ${error instanceof Error ? error.message : String(error)}, Content: ${content}`,
|
||
);
|
||
throw new Error('AI响应格式错误，无法解析');
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 更新任务状态
|
||
*/
|
||
private async updateTaskStatus(
|
||
taskId: string,
|
||
status: RecognitionStatusEnum,
|
||
currentStep: string,
|
||
progress: number,
|
||
): Promise<void> {
|
||
await this.taskModel.update(
|
||
{
|
||
status,
|
||
currentStep,
|
||
progress,
|
||
},
|
||
{
|
||
where: { id: taskId },
|
||
},
|
||
);
|
||
}
|
||
|
||
/**
|
||
* 完成任务
|
||
*/
|
||
private async completeTask(
|
||
taskId: string,
|
||
result: RecognitionResultDto,
|
||
): Promise<void> {
|
||
await this.taskModel.update(
|
||
{
|
||
status: RecognitionStatusEnum.COMPLETED,
|
||
currentStep: RECOGNITION_STATUS_DESCRIPTIONS[RecognitionStatusEnum.COMPLETED],
|
||
progress: 100,
|
||
recognitionResult: JSON.stringify(result),
|
||
completedAt: new Date(),
|
||
},
|
||
{
|
||
where: { id: taskId },
|
||
},
|
||
);
|
||
}
|
||
|
||
/**
|
||
* 任务失败
|
||
*/
|
||
private async failTask(taskId: string, errorMessage: string): Promise<void> {
|
||
await this.taskModel.update(
|
||
{
|
||
status: RecognitionStatusEnum.FAILED,
|
||
currentStep: RECOGNITION_STATUS_DESCRIPTIONS[RecognitionStatusEnum.FAILED],
|
||
progress: 0,
|
||
errorMessage,
|
||
completedAt: new Date(),
|
||
},
|
||
{
|
||
where: { id: taskId },
|
||
},
|
||
);
|
||
}
|
||
} |